Skip to contentSkip to conversation

(Re-)introducing pvops for x86_64 - Consolidation part

PATCH New Glauber de Oliveira Costa 1 participant 15 messages Oct 31, 2007

Subsystems (5)
x86 KVM virtio core Xen hypervisor x86 +1 more subsystems
lore
Share & tools
lore

Glauber de Oliveira Costa

author
#0 lore raw

cover (Re-)introducing pvops for x86_64 - Consolidation part

Hi folks,

Here is the result of the latest work on the pvops front, after the x86 arch merge. From the functionality point of view, almost nothing was changed, except for proper vsmp support - which was discussed, but not implemented before - and the introduction of smp_ops in x86_64, which eased the merging of the smp header.

Speaking of the merge, a significant part (although not majority) of this work is merging things that was not possible before, due to the lack of paravirt for x86_64. What was done then, can be thought as a "lookahead-merge", which is a normal code, but just taking into account that we'll have support for paravirt_ops in x86_64 in the future. Note for that files, there can be room left for more integration. This is just the first, not the final step.

For this reason, I'm splitting this series in two. The first one, which will follow this message, are really just integration patches. Besides making review even easier, we believe that those could get into the .24 time frame, just like all the other merge patches that are already going in.

In a separate series, you'll get the actual pvops implementation.

That said, have fun!

-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/

author
#1 lore raw

PATCH 1/7 irqflags consolidation

This patch consolidates the irqflags include files containing common paravirt definitions. The native definition for interrupt handling, halt, and such, are the same for 32 and 64 bit, and they are kept in irqflags.h. The differences are split in the arch-specific files.

The syscall function, irq_enable_sysexit, has a very specific i386 naming, and its name is then changed to a more general one.

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/asm-offsets_32.c |    2 +-
 arch/x86/kernel/entry_32.S       |    8 ++--
 arch/x86/kernel/paravirt_32.c    |   10 ++--
 arch/x86/kernel/vmi_32.c         |    4 +-
 arch/x86/xen/enlighten.c         |    2 +-
 include/asm-x86/irqflags.h       |  107 ++++++++++++++++++++++++++++++++++++
 include/asm-x86/irqflags_32.h    |  103 +----------------------------------
 include/asm-x86/irqflags_64.h    |  112 +++++++-------------------------------
 include/asm-x86/paravirt.h       |    9 ++--
 9 files changed, 147 insertions(+), 210 deletions(-)
arch/x86/kernel/asm-offsets_32.c +1 -1
index 0e45981..c1ccfab 100644
@@ -123,7 +123,7 @@ void foo(void)
 	OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable);
 	OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable);
 	OFFSET(PV_CPU_iret, pv_cpu_ops, iret);
-	OFFSET(PV_CPU_irq_enable_sysexit, pv_cpu_ops, irq_enable_sysexit);
+	OFFSET(PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret);
 	OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0);
 #endif
arch/x86/kernel/entry_32.S +4 -4
index dc7f938..d63609d 100644
@@ -58,7 +58,7 @@
  * for paravirtualization.  The following will never clobber any registers:
  *   INTERRUPT_RETURN (aka. "iret")
  *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
- *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
+ *   ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
  *
  * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).

@@ -351,7 +351,7 @@ sysenter_past_esp:
 	xorl %ebp,%ebp
 	TRACE_IRQS_ON
 1:	mov  PT_FS(%esp), %fs
-	ENABLE_INTERRUPTS_SYSEXIT
+	ENABLE_INTERRUPTS_SYSCALL_RET
 	CFI_ENDPROC
 .pushsection .fixup,"ax"
 2:	movl $0,PT_FS(%esp)

@@ -882,10 +882,10 @@ ENTRY(native_iret)
 .previous
 END(native_iret)
 
-ENTRY(native_irq_enable_sysexit)
+ENTRY(native_irq_enable_syscall_ret)
 	sti
 	sysexit
-END(native_irq_enable_sysexit)
+END(native_irq_enable_syscall_ret)
 #endif
 
 KPROBE_ENTRY(int3)
arch/x86/kernel/paravirt_32.c +5 -5
index 6a80d67..04f51d0 100644
@@ -60,7 +60,7 @@ DEF_NATIVE(pv_irq_ops, irq_enable, "sti");
 DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf");
 DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax");
 DEF_NATIVE(pv_cpu_ops, iret, "iret");
-DEF_NATIVE(pv_cpu_ops, irq_enable_sysexit, "sti; sysexit");
+DEF_NATIVE(pv_cpu_ops, irq_enable_syscall_ret, "sti; sysexit");
 DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax");
 DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3");
 DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax");

@@ -88,7 +88,7 @@ static unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
 	SITE(pv_irq_ops, restore_fl);
 	SITE(pv_irq_ops, save_fl);
 	SITE(pv_cpu_ops, iret);
-	SITE(pv_cpu_ops, irq_enable_sysexit);
+	SITE(pv_cpu_ops, irq_enable_syscall_ret);
 	SITE(pv_mmu_ops, read_cr2);
 	SITE(pv_mmu_ops, read_cr3);
 	SITE(pv_mmu_ops, write_cr3);

@@ -186,7 +186,7 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
 		/* If the operation is a nop, then nop the callsite */
 		ret = paravirt_patch_nop();
 	else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
-		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit))
+		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret))
 		/* If operation requires a jmp, then jmp */
 		ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
 	else

@@ -237,7 +237,7 @@ static void native_flush_tlb_single(unsigned long addr)
 
 /* These are in entry.S */
 extern void native_iret(void);
-extern void native_irq_enable_sysexit(void);
+extern void native_irq_enable_syscall_ret(void);
 
 static int __init print_banner(void)
 {

@@ -384,7 +384,7 @@ struct pv_cpu_ops pv_cpu_ops = {
 	.write_idt_entry = write_dt_entry,
 	.load_esp0 = native_load_esp0,
 
-	.irq_enable_sysexit = native_irq_enable_sysexit,
+	.irq_enable_syscall_ret = native_irq_enable_syscall_ret,
 	.iret = native_iret,
 
 	.set_iopl_mask = native_set_iopl_mask,
arch/x86/kernel/vmi_32.c +2 -2
index f02bad6..aacce42 100644
@@ -148,7 +148,7 @@ static unsigned vmi_patch(u8 type, u16 clobbers, void *insns,
 					      insns, eip);
 		case PARAVIRT_PATCH(pv_cpu_ops.iret):
 			return patch_internal(VMI_CALL_IRET, len, insns, eip);
-		case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit):
+		case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret):
 			return patch_internal(VMI_CALL_SYSEXIT, len, insns, eip);
 		default:
 			break;

@@ -870,7 +870,7 @@ static inline int __init activate_vmi(void)
 	 * the backend.  They are performance critical anyway, so requiring
 	 * a patch is not a big problem.
 	 */
-	pv_cpu_ops.irq_enable_sysexit = (void *)0xfeedbab0;
+	pv_cpu_ops.irq_enable_syscall_ret = (void *)0xfeedbab0;
 	pv_cpu_ops.iret = (void *)0xbadbab0;
 
 #ifdef CONFIG_SMP
arch/x86/xen/enlighten.c +1 -1
index 94c39aa..094b915 100644
@@ -953,7 +953,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = {
 	.read_pmc = native_read_pmc,
 
 	.iret = (void *)&hypercall_page[__HYPERVISOR_iret],
-	.irq_enable_sysexit = NULL,  /* never called */
+	.irq_enable_syscall_ret = NULL,  /* never called */
 
 	.load_tr_desc = paravirt_nop,
 	.set_ldt = xen_set_ldt,
include/asm-x86/irqflags.h +107 -0

112 lines not on this page · raw message

include/asm-x86/irqflags_32.h +2 -101

113 lines not on this page · raw message

include/asm-x86/irqflags_64.h +20 -92

134 lines not on this page · raw message

include/asm-x86/paravirt.h +5 -4
index f59d370..d81a361 100644
@@ -121,7 +121,7 @@ struct pv_cpu_ops {
 	u64 (*read_pmc)(void);
 
 	/* These two are jmp to, not actually called. */
-	void (*irq_enable_sysexit)(void);
+	void (*irq_enable_syscall_ret)(void);
 	void (*iret)(void);
 
 	struct pv_lazy_ops lazy_mode;

@@ -1138,9 +1138,10 @@ static inline unsigned long __raw_local_irq_save(void)
 		  call *%cs:pv_irq_ops+PV_IRQ_irq_enable;		\
 		  popl %edx; popl %ecx; popl %eax)
 
-#define ENABLE_INTERRUPTS_SYSEXIT					       \
-	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), CLBR_NONE,\
-		  jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_sysexit)
+#define ENABLE_INTERRUPTS_SYSCALL_RET					\
+	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_syscall_ret),\
+		  CLBR_NONE,						\
+		  jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_syscall_ret)
 
 #define GET_CR0_INTO_EAX			\
 	push %ecx; push %edx;			\
Signature · 8 lines
1.4.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems?  Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
author
#2 lore raw

PATCH 2/7 consolidate spinlock.h

The cli and sti instructions need to be replaced by paravirt hooks. For the i386 architecture, this is already done. The code requirements aren't much different from x86_64 POV, so this part is consolidated in the common header

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 include/asm-x86/spinlock.h    |   14 ++++++++++++++
 include/asm-x86/spinlock_32.h |    9 ---------
 include/asm-x86/spinlock_64.h |    8 +++++---
 3 files changed, 19 insertions(+), 12 deletions(-)
include/asm-x86/spinlock.h +14 -0
index d74d85e..e1d555a 100644
@@ -1,5 +1,19 @@
+#ifndef _X86_SPINLOCK_H_
+#define _X86_SPINLOCK_H_
+
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#else
+#define CLI_STRING	"cli"
+#define STI_STRING	"sti"
+#define CLI_STI_CLOBBERS
+#define CLI_STI_INPUT_ARGS
+#endif /* CONFIG_PARAVIRT */
+
 #ifdef CONFIG_X86_32
 # include "spinlock_32.h"
 #else
 # include "spinlock_64.h"
 #endif
+
+#endif
include/asm-x86/spinlock_32.h +0 -9
index d3bcebe..ebbf371 100644
@@ -7,15 +7,6 @@
 #include <asm/processor.h>
 #include <linux/compiler.h>
 
-#ifdef CONFIG_PARAVIRT
-#include <asm/paravirt.h>
-#else
-#define CLI_STRING	"cli"
-#define STI_STRING	"sti"
-#define CLI_STI_CLOBBERS
-#define CLI_STI_INPUT_ARGS
-#endif /* CONFIG_PARAVIRT */
-
 /*
  * Your basic SMP spinlocks, allowing only a single CPU anywhere
  *
include/asm-x86/spinlock_64.h +5 -3
index 88bf981..e56b17e 100644
@@ -48,12 +48,12 @@ static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long fla
 		"jns 5f\n"
 		"testl $0x200, %1\n\t"	/* interrupts were disabled? */
 		"jz 4f\n\t"
-	        "sti\n"
+		STI_STRING "\n"
 		"3:\t"
 		"rep;nop\n\t"
 		"cmpl $0, %0\n\t"
 		"jle 3b\n\t"
-		"cli\n\t"
+		CLI_STRING "\n\t"
 		"jmp 1b\n"
 		"4:\t"
 		"rep;nop\n\t"

@@ -61,7 +61,9 @@ static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long fla
 		"jg 1b\n\t"
 		"jmp 4b\n"
 		"5:\n\t"
-		: "+m" (lock->slock) : "r" ((unsigned)flags) : "memory");
+		: "+m" (lock->slock)
+		: "r" ((unsigned)flags) CLI_STI_INPUT_ARGS
+		: "memory" CLI_STI_CLOBBERS);
 }
 #endif
Signature · 1 line
1.4.4.2
author
#3 lore raw

PATCH 3/7 tlb functions consolidation

This patch consolidates part of the tlb handling functions for the x86 architecture. In this approach, we start by the parts actually used for paravirt in i386.

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/smp_64.c      |    5 ++-
 include/asm-x86/tlbflush.h    |   77 +++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/tlbflush_32.h |   77 -----------------------------------------
 include/asm-x86/tlbflush_64.h |   43 +++--------------------
 4 files changed, 85 insertions(+), 117 deletions(-)
arch/x86/kernel/smp_64.c +3 -2
index 03fa6ed..ad063a6 100644
@@ -166,11 +166,12 @@ out:
 	add_pda(irq_tlb_count, 1);
 }
 
-static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
-						unsigned long va)
+void native_flush_tlb_others(const cpumask_t *cpumaskp,
+			     struct mm_struct *mm, unsigned long va)
 {
 	int sender;
 	union smp_flush_state *f;
+	cpumask_t cpumask = *cpumaskp;
 
 	/* Caller has disabled preemption */
 	sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
include/asm-x86/tlbflush.h +77 -0

82 lines not on this page · raw message

include/asm-x86/tlbflush_32.h +0 -77

93 lines not on this page · raw message

include/asm-x86/tlbflush_64.h +5 -38

58 lines not on this page · raw message

Signature · 8 lines
1.4.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems?  Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
author
#4 lore raw

PATCH 4/7 smp x86 consolidation

This patch consolidates part of the pieces of smp for both architectures. (i386 and x86_64). It makes part the calls go through smp_ops. Later on, making the functions themselves have shared code is doable and a logical next step

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/smp_64.c     |   23 ++++++++++----
 arch/x86/kernel/smpboot_64.c |    8 ++--
 include/asm-x86/smp.h        |   66 ++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/smp_32.h     |   58 ------------------------------------
 include/asm-x86/smp_64.h     |    4 --
 5 files changed, 86 insertions(+), 73 deletions(-)
arch/x86/kernel/smp_64.c +16 -7
index ad063a6..9ddcb99 100644
@@ -291,9 +291,9 @@ void flush_tlb_all(void)
  * anything. Worst case is that we lose a reschedule ...
  */
 
-void smp_send_reschedule(int cpu)
+void native_smp_send_reschedule(int cpu)
 {
-	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
+       send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
 }
 
 /*

@@ -388,7 +388,7 @@ __smp_call_function_mask(cpumask_t mask,
  * You must not call this function with disabled interrupts or from a
  * hardware interrupt handler or from a bottom half handler.
  */
-int smp_call_function_mask(cpumask_t mask,
+int native_smp_call_function_mask(cpumask_t mask,
 			   void (*func)(void *), void *info,
 			   int wait)
 {

@@ -402,7 +402,6 @@ int smp_call_function_mask(cpumask_t mask,
 	spin_unlock(&call_lock);
 	return ret;
 }
-EXPORT_SYMBOL(smp_call_function_mask);
 
 /*
  * smp_call_function_single - Run a function on a specific CPU

@@ -418,7 +417,7 @@ EXPORT_SYMBOL(smp_call_function_mask);
  */
 
 int smp_call_function_single (int cpu, void (*func) (void *info), void *info,
-	int nonatomic, int wait)
+			      int nonatomic, int wait)
 {
 	/* prevent preemption and reschedule on another processor */
 	int ret;

@@ -458,7 +457,7 @@ EXPORT_SYMBOL(smp_call_function_single);
  * Actually there are a few legal cases, like panic.
  */
 int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
-			int wait)
+			      int wait)
 {
 	return smp_call_function_mask(cpu_online_map, func, info, wait);
 }

@@ -476,7 +475,7 @@ static void stop_this_cpu(void *dummy)
 		halt();
 } 
 
-void smp_send_stop(void)
+void native_smp_send_stop(void)
 {
 	int nolock;
 	unsigned long flags;

@@ -532,3 +531,13 @@ asmlinkage void smp_call_function_interrupt(void)
 	}
 }
 
+struct smp_ops smp_ops = {
+	.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
+	.smp_prepare_cpus = native_smp_prepare_cpus,
+	.cpu_up = native_cpu_up,
+	.smp_cpus_done = native_smp_cpus_done,
+
+	.smp_send_stop = native_smp_send_stop,
+	.smp_send_reschedule = native_smp_send_reschedule,
+	.smp_call_function_mask = native_smp_call_function_mask,
+};
arch/x86/kernel/smpboot_64.c +4 -4
index 500670c..49036d0 100644
@@ -865,7 +865,7 @@ void __init smp_set_apicids(void)
  * Prepare for SMP bootup.  The MP table or ACPI has been read
  * earlier.  Just do some sanity checking here and enable APIC mode.
  */
-void __init smp_prepare_cpus(unsigned int max_cpus)
+void __init native_smp_prepare_cpus(unsigned int max_cpus)
 {
 	nmi_watchdog_default();
 	current_cpu_data = boot_cpu_data;

@@ -909,7 +909,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 /*
  * Early setup to make printk work.
  */
-void __init smp_prepare_boot_cpu(void)
+void __init native_smp_prepare_boot_cpu(void)
 {
 	int me = smp_processor_id();
 	cpu_set(me, cpu_online_map);

@@ -920,7 +920,7 @@ void __init smp_prepare_boot_cpu(void)
 /*
  * Entry point to boot a CPU.
  */
-int __cpuinit __cpu_up(unsigned int cpu)
+int __cpuinit native_cpu_up(unsigned int cpu)
 {
 	int apicid = cpu_present_to_apicid(cpu);
 	unsigned long flags;

@@ -978,7 +978,7 @@ int __cpuinit __cpu_up(unsigned int cpu)
 /*
  * Finish the SMP boot.
  */
-void __init smp_cpus_done(unsigned int max_cpus)
+void __init native_smp_cpus_done(unsigned int max_cpus)
 {
 	smp_cleanup_boot();
 	setup_ioapic_dest();
include/asm-x86/smp.h +66 -0

71 lines not on this page · raw message

include/asm-x86/smp_32.h +0 -58

64 lines not on this page · raw message

include/asm-x86/smp_64.h +0 -4
index ab612b0..279ff92 100644
@@ -36,9 +36,6 @@ extern volatile unsigned long smp_invalidate_needed;
 extern void lock_ipi_call_lock(void);
 extern void unlock_ipi_call_lock(void);
 extern int smp_num_siblings;
-extern void smp_send_reschedule(int cpu);
-extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *),
-				  void *info, int wait);
 
 /*
  * cpu_sibling_map and cpu_core_map now live

@@ -127,4 +124,3 @@ extern unsigned int boot_cpu_id;
 #define cpu_physical_id(cpu)		boot_cpu_id
 #endif /* !CONFIG_SMP */
 #endif
-
Signature · 8 lines
1.4.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems?  Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
author
#5 lore raw

PATCH 5/7 Add debugreg/load_rsp native hooks

This patch adds native hooks for debugreg handling functions, and for the native load_rsp0 function. The later also have its call sites patched. There's some room for consolidation in the processor*.h headers, and it is done, for paravirt related functions

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/process_64.c   |    2 +-
 arch/x86/kernel/smpboot_64.c   |    2 +-
 include/asm-x86/msr.h          |   67 ------------------
 include/asm-x86/processor.h    |  146 ++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/processor_32.h |  138 +-------------------------------------
 include/asm-x86/processor_64.h |   32 ++++-----
 6 files changed, 165 insertions(+), 222 deletions(-)
arch/x86/kernel/process_64.c +1 -1
index 6309b27..d7da3e6 100644
@@ -596,7 +596,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	/*
 	 * Reload esp0, LDT and the page table pointer:
 	 */
-	tss->rsp0 = next->rsp0;
+	load_esp0(tss, next);
 
 	/* 
 	 * Switch DS and ES.
arch/x86/kernel/smpboot_64.c +1 -1
index 49036d0..ecd00f6 100644
@@ -614,7 +614,7 @@ do_rest:
 	start_rip = setup_trampoline();
 
 	init_rsp = c_idle.idle->thread.rsp;
-	per_cpu(init_tss,cpu).rsp0 = init_rsp;
+	load_esp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
 	initial_code = start_secondary;
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
include/asm-x86/msr.h +0 -67

73 lines not on this page · raw message

include/asm-x86/processor.h +146 -0

151 lines not on this page · raw message

include/asm-x86/processor_32.h +2 -136

157 lines not on this page · raw message

include/asm-x86/processor_64.h +15 -17
index e4f1997..b1e6c5a 100644
@@ -116,21 +116,13 @@ extern unsigned long mmu_cr4_features;
 static inline void set_in_cr4 (unsigned long mask)
 {
 	mmu_cr4_features |= mask;
-	__asm__("movq %%cr4,%%rax\n\t"
-		"orq %0,%%rax\n\t"
-		"movq %%rax,%%cr4\n"
-		: : "irg" (mask)
-		:"ax");
+	write_cr4(read_cr4() | mask);
 }
 
 static inline void clear_in_cr4 (unsigned long mask)
 {
 	mmu_cr4_features &= ~mask;
-	__asm__("movq %%cr4,%%rax\n\t"
-		"andq %0,%%rax\n\t"
-		"movq %%rax,%%cr4\n"
-		: : "irg" (~mask)
-		:"ax");
+	write_cr4(read_cr4() & ~mask);
 }
 
 

@@ -251,6 +243,12 @@ struct thread_struct {
 	.rsp0 = (unsigned long)&init_stack + sizeof(init_stack) \
 }
 
+static inline void native_load_esp0(struct tss_struct *tss,
+				    struct thread_struct *thread)
+{
+	tss->rsp0 = thread->rsp0;
+}
+
 #define INIT_MMAP \
 { &init_mm, 0, 0, NULL, PAGE_SHARED, VM_READ | VM_WRITE | VM_EXEC, 1, NULL, NULL }
 

@@ -266,13 +264,13 @@ struct thread_struct {
 	set_fs(USER_DS);							 \
 } while(0) 
 
-#define get_debugreg(var, register)				\
-		__asm__("movq %%db" #register ", %0"		\
-			:"=r" (var))
-#define set_debugreg(value, register)			\
-		__asm__("movq %0,%%db" #register		\
-			: /* no output */			\
-			:"r" (value))
+extern void native_swapgs(void);
+
+/* We have it for simmetry with i386 code, but we don't really use it */
+static inline void native_set_iopl_mask(unsigned mask)
+{
+
+}
 
 struct task_struct;
 struct mm_struct;
Signature · 8 lines
1.4.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems?  Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
author
#6 lore raw

PATCH 6/7 consolidate apic.h functions

This patch consolidates apic.h functions for i386 and x86_64. notice that we use u32 to be explicit about sizing requirements between them

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/smpboot_64.c |   10 ++++++++--
 include/asm-x86/apic.h       |   38 ++++++++++++++++++++++++++++++++++++++
 include/asm-x86/apic_32.h    |   31 -------------------------------
 include/asm-x86/apic_64.h    |   14 --------------
 4 files changed, 46 insertions(+), 47 deletions(-)
arch/x86/kernel/smpboot_64.c +8 -2
index ecd00f6..53db29d 100644
@@ -370,7 +370,7 @@ void __cpuinit start_secondary(void)
 
 	unlock_ipi_call_lock();
 
-	setup_secondary_APIC_clock();
+	setup_secondary_clock();
 
 	cpu_idle();
 }

@@ -462,6 +462,12 @@ static int __cpuinit wakeup_secondary_via_INIT(int phys_apicid, unsigned int sta
 	num_starts = 2;
 
 	/*
+	 * Paravirt wants a startup IPI hook here to set up the
+	 * target processor state.
+	 */
+	startup_ipi_hook(phys_apicid, (unsigned long) start_rip,
+			(unsigned long) init_rsp);
+	/*
 	 * Run STARTUP IPI loop.
 	 */
 	Dprintk("#startup loops: %d.\n", num_starts);

@@ -903,7 +909,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	 * Set up local APIC timer on boot CPU.
 	 */
 
-	setup_boot_APIC_clock();
+	setup_boot_clock();
 }
 
 /*
include/asm-x86/apic.h +38 -0
index 9fbcc0b..6f61672 100644
@@ -1,5 +1,43 @@
+#ifndef _X86_APIC_H_
+#define _X86_APIC_H_
+#include <asm/apicdef.h>
+#include <asm/fixmap.h>
+
+#ifdef CONFIG_X86_LOCAL_APIC
+/*
+ * Basic functions accessing APICs.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#else
+#define apic_write(reg, v) native_apic_write(reg, v)
+#define apic_write_atomic(reg, v) native_apic_write_atomic(reg, v)
+#define apic_read(reg) native_apic_read(reg)
+#define setup_boot_clock setup_boot_APIC_clock
+#define setup_secondary_clock setup_secondary_APIC_clock
+#endif
+static __inline fastcall void native_apic_write(unsigned long reg,
+						u32 v)
+{
+	*((volatile u32 *)(APIC_BASE+reg)) = v;
+}
+
+static __inline fastcall void native_apic_write_atomic(unsigned long reg,
+						       u32 v)
+{
+	xchg((volatile unsigned long *)(APIC_BASE+reg), v);
+}
+
+static __inline fastcall u32 native_apic_read(unsigned long reg)
+{
+	return *((volatile u32 *)(APIC_BASE+reg));
+}
+
+#endif /* CONFIG_X86_LOCAL_APIC */
+
 #ifdef CONFIG_X86_32
 # include "apic_32.h"
 #else
 # include "apic_64.h"
 #endif
+#endif
include/asm-x86/apic_32.h +0 -31
index 4091b33..dbe51b4 100644
@@ -34,37 +34,6 @@ extern int apic_verbosity;
 extern void generic_apic_probe(void);
 
 #ifdef CONFIG_X86_LOCAL_APIC
-
-/*
- * Basic functions accessing APICs.
- */
-#ifdef CONFIG_PARAVIRT
-#include <asm/paravirt.h>
-#else
-#define apic_write native_apic_write
-#define apic_write_atomic native_apic_write_atomic
-#define apic_read native_apic_read
-#define setup_boot_clock setup_boot_APIC_clock
-#define setup_secondary_clock setup_secondary_APIC_clock
-#endif
-
-static __inline fastcall void native_apic_write(unsigned long reg,
-						unsigned long v)
-{
-	*((volatile unsigned long *)(APIC_BASE+reg)) = v;
-}
-
-static __inline fastcall void native_apic_write_atomic(unsigned long reg,
-						       unsigned long v)
-{
-	xchg((volatile unsigned long *)(APIC_BASE+reg), v);
-}
-
-static __inline fastcall unsigned long native_apic_read(unsigned long reg)
-{
-	return *((volatile unsigned long *)(APIC_BASE+reg));
-}
-
 void apic_wait_icr_idle(void);
 unsigned long safe_apic_wait_icr_idle(void);
 int get_physical_broadcast(void);
include/asm-x86/apic_64.h +0 -14
index 2747a11..aaba5af 100644
@@ -34,20 +34,6 @@ extern int disable_apic_timer;
 
 struct pt_regs;
 
-/*
- * Basic functions accessing APICs.
- */
-
-static __inline void apic_write(unsigned long reg, unsigned int v)
-{
-	*((volatile unsigned int *)(APIC_BASE+reg)) = v;
-}
-
-static __inline unsigned int apic_read(unsigned long reg)
-{
-	return *((volatile unsigned int *)(APIC_BASE+reg));
-}
-
 extern void apic_wait_icr_idle(void);
 extern unsigned int safe_apic_wait_icr_idle(void);
Signature · 8 lines
1.4.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems?  Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
author
#7 lore raw

PATCH 7/7 consolidate msr.h

This patch goes one step forward in consolidating the msr.h header. It shares code between i386 and x86_64, instead of duplicating the code for tsc reading, msr reading/writing, etc.

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/ia32/syscall32.c        |    2 +-
 arch/x86/kernel/setup64.c        |    6 +-
 arch/x86/kernel/tsc_64.c         |   17 +++-
 arch/x86/kernel/vsyscall_64.c    |    4 +-
 arch/x86/vdso/vgetcpu.c          |    4 +-
 include/asm-x86/alternative_32.h |   17 +++-
 include/asm-x86/alternative_64.h |   27 ++++-
 include/asm-x86/msr.h            |  225 ++++++++++----------------------------
 include/asm-x86/tsc.h            |   33 +++++-
 9 files changed, 151 insertions(+), 184 deletions(-)
arch/x86/ia32/syscall32.c +1 -1
index 15013ba..dd1b4a3 100644
@@ -79,5 +79,5 @@ void syscall32_cpu_init(void)
 	checking_wrmsrl(MSR_IA32_SYSENTER_ESP, 0ULL);
 	checking_wrmsrl(MSR_IA32_SYSENTER_EIP, (u64)ia32_sysenter_target);
 
-	wrmsrl(MSR_CSTAR, ia32_cstar_target);
+	wrmsrl(MSR_CSTAR, (u64)ia32_cstar_target);
 }
arch/x86/kernel/setup64.c +3 -3
index 3558ac7..50b7514 100644
@@ -122,7 +122,7 @@ void pda_init(int cpu)
 	asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0)); 
 	/* Memory clobbers used to order PDA accessed */
 	mb();
-	wrmsrl(MSR_GS_BASE, pda);
+	wrmsrl(MSR_GS_BASE, (u64)pda);
 	mb();
 
 	pda->cpunumber = cpu; 

@@ -161,8 +161,8 @@ void syscall_init(void)
 	 * but only a 32bit target. LSTAR sets the 64bit rip. 	 
 	 */ 
 	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32); 
-	wrmsrl(MSR_LSTAR, system_call); 
-	wrmsrl(MSR_CSTAR, ignore_sysret);
+	wrmsrl(MSR_LSTAR, (u64)system_call);
+	wrmsrl(MSR_CSTAR, (u64)ignore_sysret);
 
 #ifdef CONFIG_IA32_EMULATION   		
 	syscall32_cpu_init ();
arch/x86/kernel/tsc_64.c +15 -2
index 9c70af4..4502539 100644
@@ -30,7 +30,7 @@ static unsigned long long cycles_2_ns(unsigned long long cyc)
 	return (cyc * cyc2ns_scale) >> NS_SCALE;
 }
 
-unsigned long long sched_clock(void)
+unsigned long long native_sched_clock(void)
 {
 	unsigned long a = 0;
 

@@ -44,6 +44,19 @@ unsigned long long sched_clock(void)
 	return cycles_2_ns(a);
 }
 
+/* We need to define a real function for sched_clock, to override the
+   weak default version */
+#ifdef CONFIG_PARAVIRT
+unsigned long long sched_clock(void)
+{
+	return paravirt_sched_clock();
+}
+#else
+unsigned long long
+sched_clock(void) __attribute__((alias("native_sched_clock")));
+#endif
+
+
 static int tsc_unstable;
 
 inline int check_tsc_unstable(void)

@@ -256,7 +269,7 @@ static cycle_t read_tsc(void)
 
 static cycle_t __vsyscall_fn vread_tsc(void)
 {
-	cycle_t ret = (cycle_t)get_cycles_sync();
+	cycle_t ret = (cycle_t)vget_cycles_sync();
 	return ret;
 }
arch/x86/kernel/vsyscall_64.c +2 -2
index ad4005c..1425d02 100644
@@ -190,7 +190,7 @@ time_t __vsyscall(1) vtime(time_t *t)
 long __vsyscall(2)
 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 {
-	unsigned int dummy, p;
+	unsigned int p;
 	unsigned long j = 0;
 
 	/* Fast cache - only recompute value once per jiffies and avoid

@@ -205,7 +205,7 @@ vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 		p = tcache->blob[1];
 	} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 		/* Load per CPU data from RDTSCP */
-		rdtscp(dummy, dummy, p);
+		native_read_tscp(&p);
 	} else {
 		/* Load per CPU data from GDT */
 		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
arch/x86/vdso/vgetcpu.c +2 -2
index 91f6e85..61d0def 100644
@@ -15,7 +15,7 @@
 
 long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 {
-	unsigned int dummy, p;
+	unsigned int p;
 	unsigned long j = 0;
 
 	/* Fast cache - only recompute value once per jiffies and avoid

@@ -30,7 +30,7 @@ long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 		p = tcache->blob[1];
 	} else if (*vdso_vgetcpu_mode == VGETCPU_RDTSCP) {
 		/* Load per CPU data from RDTSCP */
-		rdtscp(dummy, dummy, p);
+		native_read_tscp(&p);
 	} else {
 		/* Load per CPU data from GDT */
 		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
include/asm-x86/alternative_32.h +16 -1
index bda6c81..1ed7708 100644
@@ -101,7 +101,22 @@ static inline void alternatives_smp_switch(int smp) {}
  * use this macro(s) if you need more than one output parameter
  * in alternative_io
  */
-#define ASM_OUTPUT2(a, b) a, b
+#define ASM_OUTPUT2(a, b...) a, b
+
+#define fixup_section(code, fixup, output, input...)		\
+		asm volatile("2: " code "\n"                    \
+			     "1:\n\t"                           \
+			     ".section .fixup,\"ax\"\n\t"       \
+			     "3: "  fixup "\n\t"                \
+			     "jmp 1b\n\t"                       \
+			     ".previous\n\t"                    \
+			     ".section __ex_table,\"a\"\n"      \
+			     "   .align 4\n\t"                  \
+			     "   .long  2b,3b\n\t"              \
+			     ".previous"                        \
+			     : output                           \
+			     : input)
+
 
 /*
  * Alternative inline assembly for SMP.
include/asm-x86/alternative_64.h +21 -6
index ab161e8..f080b69 100644
@@ -141,14 +141,29 @@ static inline void alternatives_smp_switch(int smp) {}
  * use this macro(s) if you need more than one output parameter
  * in alternative_io
  */
-#define ASM_OUTPUT2(a, b) a, b
-
-struct paravirt_patch;
+#define ASM_OUTPUT2(a, b...) a, b
+
+#define fixup_section(code, fixup, output, input...)   \
+		asm volatile("2: " code "\n"                    \
+			     "1:\n\t"                           \
+			     ".section .fixup,\"ax\"\n\t"       \
+			     "3: " fixup "\n\t"                 \
+			     " jmp 1b\n\t"                      \
+			     ".previous\n\t"                    \
+			     ".section __ex_table,\"a\"\n"      \
+			     "   .align 8\n\t"                  \
+			     "   .quad  2b,3b\n\t"              \
+			     ".previous"                        \
+			     : output                           \
+			     : input)
+
+struct paravirt_patch_site;
 #ifdef CONFIG_PARAVIRT
-void apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end);
+void apply_paravirt(struct paravirt_patch_site *start,
+		    struct paravirt_patch_site *end);
 #else
-static inline void
-apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end)
+static inline void apply_paravirt(struct paravirt_patch_site *start,
+				  struct paravirt_patch_site *end)
 {}
 #define __parainstructions NULL
 #define __parainstructions_end NULL
include/asm-x86/msr.h +60 -165

297 lines not on this page · raw message

include/asm-x86/tsc.h +31 -2
index 6baab30..f6460ba 100644
@@ -33,7 +33,7 @@ static inline cycles_t get_cycles(void)
 }
 
 /* Like get_cycles, but make sure the CPU is synchronized. */
-static __always_inline cycles_t get_cycles_sync(void)
+static __always_inline cycles_t __get_cycles_sync(void)
 {
 	unsigned long long ret;
 	unsigned eax, edx;

@@ -55,11 +55,40 @@ static __always_inline cycles_t get_cycles_sync(void)
 	 */
 	alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC,
 			  "=a" (eax), "0" (1) : "ebx","ecx","edx","memory");
-	rdtscll(ret);
 
+	return 0;
+}
+
+static __always_inline cycles_t get_cycles_sync(void)
+{
+	unsigned long long ret;
+	ret = __get_cycles_sync();
+	if (!ret)
+		rdtscll(ret);
 	return ret;
 }
 
+#ifdef CONFIG_PARAVIRT
+/*
+ * For paravirt guests, some functionalities are executed through function
+ * pointers in the various pvops structures.
+ * These function pointers exist inside the kernel and can not
+ * be accessed by user space. To avoid this, we make a copy of the
+ * get_cycles_sync (called in kernel) but force the use of native_read_tsc.
+ * Ideally, the guest should set up it's own clock and vread
+ */
+static __always_inline long long vget_cycles_sync(void)
+{
+	unsigned long long ret;
+	ret = __get_cycles_sync();
+	if (!ret)
+		ret = native_read_tsc();
+	return ret;
+}
+#else
+# define vget_cycles_sync() get_cycles_sync()
+#endif
+
 extern void tsc_init(void);
 extern void mark_tsc_unstable(char *reason);
 extern int unsynchronized_tsc(void);
Signature · 8 lines
1.4.4.2


-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems?  Stop.
Now Search log events and configuration files using AJAX and a browser.
Download your FREE copy of Splunk now >> http://get.splunk.com/
author
#8 lore raw

PATCH 7/7 consolidate msr.h

In reply to Glauber de Oliveira Costa

This patch goes one step forward in consolidating the msr.h header. It shares code between i386 and x86_64, instead of duplicating the code for tsc reading, msr reading/writing, etc.

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/ia32/syscall32.c        |    2 +-
 arch/x86/kernel/setup64.c        |    6 +-
 arch/x86/kernel/tsc_64.c         |   17 +++-
 arch/x86/kernel/vsyscall_64.c    |    4 +-
 arch/x86/vdso/vgetcpu.c          |    4 +-
 include/asm-x86/alternative_32.h |   17 +++-
 include/asm-x86/alternative_64.h |   27 ++++-
 include/asm-x86/msr.h            |  225 ++++++++++----------------------------
 include/asm-x86/tsc.h            |   33 +++++-
 9 files changed, 151 insertions(+), 184 deletions(-)
arch/x86/ia32/syscall32.c +1 -1
index 15013ba..dd1b4a3 100644
@@ -79,5 +79,5 @@ void syscall32_cpu_init(void)
 	checking_wrmsrl(MSR_IA32_SYSENTER_ESP, 0ULL);
 	checking_wrmsrl(MSR_IA32_SYSENTER_EIP, (u64)ia32_sysenter_target);
 
-	wrmsrl(MSR_CSTAR, ia32_cstar_target);
+	wrmsrl(MSR_CSTAR, (u64)ia32_cstar_target);
 }
arch/x86/kernel/setup64.c +3 -3
index 3558ac7..50b7514 100644
@@ -122,7 +122,7 @@ void pda_init(int cpu)
 	asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0)); 
 	/* Memory clobbers used to order PDA accessed */
 	mb();
-	wrmsrl(MSR_GS_BASE, pda);
+	wrmsrl(MSR_GS_BASE, (u64)pda);
 	mb();
 
 	pda->cpunumber = cpu; 

@@ -161,8 +161,8 @@ void syscall_init(void)
 	 * but only a 32bit target. LSTAR sets the 64bit rip. 	 
 	 */ 
 	wrmsrl(MSR_STAR,  ((u64)__USER32_CS)<<48  | ((u64)__KERNEL_CS)<<32); 
-	wrmsrl(MSR_LSTAR, system_call); 
-	wrmsrl(MSR_CSTAR, ignore_sysret);
+	wrmsrl(MSR_LSTAR, (u64)system_call);
+	wrmsrl(MSR_CSTAR, (u64)ignore_sysret);
 
 #ifdef CONFIG_IA32_EMULATION   		
 	syscall32_cpu_init ();
arch/x86/kernel/tsc_64.c +15 -2
index 9c70af4..4502539 100644
@@ -30,7 +30,7 @@ static unsigned long long cycles_2_ns(unsigned long long cyc)
 	return (cyc * cyc2ns_scale) >> NS_SCALE;
 }
 
-unsigned long long sched_clock(void)
+unsigned long long native_sched_clock(void)
 {
 	unsigned long a = 0;
 

@@ -44,6 +44,19 @@ unsigned long long sched_clock(void)
 	return cycles_2_ns(a);
 }
 
+/* We need to define a real function for sched_clock, to override the
+   weak default version */
+#ifdef CONFIG_PARAVIRT
+unsigned long long sched_clock(void)
+{
+	return paravirt_sched_clock();
+}
+#else
+unsigned long long
+sched_clock(void) __attribute__((alias("native_sched_clock")));
+#endif
+
+
 static int tsc_unstable;
 
 inline int check_tsc_unstable(void)

@@ -256,7 +269,7 @@ static cycle_t read_tsc(void)
 
 static cycle_t __vsyscall_fn vread_tsc(void)
 {
-	cycle_t ret = (cycle_t)get_cycles_sync();
+	cycle_t ret = (cycle_t)vget_cycles_sync();
 	return ret;
 }
arch/x86/kernel/vsyscall_64.c +2 -2
index ad4005c..1425d02 100644
@@ -190,7 +190,7 @@ time_t __vsyscall(1) vtime(time_t *t)
 long __vsyscall(2)
 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 {
-	unsigned int dummy, p;
+	unsigned int p;
 	unsigned long j = 0;
 
 	/* Fast cache - only recompute value once per jiffies and avoid

@@ -205,7 +205,7 @@ vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 		p = tcache->blob[1];
 	} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 		/* Load per CPU data from RDTSCP */
-		rdtscp(dummy, dummy, p);
+		native_read_tscp(&p);
 	} else {
 		/* Load per CPU data from GDT */
 		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
arch/x86/vdso/vgetcpu.c +2 -2
index 91f6e85..61d0def 100644
@@ -15,7 +15,7 @@
 
 long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 {
-	unsigned int dummy, p;
+	unsigned int p;
 	unsigned long j = 0;
 
 	/* Fast cache - only recompute value once per jiffies and avoid

@@ -30,7 +30,7 @@ long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 		p = tcache->blob[1];
 	} else if (*vdso_vgetcpu_mode == VGETCPU_RDTSCP) {
 		/* Load per CPU data from RDTSCP */
-		rdtscp(dummy, dummy, p);
+		native_read_tscp(&p);
 	} else {
 		/* Load per CPU data from GDT */
 		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
include/asm-x86/alternative_32.h +16 -1
index bda6c81..1ed7708 100644
@@ -101,7 +101,22 @@ static inline void alternatives_smp_switch(int smp) {}
  * use this macro(s) if you need more than one output parameter
  * in alternative_io
  */
-#define ASM_OUTPUT2(a, b) a, b
+#define ASM_OUTPUT2(a, b...) a, b
+
+#define fixup_section(code, fixup, output, input...)		\
+		asm volatile("2: " code "\n"                    \
+			     "1:\n\t"                           \
+			     ".section .fixup,\"ax\"\n\t"       \
+			     "3: "  fixup "\n\t"                \
+			     "jmp 1b\n\t"                       \
+			     ".previous\n\t"                    \
+			     ".section __ex_table,\"a\"\n"      \
+			     "   .align 4\n\t"                  \
+			     "   .long  2b,3b\n\t"              \
+			     ".previous"                        \
+			     : output                           \
+			     : input)
+
 
 /*
  * Alternative inline assembly for SMP.
include/asm-x86/alternative_64.h +21 -6
index ab161e8..f080b69 100644
@@ -141,14 +141,29 @@ static inline void alternatives_smp_switch(int smp) {}
  * use this macro(s) if you need more than one output parameter
  * in alternative_io
  */
-#define ASM_OUTPUT2(a, b) a, b
-
-struct paravirt_patch;
+#define ASM_OUTPUT2(a, b...) a, b
+
+#define fixup_section(code, fixup, output, input...)   \
+		asm volatile("2: " code "\n"                    \
+			     "1:\n\t"                           \
+			     ".section .fixup,\"ax\"\n\t"       \
+			     "3: " fixup "\n\t"                 \
+			     " jmp 1b\n\t"                      \
+			     ".previous\n\t"                    \
+			     ".section __ex_table,\"a\"\n"      \
+			     "   .align 8\n\t"                  \
+			     "   .quad  2b,3b\n\t"              \
+			     ".previous"                        \
+			     : output                           \
+			     : input)
+
+struct paravirt_patch_site;
 #ifdef CONFIG_PARAVIRT
-void apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end);
+void apply_paravirt(struct paravirt_patch_site *start,
+		    struct paravirt_patch_site *end);
 #else
-static inline void
-apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end)
+static inline void apply_paravirt(struct paravirt_patch_site *start,
+				  struct paravirt_patch_site *end)
 {}
 #define __parainstructions NULL
 #define __parainstructions_end NULL
include/asm-x86/msr.h +60 -165

297 lines not on this page · raw message

include/asm-x86/tsc.h +31 -2
index 6baab30..f6460ba 100644
@@ -33,7 +33,7 @@ static inline cycles_t get_cycles(void)
 }
 
 /* Like get_cycles, but make sure the CPU is synchronized. */
-static __always_inline cycles_t get_cycles_sync(void)
+static __always_inline cycles_t __get_cycles_sync(void)
 {
 	unsigned long long ret;
 	unsigned eax, edx;

@@ -55,11 +55,40 @@ static __always_inline cycles_t get_cycles_sync(void)
 	 */
 	alternative_io("cpuid", ASM_NOP2, X86_FEATURE_SYNC_RDTSC,
 			  "=a" (eax), "0" (1) : "ebx","ecx","edx","memory");
-	rdtscll(ret);
 
+	return 0;
+}
+
+static __always_inline cycles_t get_cycles_sync(void)
+{
+	unsigned long long ret;
+	ret = __get_cycles_sync();
+	if (!ret)
+		rdtscll(ret);
 	return ret;
 }
 
+#ifdef CONFIG_PARAVIRT
+/*
+ * For paravirt guests, some functionalities are executed through function
+ * pointers in the various pvops structures.
+ * These function pointers exist inside the kernel and can not
+ * be accessed by user space. To avoid this, we make a copy of the
+ * get_cycles_sync (called in kernel) but force the use of native_read_tsc.
+ * Ideally, the guest should set up it's own clock and vread
+ */
+static __always_inline long long vget_cycles_sync(void)
+{
+	unsigned long long ret;
+	ret = __get_cycles_sync();
+	if (!ret)
+		ret = native_read_tsc();
+	return ret;
+}
+#else
+# define vget_cycles_sync() get_cycles_sync()
+#endif
+
 extern void tsc_init(void);
 extern void mark_tsc_unstable(char *reason);
 extern int unsynchronized_tsc(void);
Signature · 1 line
1.4.4.2
author
#9 lore raw

PATCH 6/7 consolidate apic.h functions

In reply to Glauber de Oliveira Costa

This patch consolidates apic.h functions for i386 and x86_64. notice that we use u32 to be explicit about sizing requirements between them

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/smpboot_64.c |   10 ++++++++--
 include/asm-x86/apic.h       |   38 ++++++++++++++++++++++++++++++++++++++
 include/asm-x86/apic_32.h    |   31 -------------------------------
 include/asm-x86/apic_64.h    |   14 --------------
 4 files changed, 46 insertions(+), 47 deletions(-)
arch/x86/kernel/smpboot_64.c +8 -2
index ecd00f6..53db29d 100644
@@ -370,7 +370,7 @@ void __cpuinit start_secondary(void)
 
 	unlock_ipi_call_lock();
 
-	setup_secondary_APIC_clock();
+	setup_secondary_clock();
 
 	cpu_idle();
 }

@@ -462,6 +462,12 @@ static int __cpuinit wakeup_secondary_via_INIT(int phys_apicid, unsigned int sta
 	num_starts = 2;
 
 	/*
+	 * Paravirt wants a startup IPI hook here to set up the
+	 * target processor state.
+	 */
+	startup_ipi_hook(phys_apicid, (unsigned long) start_rip,
+			(unsigned long) init_rsp);
+	/*
 	 * Run STARTUP IPI loop.
 	 */
 	Dprintk("#startup loops: %d.\n", num_starts);

@@ -903,7 +909,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
 	 * Set up local APIC timer on boot CPU.
 	 */
 
-	setup_boot_APIC_clock();
+	setup_boot_clock();
 }
 
 /*
include/asm-x86/apic.h +38 -0
index 9fbcc0b..6f61672 100644
@@ -1,5 +1,43 @@
+#ifndef _X86_APIC_H_
+#define _X86_APIC_H_
+#include <asm/apicdef.h>
+#include <asm/fixmap.h>
+
+#ifdef CONFIG_X86_LOCAL_APIC
+/*
+ * Basic functions accessing APICs.
+ */
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#else
+#define apic_write(reg, v) native_apic_write(reg, v)
+#define apic_write_atomic(reg, v) native_apic_write_atomic(reg, v)
+#define apic_read(reg) native_apic_read(reg)
+#define setup_boot_clock setup_boot_APIC_clock
+#define setup_secondary_clock setup_secondary_APIC_clock
+#endif
+static __inline fastcall void native_apic_write(unsigned long reg,
+						u32 v)
+{
+	*((volatile u32 *)(APIC_BASE+reg)) = v;
+}
+
+static __inline fastcall void native_apic_write_atomic(unsigned long reg,
+						       u32 v)
+{
+	xchg((volatile unsigned long *)(APIC_BASE+reg), v);
+}
+
+static __inline fastcall u32 native_apic_read(unsigned long reg)
+{
+	return *((volatile u32 *)(APIC_BASE+reg));
+}
+
+#endif /* CONFIG_X86_LOCAL_APIC */
+
 #ifdef CONFIG_X86_32
 # include "apic_32.h"
 #else
 # include "apic_64.h"
 #endif
+#endif
include/asm-x86/apic_32.h +0 -31
index 4091b33..dbe51b4 100644
@@ -34,37 +34,6 @@ extern int apic_verbosity;
 extern void generic_apic_probe(void);
 
 #ifdef CONFIG_X86_LOCAL_APIC
-
-/*
- * Basic functions accessing APICs.
- */
-#ifdef CONFIG_PARAVIRT
-#include <asm/paravirt.h>
-#else
-#define apic_write native_apic_write
-#define apic_write_atomic native_apic_write_atomic
-#define apic_read native_apic_read
-#define setup_boot_clock setup_boot_APIC_clock
-#define setup_secondary_clock setup_secondary_APIC_clock
-#endif
-
-static __inline fastcall void native_apic_write(unsigned long reg,
-						unsigned long v)
-{
-	*((volatile unsigned long *)(APIC_BASE+reg)) = v;
-}
-
-static __inline fastcall void native_apic_write_atomic(unsigned long reg,
-						       unsigned long v)
-{
-	xchg((volatile unsigned long *)(APIC_BASE+reg), v);
-}
-
-static __inline fastcall unsigned long native_apic_read(unsigned long reg)
-{
-	return *((volatile unsigned long *)(APIC_BASE+reg));
-}
-
 void apic_wait_icr_idle(void);
 unsigned long safe_apic_wait_icr_idle(void);
 int get_physical_broadcast(void);
include/asm-x86/apic_64.h +0 -14
index 2747a11..aaba5af 100644
@@ -34,20 +34,6 @@ extern int disable_apic_timer;
 
 struct pt_regs;
 
-/*
- * Basic functions accessing APICs.
- */
-
-static __inline void apic_write(unsigned long reg, unsigned int v)
-{
-	*((volatile unsigned int *)(APIC_BASE+reg)) = v;
-}
-
-static __inline unsigned int apic_read(unsigned long reg)
-{
-	return *((volatile unsigned int *)(APIC_BASE+reg));
-}
-
 extern void apic_wait_icr_idle(void);
 extern unsigned int safe_apic_wait_icr_idle(void);
Signature · 1 line
1.4.4.2
author
#10 lore raw

PATCH 5/7 Add debugreg/load_rsp native hooks

In reply to Glauber de Oliveira Costa

This patch adds native hooks for debugreg handling functions, and for the native load_rsp0 function. The later also have its call sites patched. There's some room for consolidation in the processor*.h headers, and it is done, for paravirt related functions

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/process_64.c   |    2 +-
 arch/x86/kernel/smpboot_64.c   |    2 +-
 include/asm-x86/msr.h          |   67 ------------------
 include/asm-x86/processor.h    |  146 ++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/processor_32.h |  138 +-------------------------------------
 include/asm-x86/processor_64.h |   32 ++++-----
 6 files changed, 165 insertions(+), 222 deletions(-)
arch/x86/kernel/process_64.c +1 -1
index 6309b27..d7da3e6 100644
@@ -596,7 +596,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	/*
 	 * Reload esp0, LDT and the page table pointer:
 	 */
-	tss->rsp0 = next->rsp0;
+	load_esp0(tss, next);
 
 	/* 
 	 * Switch DS and ES.
arch/x86/kernel/smpboot_64.c +1 -1
index 49036d0..ecd00f6 100644
@@ -614,7 +614,7 @@ do_rest:
 	start_rip = setup_trampoline();
 
 	init_rsp = c_idle.idle->thread.rsp;
-	per_cpu(init_tss,cpu).rsp0 = init_rsp;
+	load_esp0(&per_cpu(init_tss, cpu), &c_idle.idle->thread);
 	initial_code = start_secondary;
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
include/asm-x86/msr.h +0 -67

73 lines not on this page · raw message

include/asm-x86/processor.h +146 -0

151 lines not on this page · raw message

include/asm-x86/processor_32.h +2 -136

157 lines not on this page · raw message

include/asm-x86/processor_64.h +15 -17
index e4f1997..b1e6c5a 100644
@@ -116,21 +116,13 @@ extern unsigned long mmu_cr4_features;
 static inline void set_in_cr4 (unsigned long mask)
 {
 	mmu_cr4_features |= mask;
-	__asm__("movq %%cr4,%%rax\n\t"
-		"orq %0,%%rax\n\t"
-		"movq %%rax,%%cr4\n"
-		: : "irg" (mask)
-		:"ax");
+	write_cr4(read_cr4() | mask);
 }
 
 static inline void clear_in_cr4 (unsigned long mask)
 {
 	mmu_cr4_features &= ~mask;
-	__asm__("movq %%cr4,%%rax\n\t"
-		"andq %0,%%rax\n\t"
-		"movq %%rax,%%cr4\n"
-		: : "irg" (~mask)
-		:"ax");
+	write_cr4(read_cr4() & ~mask);
 }
 
 

@@ -251,6 +243,12 @@ struct thread_struct {
 	.rsp0 = (unsigned long)&init_stack + sizeof(init_stack) \
 }
 
+static inline void native_load_esp0(struct tss_struct *tss,
+				    struct thread_struct *thread)
+{
+	tss->rsp0 = thread->rsp0;
+}
+
 #define INIT_MMAP \
 { &init_mm, 0, 0, NULL, PAGE_SHARED, VM_READ | VM_WRITE | VM_EXEC, 1, NULL, NULL }
 

@@ -266,13 +264,13 @@ struct thread_struct {
 	set_fs(USER_DS);							 \
 } while(0) 
 
-#define get_debugreg(var, register)				\
-		__asm__("movq %%db" #register ", %0"		\
-			:"=r" (var))
-#define set_debugreg(value, register)			\
-		__asm__("movq %0,%%db" #register		\
-			: /* no output */			\
-			:"r" (value))
+extern void native_swapgs(void);
+
+/* We have it for simmetry with i386 code, but we don't really use it */
+static inline void native_set_iopl_mask(unsigned mask)
+{
+
+}
 
 struct task_struct;
 struct mm_struct;
Signature · 1 line
1.4.4.2
author
#11 lore raw

PATCH 4/7 smp x86 consolidation

In reply to Glauber de Oliveira Costa

This patch consolidates part of the pieces of smp for both architectures. (i386 and x86_64). It makes part the calls go through smp_ops. Later on, making the functions themselves have shared code is doable and a logical next step

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/smp_64.c     |   23 ++++++++++----
 arch/x86/kernel/smpboot_64.c |    8 ++--
 include/asm-x86/smp.h        |   66 ++++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/smp_32.h     |   58 ------------------------------------
 include/asm-x86/smp_64.h     |    4 --
 5 files changed, 86 insertions(+), 73 deletions(-)
arch/x86/kernel/smp_64.c +16 -7
index ad063a6..9ddcb99 100644
@@ -291,9 +291,9 @@ void flush_tlb_all(void)
  * anything. Worst case is that we lose a reschedule ...
  */
 
-void smp_send_reschedule(int cpu)
+void native_smp_send_reschedule(int cpu)
 {
-	send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
+       send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR);
 }
 
 /*

@@ -388,7 +388,7 @@ __smp_call_function_mask(cpumask_t mask,
  * You must not call this function with disabled interrupts or from a
  * hardware interrupt handler or from a bottom half handler.
  */
-int smp_call_function_mask(cpumask_t mask,
+int native_smp_call_function_mask(cpumask_t mask,
 			   void (*func)(void *), void *info,
 			   int wait)
 {

@@ -402,7 +402,6 @@ int smp_call_function_mask(cpumask_t mask,
 	spin_unlock(&call_lock);
 	return ret;
 }
-EXPORT_SYMBOL(smp_call_function_mask);
 
 /*
  * smp_call_function_single - Run a function on a specific CPU

@@ -418,7 +417,7 @@ EXPORT_SYMBOL(smp_call_function_mask);
  */
 
 int smp_call_function_single (int cpu, void (*func) (void *info), void *info,
-	int nonatomic, int wait)
+			      int nonatomic, int wait)
 {
 	/* prevent preemption and reschedule on another processor */
 	int ret;

@@ -458,7 +457,7 @@ EXPORT_SYMBOL(smp_call_function_single);
  * Actually there are a few legal cases, like panic.
  */
 int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
-			int wait)
+			      int wait)
 {
 	return smp_call_function_mask(cpu_online_map, func, info, wait);
 }

@@ -476,7 +475,7 @@ static void stop_this_cpu(void *dummy)
 		halt();
 } 
 
-void smp_send_stop(void)
+void native_smp_send_stop(void)
 {
 	int nolock;
 	unsigned long flags;

@@ -532,3 +531,13 @@ asmlinkage void smp_call_function_interrupt(void)
 	}
 }
 
+struct smp_ops smp_ops = {
+	.smp_prepare_boot_cpu = native_smp_prepare_boot_cpu,
+	.smp_prepare_cpus = native_smp_prepare_cpus,
+	.cpu_up = native_cpu_up,
+	.smp_cpus_done = native_smp_cpus_done,
+
+	.smp_send_stop = native_smp_send_stop,
+	.smp_send_reschedule = native_smp_send_reschedule,
+	.smp_call_function_mask = native_smp_call_function_mask,
+};
arch/x86/kernel/smpboot_64.c +4 -4
index 500670c..49036d0 100644
@@ -865,7 +865,7 @@ void __init smp_set_apicids(void)
  * Prepare for SMP bootup.  The MP table or ACPI has been read
  * earlier.  Just do some sanity checking here and enable APIC mode.
  */
-void __init smp_prepare_cpus(unsigned int max_cpus)
+void __init native_smp_prepare_cpus(unsigned int max_cpus)
 {
 	nmi_watchdog_default();
 	current_cpu_data = boot_cpu_data;

@@ -909,7 +909,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
 /*
  * Early setup to make printk work.
  */
-void __init smp_prepare_boot_cpu(void)
+void __init native_smp_prepare_boot_cpu(void)
 {
 	int me = smp_processor_id();
 	cpu_set(me, cpu_online_map);

@@ -920,7 +920,7 @@ void __init smp_prepare_boot_cpu(void)
 /*
  * Entry point to boot a CPU.
  */
-int __cpuinit __cpu_up(unsigned int cpu)
+int __cpuinit native_cpu_up(unsigned int cpu)
 {
 	int apicid = cpu_present_to_apicid(cpu);
 	unsigned long flags;

@@ -978,7 +978,7 @@ int __cpuinit __cpu_up(unsigned int cpu)
 /*
  * Finish the SMP boot.
  */
-void __init smp_cpus_done(unsigned int max_cpus)
+void __init native_smp_cpus_done(unsigned int max_cpus)
 {
 	smp_cleanup_boot();
 	setup_ioapic_dest();
include/asm-x86/smp.h +66 -0

71 lines not on this page · raw message

include/asm-x86/smp_32.h +0 -58

64 lines not on this page · raw message

include/asm-x86/smp_64.h +0 -4
index ab612b0..279ff92 100644
@@ -36,9 +36,6 @@ extern volatile unsigned long smp_invalidate_needed;
 extern void lock_ipi_call_lock(void);
 extern void unlock_ipi_call_lock(void);
 extern int smp_num_siblings;
-extern void smp_send_reschedule(int cpu);
-extern int smp_call_function_mask(cpumask_t mask, void (*func)(void *),
-				  void *info, int wait);
 
 /*
  * cpu_sibling_map and cpu_core_map now live

@@ -127,4 +124,3 @@ extern unsigned int boot_cpu_id;
 #define cpu_physical_id(cpu)		boot_cpu_id
 #endif /* !CONFIG_SMP */
 #endif
-
Signature · 1 line
1.4.4.2
author
#12 lore raw

PATCH 3/7 tlb functions consolidation

In reply to Glauber de Oliveira Costa

This patch consolidates part of the tlb handling functions for the x86 architecture. In this approach, we start by the parts actually used for paravirt in i386.

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/smp_64.c      |    5 ++-
 include/asm-x86/tlbflush.h    |   77 +++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/tlbflush_32.h |   77 -----------------------------------------
 include/asm-x86/tlbflush_64.h |   43 +++--------------------
 4 files changed, 85 insertions(+), 117 deletions(-)
arch/x86/kernel/smp_64.c +3 -2
index 03fa6ed..ad063a6 100644
@@ -166,11 +166,12 @@ out:
 	add_pda(irq_tlb_count, 1);
 }
 
-static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
-						unsigned long va)
+void native_flush_tlb_others(const cpumask_t *cpumaskp,
+			     struct mm_struct *mm, unsigned long va)
 {
 	int sender;
 	union smp_flush_state *f;
+	cpumask_t cpumask = *cpumaskp;
 
 	/* Caller has disabled preemption */
 	sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
include/asm-x86/tlbflush.h +77 -0

82 lines not on this page · raw message

include/asm-x86/tlbflush_32.h +0 -77

93 lines not on this page · raw message

include/asm-x86/tlbflush_64.h +5 -38

58 lines not on this page · raw message

Signature · 1 line
1.4.4.2
author
#13 lore raw

PATCH 2/7 consolidate spinlock.h

In reply to Glauber de Oliveira Costa

The cli and sti instructions need to be replaced by paravirt hooks. For the i386 architecture, this is already done. The code requirements aren't much different from x86_64 POV, so this part is consolidated in the common header

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 include/asm-x86/spinlock.h    |   14 ++++++++++++++
 include/asm-x86/spinlock_32.h |    9 ---------
 include/asm-x86/spinlock_64.h |    8 +++++---
 3 files changed, 19 insertions(+), 12 deletions(-)
include/asm-x86/spinlock.h +14 -0
index d74d85e..e1d555a 100644
@@ -1,5 +1,19 @@
+#ifndef _X86_SPINLOCK_H_
+#define _X86_SPINLOCK_H_
+
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#else
+#define CLI_STRING	"cli"
+#define STI_STRING	"sti"
+#define CLI_STI_CLOBBERS
+#define CLI_STI_INPUT_ARGS
+#endif /* CONFIG_PARAVIRT */
+
 #ifdef CONFIG_X86_32
 # include "spinlock_32.h"
 #else
 # include "spinlock_64.h"
 #endif
+
+#endif
include/asm-x86/spinlock_32.h +0 -9
index d3bcebe..ebbf371 100644
@@ -7,15 +7,6 @@
 #include <asm/processor.h>
 #include <linux/compiler.h>
 
-#ifdef CONFIG_PARAVIRT
-#include <asm/paravirt.h>
-#else
-#define CLI_STRING	"cli"
-#define STI_STRING	"sti"
-#define CLI_STI_CLOBBERS
-#define CLI_STI_INPUT_ARGS
-#endif /* CONFIG_PARAVIRT */
-
 /*
  * Your basic SMP spinlocks, allowing only a single CPU anywhere
  *
include/asm-x86/spinlock_64.h +5 -3
index 88bf981..e56b17e 100644
@@ -48,12 +48,12 @@ static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long fla
 		"jns 5f\n"
 		"testl $0x200, %1\n\t"	/* interrupts were disabled? */
 		"jz 4f\n\t"
-	        "sti\n"
+		STI_STRING "\n"
 		"3:\t"
 		"rep;nop\n\t"
 		"cmpl $0, %0\n\t"
 		"jle 3b\n\t"
-		"cli\n\t"
+		CLI_STRING "\n\t"
 		"jmp 1b\n"
 		"4:\t"
 		"rep;nop\n\t"

@@ -61,7 +61,9 @@ static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long fla
 		"jg 1b\n\t"
 		"jmp 4b\n"
 		"5:\n\t"
-		: "+m" (lock->slock) : "r" ((unsigned)flags) : "memory");
+		: "+m" (lock->slock)
+		: "r" ((unsigned)flags) CLI_STI_INPUT_ARGS
+		: "memory" CLI_STI_CLOBBERS);
 }
 #endif
Signature · 1 line
1.4.4.2
author
#14 lore raw

PATCH 1/7 irqflags consolidation

In reply to Glauber de Oliveira Costa

This patch consolidates the irqflags include files containing common paravirt definitions. The native definition for interrupt handling, halt, and such, are the same for 32 and 64 bit, and they are kept in irqflags.h. The differences are split in the arch-specific files.

The syscall function, irq_enable_sysexit, has a very specific i386 naming, and its name is then changed to a more general one.

Signed-off-by Glauber de Oliveira Costa <[email protected]>
Signed-off-by Steven Rostedt <[email protected]>
Acked-by Jeremy Fitzhardinge <[email protected]>
---
 arch/x86/kernel/asm-offsets_32.c |    2 +-
 arch/x86/kernel/entry_32.S       |    8 ++--
 arch/x86/kernel/paravirt_32.c    |   10 ++--
 arch/x86/kernel/vmi_32.c         |    4 +-
 arch/x86/xen/enlighten.c         |    2 +-
 include/asm-x86/irqflags.h       |  107 ++++++++++++++++++++++++++++++++++++
 include/asm-x86/irqflags_32.h    |  103 +----------------------------------
 include/asm-x86/irqflags_64.h    |  112 +++++++-------------------------------
 include/asm-x86/paravirt.h       |    9 ++--
 9 files changed, 147 insertions(+), 210 deletions(-)
arch/x86/kernel/asm-offsets_32.c +1 -1
index 0e45981..c1ccfab 100644
@@ -123,7 +123,7 @@ void foo(void)
 	OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable);
 	OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable);
 	OFFSET(PV_CPU_iret, pv_cpu_ops, iret);
-	OFFSET(PV_CPU_irq_enable_sysexit, pv_cpu_ops, irq_enable_sysexit);
+	OFFSET(PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret);
 	OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0);
 #endif
arch/x86/kernel/entry_32.S +4 -4
index dc7f938..d63609d 100644
@@ -58,7 +58,7 @@
  * for paravirtualization.  The following will never clobber any registers:
  *   INTERRUPT_RETURN (aka. "iret")
  *   GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
- *   ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
+ *   ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
  *
  * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
  * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).

@@ -351,7 +351,7 @@ sysenter_past_esp:
 	xorl %ebp,%ebp
 	TRACE_IRQS_ON
 1:	mov  PT_FS(%esp), %fs
-	ENABLE_INTERRUPTS_SYSEXIT
+	ENABLE_INTERRUPTS_SYSCALL_RET
 	CFI_ENDPROC
 .pushsection .fixup,"ax"
 2:	movl $0,PT_FS(%esp)

@@ -882,10 +882,10 @@ ENTRY(native_iret)
 .previous
 END(native_iret)
 
-ENTRY(native_irq_enable_sysexit)
+ENTRY(native_irq_enable_syscall_ret)
 	sti
 	sysexit
-END(native_irq_enable_sysexit)
+END(native_irq_enable_syscall_ret)
 #endif
 
 KPROBE_ENTRY(int3)
arch/x86/kernel/paravirt_32.c +5 -5
index 6a80d67..04f51d0 100644
@@ -60,7 +60,7 @@ DEF_NATIVE(pv_irq_ops, irq_enable, "sti");
 DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf");
 DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax");
 DEF_NATIVE(pv_cpu_ops, iret, "iret");
-DEF_NATIVE(pv_cpu_ops, irq_enable_sysexit, "sti; sysexit");
+DEF_NATIVE(pv_cpu_ops, irq_enable_syscall_ret, "sti; sysexit");
 DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax");
 DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3");
 DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax");

@@ -88,7 +88,7 @@ static unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
 	SITE(pv_irq_ops, restore_fl);
 	SITE(pv_irq_ops, save_fl);
 	SITE(pv_cpu_ops, iret);
-	SITE(pv_cpu_ops, irq_enable_sysexit);
+	SITE(pv_cpu_ops, irq_enable_syscall_ret);
 	SITE(pv_mmu_ops, read_cr2);
 	SITE(pv_mmu_ops, read_cr3);
 	SITE(pv_mmu_ops, write_cr3);

@@ -186,7 +186,7 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
 		/* If the operation is a nop, then nop the callsite */
 		ret = paravirt_patch_nop();
 	else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
-		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit))
+		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret))
 		/* If operation requires a jmp, then jmp */
 		ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
 	else

@@ -237,7 +237,7 @@ static void native_flush_tlb_single(unsigned long addr)
 
 /* These are in entry.S */
 extern void native_iret(void);
-extern void native_irq_enable_sysexit(void);
+extern void native_irq_enable_syscall_ret(void);
 
 static int __init print_banner(void)
 {

@@ -384,7 +384,7 @@ struct pv_cpu_ops pv_cpu_ops = {
 	.write_idt_entry = write_dt_entry,
 	.load_esp0 = native_load_esp0,
 
-	.irq_enable_sysexit = native_irq_enable_sysexit,
+	.irq_enable_syscall_ret = native_irq_enable_syscall_ret,
 	.iret = native_iret,
 
 	.set_iopl_mask = native_set_iopl_mask,
arch/x86/kernel/vmi_32.c +2 -2
index f02bad6..aacce42 100644
@@ -148,7 +148,7 @@ static unsigned vmi_patch(u8 type, u16 clobbers, void *insns,
 					      insns, eip);
 		case PARAVIRT_PATCH(pv_cpu_ops.iret):
 			return patch_internal(VMI_CALL_IRET, len, insns, eip);
-		case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit):
+		case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret):
 			return patch_internal(VMI_CALL_SYSEXIT, len, insns, eip);
 		default:
 			break;

@@ -870,7 +870,7 @@ static inline int __init activate_vmi(void)
 	 * the backend.  They are performance critical anyway, so requiring
 	 * a patch is not a big problem.
 	 */
-	pv_cpu_ops.irq_enable_sysexit = (void *)0xfeedbab0;
+	pv_cpu_ops.irq_enable_syscall_ret = (void *)0xfeedbab0;
 	pv_cpu_ops.iret = (void *)0xbadbab0;
 
 #ifdef CONFIG_SMP
arch/x86/xen/enlighten.c +1 -1
index 94c39aa..094b915 100644
@@ -953,7 +953,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = {
 	.read_pmc = native_read_pmc,
 
 	.iret = (void *)&hypercall_page[__HYPERVISOR_iret],
-	.irq_enable_sysexit = NULL,  /* never called */
+	.irq_enable_syscall_ret = NULL,  /* never called */
 
 	.load_tr_desc = paravirt_nop,
 	.set_ldt = xen_set_ldt,
include/asm-x86/irqflags.h +107 -0

112 lines not on this page · raw message

include/asm-x86/irqflags_32.h +2 -101

113 lines not on this page · raw message

include/asm-x86/irqflags_64.h +20 -92

134 lines not on this page · raw message

include/asm-x86/paravirt.h +5 -4
index f59d370..d81a361 100644
@@ -121,7 +121,7 @@ struct pv_cpu_ops {
 	u64 (*read_pmc)(void);
 
 	/* These two are jmp to, not actually called. */
-	void (*irq_enable_sysexit)(void);
+	void (*irq_enable_syscall_ret)(void);
 	void (*iret)(void);
 
 	struct pv_lazy_ops lazy_mode;

@@ -1138,9 +1138,10 @@ static inline unsigned long __raw_local_irq_save(void)
 		  call *%cs:pv_irq_ops+PV_IRQ_irq_enable;		\
 		  popl %edx; popl %ecx; popl %eax)
 
-#define ENABLE_INTERRUPTS_SYSEXIT					       \
-	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), CLBR_NONE,\
-		  jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_sysexit)
+#define ENABLE_INTERRUPTS_SYSCALL_RET					\
+	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_syscall_ret),\
+		  CLBR_NONE,						\
+		  jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_syscall_ret)
 
 #define GET_CR0_INTO_EAX			\
 	push %ecx; push %edx;			\
Signature · 1 line
1.4.4.2