Skip to contentSkip to conversation

x86: Consolidate 32 and 64 bit thread_info

GIT PULL New Steven Rostedt 4 participants 23 messages Aug 6, 2011 – Mar 11, 2014

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

[
Repost with removing the first patch:
"x86: Do not differentiate between x86_64 and i386 with init_per_cpu"
as requested by Brian Gerst
]

Thomas, Ingo, and Peter(s),

On IRC, Peter Zijlstra mentioned to me the difference between 32bit and 64bit x86 on how thread_info is handled with interrupts. I mentioned that they should be more the same, and Peter agreed.

I decided to do so, and ended up with the following patches.
I marked this for v3.2 as it should be tested a bit before going into mainline. I've ran several configs on it, but more testing should be done.

Some of the patches are clean ups, but the main ones are patch 3 and 4 that do the work to convert 32bit of thread_info to be more like the 64bit.

-- Steve

Please pull the latest tip/x86/thread_info tree, which can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-2.6-trace.git
tip/x86/thread_info

Head SHA1: 326c03f828beaff087aa6bd9e741cb4d4915a685

Steven Rostedt (4):
x86: Nuke the supervisor_stack field in i386 thread_info
x86: Remove previous_esp from i386 thread_info structure
x86: Keep thread_info on thread stack in x86_32
x86: Clean up dumpstack_64.c code

----

 arch/x86/include/asm/processor.h   |    9 +++
 arch/x86/include/asm/thread_info.h |   56 +++--------------
 arch/x86/kernel/cpu/common.c       |    8 +-
 arch/x86/kernel/dumpstack_32.c     |   44 ++++++++++++-
 arch/x86/kernel/dumpstack_64.c     |  121 +++++++++++++++++++++++++-----------
 arch/x86/kernel/irq_32.c           |   86 +++++++++++---------------
 arch/x86/kernel/process_32.c       |    4 +
 arch/x86/kernel/smpboot.c          |    2 +-
 8 files changed, 189 insertions(+), 141 deletions(-)
#1 lore raw

PATCH v2 1/4 x86: Nuke the supervisor_stack field in i386 thread_info

From: Steven Rostedt <[email protected]>

Nothing references the supervisor_stack in the thread_info field, and it does not exist in x86_64. To make the two more the same, it is being removed.

---
 arch/x86/include/asm/thread_info.h |    1 -
 1 files changed, 0 insertions(+), 1 deletions(-)
arch/x86/include/asm/thread_info.h +0 -1
index 1f2e61e..625910f 100644
@@ -38,7 +38,6 @@ struct thread_info {
 	unsigned long           previous_esp;   /* ESP of the previous stack in
 						   case of nested (IRQ) stacks
 						*/
-	__u8			supervisor_stack[0];
 #endif
 	int			uaccess_err;
 };
Signature · 1 line
1.7.5.4
#3 lore raw

PATCH v2 3/4 x86: Keep thread_info on thread stack in x86_32

In reply to Steven Rostedt

From: Steven Rostedt <[email protected]>

x86_64 uses a per_cpu variable kernel_stack to always point to the thread stack of current. This is where the thread_info is stored and is accessed from this location even when the irq or exception stack is in use. This removes the complexity of having to maintain the thread info on the stack when interrupts are running and having to copy the preempt_count and other fields to the interrupt stack.

x86_32 uses the old method of copying the thread_info from the thread stack to the exception stack just before executing the exception.

Having the two different requires #ifdefs and also the x86_32 way is a bit of a pain to maintain. By converting x86_32 to the same method of x86_64, we can remove #ifdefs, clean up the x86_32 code a little, and remove the overhead of the copy.

Cc Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 arch/x86/include/asm/processor.h   |    9 ++++
 arch/x86/include/asm/thread_info.h |   50 ++++------------------
 arch/x86/kernel/cpu/common.c       |    8 ++--
 arch/x86/kernel/dumpstack_32.c     |   40 +++++++++++++++---
 arch/x86/kernel/irq_32.c           |   81 ++++++++++++++----------------------
 arch/x86/kernel/process_32.c       |    4 ++
 arch/x86/kernel/smpboot.c          |    2 +-
 7 files changed, 93 insertions(+), 101 deletions(-)
arch/x86/include/asm/processor.h +9 -0
index 2193715..1d01248 100644
@@ -414,6 +414,15 @@ struct stack_canary {
 };
 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
+/*
+ * per-CPU IRQ handling stacks
+ */
+struct irq_stack {
+	u32                     stack[THREAD_SIZE/sizeof(u32)];
+} __attribute__((aligned(THREAD_SIZE)));
+
+DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
+DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
 #endif	/* X86_64 */
 
 extern unsigned int xstate_size;
arch/x86/include/asm/thread_info.h +10 -40

71 lines not on this page · raw message

arch/x86/kernel/cpu/common.c +4 -4
index 22a073d..b0304fb 100644
@@ -1019,6 +1019,10 @@ static __init int setup_disablecpuid(char *arg)
 }
 __setup("clearcpuid=", setup_disablecpuid);
 
+DEFINE_PER_CPU(unsigned long, kernel_stack) =
+	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+EXPORT_PER_CPU_SYMBOL(kernel_stack);
+
 #ifdef CONFIG_X86_64
 struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
 
@@ -1033,10 +1037,6 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
 	&init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
 
-DEFINE_PER_CPU(unsigned long, kernel_stack) =
-	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
-EXPORT_PER_CPU_SYMBOL(kernel_stack);
-
 DEFINE_PER_CPU(char *, irq_stack_ptr) =
 	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
 
arch/x86/kernel/dumpstack_32.c +34 -6
index 8955e25..6341cdc 100644
@@ -16,11 +16,33 @@
 
 #include <asm/stacktrace.h>
 
+static void *is_irq_stack(void *p, void *irq)
+{
+	if (p < irq || p >= (irq + THREAD_SIZE))
+		return NULL;
+	return irq + THREAD_SIZE;
+}
+
+
+static void *is_hardirq_stack(unsigned long *stack, int cpu)
+{
+	void *irq = per_cpu(hardirq_stack, cpu);
+
+	return is_irq_stack(stack, irq);
+}
+
+static void *is_softirq_stack(unsigned long *stack, int cpu)
+{
+	void *irq = per_cpu(softirq_stack, cpu);
+
+	return is_irq_stack(stack, irq);
+}
 
 void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		unsigned long *stack, unsigned long bp,
 		const struct stacktrace_ops *ops, void *data)
 {
+	const unsigned cpu = get_cpu();
 	int graph = 0;
 	u32 *prev_esp;
 
@@ -40,17 +62,22 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 
 	for (;;) {
 		struct thread_info *context;
+		void *end_stack = NULL;
+
+		end_stack = is_hardirq_stack(stack, cpu);
+		if (!end_stack)
+			end_stack = is_softirq_stack(stack, cpu);
 
-		context = (struct thread_info *)
-			((unsigned long)stack & (~(THREAD_SIZE - 1)));
-		bp = ops->walk_stack(context, stack, bp, ops, data, NULL, &graph);
+		context = task_thread_info(task);
+		bp = ops->walk_stack(context, stack, bp, ops, data,
+				     end_stack, &graph);
 
 		/* Stop if not on irq stack */
-		if (task_stack_page(task) == context)
+		if (!end_stack)
 			break;
 
-		/* The previous esp is just above the context */
-		prev_esp = (u32 *) ((char *)context + sizeof(struct thread_info));
+		/* The previous esp is saved on the bottom of the stack */
+		prev_esp = (u32 *)(end_stack - THREAD_SIZE);
 		stack = (unsigned long *)*prev_esp;
 		if (!stack)
 			break;
@@ -59,6 +86,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 			break;
 		touch_nmi_watchdog();
 	}
+	put_cpu();
 }
 EXPORT_SYMBOL(dump_trace);
 
arch/x86/kernel/irq_32.c +31 -50

138 lines not on this page · raw message

arch/x86/kernel/process_32.c +4 -0
index a3d0dc5..19e1468 100644
@@ -341,6 +341,10 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl))
 		set_iopl_mask(next->iopl);
 
+	percpu_write(kernel_stack,
+		  (unsigned long)task_stack_page(next_p) +
+		  THREAD_SIZE - KERNEL_STACK_OFFSET);
+
 	/*
 	 * Now maybe handle debug registers and/or IO bitmaps
 	 */
arch/x86/kernel/smpboot.c +1 -1
index 9fd3137..d6518a9 100644
@@ -715,10 +715,10 @@ do_rest:
 #else
 	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
+#endif
 	per_cpu(kernel_stack, cpu) =
 		(unsigned long)task_stack_page(c_idle.idle) -
 		KERNEL_STACK_OFFSET + THREAD_SIZE;
-#endif
 	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
 	initial_code = (unsigned long)start_secondary;
 	stack_start  = c_idle.idle->thread.sp;
Signature · 1 line
1.7.5.4
#4 lore raw

PATCH v2 4/4 x86: Clean up dumpstack_64.c code

In reply to Steven Rostedt

From: Steven Rostedt <[email protected]>

The dump_trace() function in dumpstack_64.c is hard to follow. The test for exception stack is processed differently than the test for irq stack, and the normal stack is outside completely.

By restructuring this code to have all the stacks determined by a single function that returns an enum of the following:

STACK_IS_NORMAL
STACK_IS_EXCEPTION
STACK_IS_IRQ
STACK_IS_UNKNOWN

and has the logic of each within a switch statement. This should make the code much easier to read and understand.

Cc Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 arch/x86/kernel/dumpstack_64.c |  121 ++++++++++++++++++++++++++++------------
 1 files changed, 85 insertions(+), 36 deletions(-)
arch/x86/kernel/dumpstack_64.c +85 -36
index e71c98d..d0b2ace 100644
@@ -132,6 +132,45 @@ fixup_bp_irq_link(unsigned long bp, unsigned long *stack,
 	return bp;
 }
 
+static const unsigned long irq_stack_size =
+	(IRQ_STACK_SIZE - 64) / sizeof(unsigned long);
+
+enum stack_type {
+	STACK_IS_UNKNOWN,
+	STACK_IS_NORMAL,
+	STACK_IS_EXCEPTION,
+	STACK_IS_IRQ,
+};
+
+static enum stack_type
+analyze_stack(int cpu, struct task_struct *task,
+	      unsigned long *stack, unsigned long **stack_end, char **id)
+{
+	unsigned long *irq_stack;
+	unsigned long addr;
+	unsigned used = 0;
+
+	addr = ((unsigned long)stack & (~(THREAD_SIZE - 1)));
+	if ((unsigned long)task_stack_page(task) == addr)
+		return STACK_IS_NORMAL;
+
+	*stack_end = in_exception_stack(cpu, (unsigned long)stack,
+					 &used, id);
+	if (*stack_end)
+		return STACK_IS_EXCEPTION;
+
+	*stack_end = (unsigned long *)per_cpu(irq_stack_ptr, cpu);
+	if (!*stack_end)
+		return STACK_IS_UNKNOWN;
+
+	irq_stack = *stack_end - irq_stack_size;
+
+	if (in_irq_stack(stack, irq_stack, *stack_end))
+		return STACK_IS_IRQ;
+
+	return STACK_IS_UNKNOWN;
+}
+
 /*
  * x86-64 can have up to three kernel stacks:
  * process stack
@@ -144,12 +183,11 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		const struct stacktrace_ops *ops, void *data)
 {
 	const unsigned cpu = get_cpu();
-	unsigned long *irq_stack_end =
-		(unsigned long *)per_cpu(irq_stack_ptr, cpu);
-	unsigned used = 0;
 	struct thread_info *tinfo;
-	int graph = 0;
+	unsigned long *irq_stack;
 	unsigned long dummy;
+	int graph = 0;
+	int done = 0;
 
 	if (!task)
 		task = current;
@@ -168,51 +206,62 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 	 * exceptions
 	 */
 	tinfo = task_thread_info(task);
-	for (;;) {
+	while (!done) {
+		unsigned long *stack_end;
+		enum stack_type stype;
 		char *id;
-		unsigned long *estack_end;
-		estack_end = in_exception_stack(cpu, (unsigned long)stack,
-						&used, &id);
 
-		if (estack_end) {
+		stype = analyze_stack(cpu, task, stack, &stack_end, &id);
+
+		/* Default finish unless specified to continue */
+		done = 1;
+
+		switch (stype) {
+
+		/* Break out early if we are on the thread stack */
+		case STACK_IS_NORMAL:
+			break;
+
+		case STACK_IS_EXCEPTION:
+
 			if (ops->stack(data, id) < 0)
 				break;
 
 			bp = ops->walk_stack(tinfo, stack, bp, ops,
-					     data, estack_end, &graph);
+					     data, stack_end, &graph);
 			ops->stack(data, "<EOE>");
 			/*
 			 * We link to the next stack via the
 			 * second-to-last pointer (index -2 to end) in the
 			 * exception stack:
 			 */
-			stack = (unsigned long *) estack_end[-2];
-			continue;
-		}
-		if (irq_stack_end) {
-			unsigned long *irq_stack;
-			irq_stack = irq_stack_end -
-				(IRQ_STACK_SIZE - 64) / sizeof(*irq_stack);
-
-			if (in_irq_stack(stack, irq_stack, irq_stack_end)) {
-				if (ops->stack(data, "IRQ") < 0)
-					break;
-				bp = ops->walk_stack(tinfo, stack, bp,
-					ops, data, irq_stack_end, &graph);
-				/*
-				 * We link to the next stack (which would be
-				 * the process stack normally) the last
-				 * pointer (index -1 to end) in the IRQ stack:
-				 */
-				stack = (unsigned long *) (irq_stack_end[-1]);
-				bp = fixup_bp_irq_link(bp, stack, irq_stack,
-						       irq_stack_end);
-				irq_stack_end = NULL;
-				ops->stack(data, "EOI");
-				continue;
-			}
+			stack = (unsigned long *) stack_end[-2];
+			done = 0;
+			break;
+
+		case STACK_IS_IRQ:
+
+			if (ops->stack(data, "IRQ") < 0)
+				break;
+			bp = ops->walk_stack(tinfo, stack, bp,
+				     ops, data, stack_end, &graph);
+			/*
+			 * We link to the next stack (which would be
+			 * the process stack normally) the last
+			 * pointer (index -1 to end) in the IRQ stack:
+			 */
+			stack = (unsigned long *) (stack_end[-1]);
+			irq_stack = stack_end - irq_stack_size;
+			bp = fixup_bp_irq_link(bp, stack, irq_stack,
+					       stack_end);
+			ops->stack(data, "EOI");
+			done = 0;
+			break;
+
+		case STACK_IS_UNKNOWN:
+			ops->stack(data, "UNK");
+			break;
 		}
-		break;
 	}
 
 	/*
Signature · 1 line
1.7.5.4
#2 lore raw

PATCH v2 2/4 x86: Remove previous_esp from i386 thread_info structure

In reply to Steven Rostedt

From: Steven Rostedt <[email protected]>

The i386 thread_info contains a previous_esp field that is used to daisy chain the different stacks for dump_stack() (ie. irq, softirq, thread stacks).

The goal is to eventual make i386 handling of thread_info the same as x86_64, which means that the thread_info will not be in the stack but as a per_cpu variable. We will no longer depend on thread_info being able to daisy chain different stacks as it will only exist in one location (the thread stack).

By moving previous_esp out of thread_info, and just placing it above the thread_info of the current stack, we can still use the stacks to daisy chain which stack was in use when preempted by an interrupt.

Cc Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 arch/x86/include/asm/thread_info.h |    5 -----
 arch/x86/kernel/dumpstack_32.c     |   10 +++++++++-
 arch/x86/kernel/irq_32.c           |   13 +++++++++----
 3 files changed, 18 insertions(+), 10 deletions(-)
arch/x86/include/asm/thread_info.h +0 -5
index 625910f..8f2b249 100644
@@ -34,11 +34,6 @@ struct thread_info {
 	mm_segment_t		addr_limit;
 	struct restart_block    restart_block;
 	void __user		*sysenter_return;
-#ifdef CONFIG_X86_32
-	unsigned long           previous_esp;   /* ESP of the previous stack in
-						   case of nested (IRQ) stacks
-						*/
-#endif
 	int			uaccess_err;
 };
 
arch/x86/kernel/dumpstack_32.c +9 -1
index 3b97a80..8955e25 100644
@@ -22,6 +22,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		const struct stacktrace_ops *ops, void *data)
 {
 	int graph = 0;
+	u32 *prev_esp;
 
 	if (!task)
 		task = current;
@@ -44,9 +45,16 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 			((unsigned long)stack & (~(THREAD_SIZE - 1)));
 		bp = ops->walk_stack(context, stack, bp, ops, data, NULL, &graph);
 
-		stack = (unsigned long *)context->previous_esp;
+		/* Stop if not on irq stack */
+		if (task_stack_page(task) == context)
+			break;
+
+		/* The previous esp is just above the context */
+		prev_esp = (u32 *) ((char *)context + sizeof(struct thread_info));
+		stack = (unsigned long *)*prev_esp;
 		if (!stack)
 			break;
+
 		if (ops->stack(data, "IRQ") < 0)
 			break;
 		touch_nmi_watchdog();
arch/x86/kernel/irq_32.c +9 -4
index 7209070..47909a0 100644
@@ -76,7 +76,7 @@ static inline int
 execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 {
 	union irq_ctx *curctx, *irqctx;
-	u32 *isp, arg1, arg2;
+	u32 *isp, *prev_esp, arg1, arg2;
 
 	curctx = (union irq_ctx *) current_thread_info();
 	irqctx = __this_cpu_read(hardirq_ctx);
@@ -93,7 +93,9 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 	/* build the stack frame on the IRQ stack */
 	isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
 	irqctx->tinfo.task = curctx->tinfo.task;
-	irqctx->tinfo.previous_esp = current_stack_pointer;
+	/* Save the next esp after thread_info */
+	prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info));
+	*prev_esp = current_stack_pointer;
 
 	/*
 	 * Copy the softirq bits in preempt_count so that the
@@ -154,7 +156,7 @@ asmlinkage void do_softirq(void)
 	unsigned long flags;
 	struct thread_info *curctx;
 	union irq_ctx *irqctx;
-	u32 *isp;
+	u32 *isp, *prev_esp;
 
 	if (in_interrupt())
 		return;
@@ -165,11 +167,14 @@ asmlinkage void do_softirq(void)
 		curctx = current_thread_info();
 		irqctx = __this_cpu_read(softirq_ctx);
 		irqctx->tinfo.task = curctx->task;
-		irqctx->tinfo.previous_esp = current_stack_pointer;
 
 		/* build the stack frame on the softirq stack */
 		isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
 
+		/* Push the previous esp onto the stack */
+		prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info));
+		*prev_esp = current_stack_pointer;
+
 		call_on_stack(__do_softirq, isp);
 		/*
 		 * Shouldn't happen, we returned above if in_interrupt():
Signature · 1 line
1.7.5.4
#5 lore raw
In reply to Steven Rostedt
Quoted 7 lines · Steven Rostedt

Thomas, Ingo, and Peter(s),

On IRC, Peter Zijlstra mentioned to me the difference between 32bit and 64bit x86 on how thread_info is handled with interrupts. I mentioned that they should be more the same, and Peter agreed.

Quoted 8 lines

I decided to do so, and ended up with the following patches.
I marked this for v3.2 as it should be tested a bit before going into mainline. I've ran several configs on it, but more testing should be done.

Some of the patches are clean ups, but the main ones are patch 3 and 4 that do the work to convert 32bit of thread_info to be more like the 64bit.

Hi Steven,

I'd like to wait until the merge window is over with pulling in 3.2 material ... nothing personal or technical, just to simplify management.

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#6 lore raw
Quoted 2 lines · H. Peter Anvin

I'd like to wait until the merge window is over with pulling in 3.2 material ... nothing personal or technical, just to simplify management.

No prob, and the merge window is now closed :)

-- Steve

#7 lore raw
In reply to H. Peter Anvin
Quoted 17 lines · H. Peter Anvin
Quoted 7 lines · Steven Rostedt

Thomas, Ingo, and Peter(s),

On IRC, Peter Zijlstra mentioned to me the difference between 32bit and 64bit x86 on how thread_info is handled with interrupts. I mentioned that they should be more the same, and Peter agreed.

Quoted 8 lines

I decided to do so, and ended up with the following patches.
I marked this for v3.2 as it should be tested a bit before going into mainline. I've ran several configs on it, but more testing should be done.

Some of the patches are clean ups, but the main ones are patch 3 and 4 that do the work to convert 32bit of thread_info to be more like the 64bit.

Quoted 7 lines

Hi Steven,

I'd like to wait until the merge window is over with pulling in 3.2 material ... nothing personal or technical, just to simplify management.

-hpa

Ping?

-- Steve

#8 lore raw
Quoted 25 lines · Steven Rostedt
Quoted 10 lines · H. Peter Anvin
Quoted 9 lines · Steven Rostedt

Thomas, Ingo, and Peter(s),

On IRC, Peter Zijlstra mentioned to me the difference between 32bit and 64bit x86 on how thread_info is handled with interrupts. I mentioned that they should be more the same, and Peter agreed.

I decided to do so, and ended up with the following patches. I marked this for v3.2 as it should be tested a bit before going

into

Quoted 1 line
Quoted 1 line

mainline. I've ran several configs on it, but more testing should

be done.

Quoted 2 lines
Quoted 2 lines

Some of the patches are clean ups, but the main ones are patch 3

and 4

Quoted 8 lines
Quoted 3 lines

that do the work to convert 32bit of thread_info to be more like the 64bit.

Hi Steven,

I'd like to wait until the merge window is over with pulling in 3.2 material ... nothing personal or technical, just to simplify

Quoted 8 lines

management.

Quoted 3 lines

-hpa

Ping?

-- Steve

Travelling last and this week... if Ingo doesn't beat me to it pung me again Tuesday if I haven't grabbed it already....

Signature · 1 line
Sent from my mobile phone. Please excuse my brevity and lack of formatting.
#10 lore raw

PATCH 1/5 x86: Nuke the supervisor_stack field in i386 thread_info

In reply to Steven Rostedt

From: Steven Rostedt <[email protected]>

Nothing references the supervisor_stack in the thread_info field, and it does not exist in x86_64. To make the two more the same, it is being removed.

Cc Andrew Morton <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 arch/x86/include/asm/thread_info.h | 1 -
 1 file changed, 1 deletion(-)
arch/x86/include/asm/thread_info.h +0 -1
index 3ba3de4..e012ed0 100644
@@ -36,7 +36,6 @@ struct thread_info {
 	unsigned long           previous_esp;   /* ESP of the previous stack in
 						   case of nested (IRQ) stacks
 						*/
-	__u8			supervisor_stack[0];
 #endif
 	unsigned int		sig_on_uaccess_error:1;
 	unsigned int		uaccess_err:1;	/* uaccess failed */
Signature · 1 line
1.8.4.3
#14 lore raw

PATCH 3/5 x86: Prepare removal of previous_esp from i386 thread_info structure

In reply to Steven Rostedt

From: Steven Rostedt <[email protected]>

The i386 thread_info contains a previous_esp field that is used to daisy chain the different stacks for dump_stack() (ie. irq, softirq, thread stacks).

The goal is to eventual make i386 handling of thread_info the same as x86_64, which means that the thread_info will not be in the stack but as a per_cpu variable. We will no longer depend on thread_info being able to daisy chain different stacks as it will only exist in one location (the thread stack).

By moving previous_esp to the end of thread_info and referencing it as an offset instead of using a thread_info field, this becomes a stepping stone to moving the thread_info.

The offset to get to the previous stack is rather ugly in this patch, but this is only temporary and the prev_esp will be changed in the next commit. This commit is more for sanity checks of the change.

Cc Andrew Morton <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Robert Richter <[email protected]>
---
 arch/x86/include/asm/thread_info.h |  7 ++++---
 arch/x86/kernel/dumpstack_32.c     | 11 ++++++++++-
 arch/x86/kernel/irq_32.c           | 15 +++++++++++----
 arch/x86/kernel/ptrace.c           |  8 ++++----
 4 files changed, 29 insertions(+), 12 deletions(-)
arch/x86/include/asm/thread_info.h +4 -3
index ee029e4..ed96e37 100644
@@ -32,13 +32,14 @@ struct thread_info {
 	mm_segment_t		addr_limit;
 	struct restart_block    restart_block;
 	void __user		*sysenter_return;
+	unsigned int		sig_on_uaccess_error:1;
+	unsigned int		uaccess_err:1;	/* uaccess failed */
 #ifdef CONFIG_X86_32
 	unsigned long           previous_esp;   /* ESP of the previous stack in
 						   case of nested (IRQ) stacks
+						   (Moved to end, to be removed soon)
 						*/
-#endif
-	unsigned int		sig_on_uaccess_error:1;
-	unsigned int		uaccess_err:1;	/* uaccess failed */
+#endif 
 };
 
 #define INIT_THREAD_INFO(tsk)			\
arch/x86/kernel/dumpstack_32.c +10 -1
index f2a1770..187d6a7 100644
@@ -22,6 +22,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		const struct stacktrace_ops *ops, void *data)
 {
 	int graph = 0;
+	u32 *prev_esp;
 
 	if (!task)
 		task = current;
@@ -44,9 +45,17 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 			((unsigned long)stack & (~(THREAD_SIZE - 1)));
 		bp = ops->walk_stack(context, stack, bp, ops, data, NULL, &graph);
 
-		stack = (unsigned long *)context->previous_esp;
+		/* Stop if not on irq stack */
+		if (task_stack_page(task) == context)
+			break;
+
+		/* The previous esp is just above the context */
+		prev_esp = (u32 *) ((char *)context + sizeof(struct thread_info) -
+				    sizeof(long));
+		stack = (unsigned long *)*prev_esp;
 		if (!stack)
 			break;
+
 		if (ops->stack(data, "IRQ") < 0)
 			break;
 		touch_nmi_watchdog();
arch/x86/kernel/irq_32.c +11 -4
index d7fcbed..f135cc2 100644
@@ -81,7 +81,7 @@ static inline int
 execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 {
 	union irq_ctx *curctx, *irqctx;
-	u32 *isp, arg1, arg2;
+	u32 *isp, *prev_esp, arg1, arg2;
 
 	curctx = (union irq_ctx *) current_thread_info();
 	irqctx = __this_cpu_read(hardirq_ctx);
@@ -98,7 +98,10 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 	/* build the stack frame on the IRQ stack */
 	isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
 	irqctx->tinfo.task = curctx->tinfo.task;
-	irqctx->tinfo.previous_esp = current_stack_pointer;
+	/* Save the next esp after thread_info */
+	prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info) -
+			    sizeof(long));
+	*prev_esp = current_stack_pointer;
 
 	if (unlikely(overflow))
 		call_on_stack(print_stack_overflow, isp);
@@ -149,16 +152,20 @@ void do_softirq_own_stack(void)
 {
 	struct thread_info *curctx;
 	union irq_ctx *irqctx;
-	u32 *isp;
+	u32 *isp, *prev_esp;
 
 	curctx = current_thread_info();
 	irqctx = __this_cpu_read(softirq_ctx);
 	irqctx->tinfo.task = curctx->task;
-	irqctx->tinfo.previous_esp = current_stack_pointer;
 
 	/* build the stack frame on the softirq stack */
 	isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
 
+	/* Push the previous esp onto the stack */
+	prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info) -
+			    sizeof(long));
+	*prev_esp = current_stack_pointer;
+
 	call_on_stack(__do_softirq, isp);
 }
 
arch/x86/kernel/ptrace.c +4 -4
index 7461f50..f352a7c 100644
@@ -184,14 +184,14 @@ unsigned long kernel_stack_pointer(struct pt_regs *regs)
 {
 	unsigned long context = (unsigned long)regs & ~(THREAD_SIZE - 1);
 	unsigned long sp = (unsigned long)&regs->sp;
-	struct thread_info *tinfo;
+	u32 *prev_esp;
 
 	if (context == (sp & ~(THREAD_SIZE - 1)))
 		return sp;
 
-	tinfo = (struct thread_info *)context;
-	if (tinfo->previous_esp)
-		return tinfo->previous_esp;
+	prev_esp = (u32 *)(context + sizeof(struct thread_info) - sizeof(long));
+	if (prev_esp)
+		return (unsigned long)prev_esp;
 
 	return (unsigned long)regs;
 }
Signature · 1 line
1.8.4.3
#12 lore raw

PATCH 2/5 x86: Nuke GET_THREAD_INFO_WITH_ESP() macro for i386

In reply to Steven Rostedt

From: "Steven Rostedt (Red Hat)" <[email protected]>

According to a git log -p, GET_THREAD_INFO_WITH_ESP() has only been defined and never been used. Get rid of it.

---
 arch/x86/include/asm/thread_info.h | 4 ----
 1 file changed, 4 deletions(-)
arch/x86/include/asm/thread_info.h +0 -4
index e012ed0..ee029e4 100644
@@ -180,10 +180,6 @@ static inline struct thread_info *current_thread_info(void)
 	movl $-THREAD_SIZE, reg; \
 	andl %esp, reg
 
-/* use this one if reg already contains %esp */
-#define GET_THREAD_INFO_WITH_ESP(reg) \
-	andl $-THREAD_SIZE, reg
-
 #endif
 
 #else /* X86_32 */
Signature · 1 line
1.8.4.3
#9 lore raw

[PATCH 0/5] [GIT PULL] x86: Consolidate 32 and 64 bit thead_info (again)

Peter,

Back in August 2011, I had a pull request to make the irq stack of i386 more like the x86_64 code. There seemed to be acceptance back then and you said you would even pull it after the merge window. But due to your travels it was forgotten (and I forgot about it too).

http://lkml.kernel.org/r/[email protected]

I stumbled on these changes again and noticed that they were never applied so I forward ported them with a few more adjustments.

Hopefully they are still worth pulling. This time I waited till after the merge window to post :-)

Please pull the latest tip/x86/core tree, which can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
tip/x86/core

Head SHA1: 3132e107d608f8753240d82d61303c500fd515b4 237dfcadc93749ca5cdd2f683f887c3dcf3a0c04

Steven Rostedt (4):
x86: Nuke the supervisor_stack field in i386 thread_info
x86: Prepare removal of previous_esp from i386 thread_info structure
x86: Keep thread_info on thread stack in x86_32
x86: Clean up dumpstack_64.c code

Steven Rostedt (Red Hat) (1):
x86: Nuke GET_THREAD_INFO_WITH_ESP() macro for i386

----

 arch/x86/include/asm/processor.h   |   9 +++
 arch/x86/include/asm/thread_info.h |  52 ++---------------
 arch/x86/kernel/cpu/common.c       |   8 +--
 arch/x86/kernel/dumpstack_32.c     |  44 ++++++++++++--
 arch/x86/kernel/dumpstack_64.c     | 117 ++++++++++++++++++++++++++-----------
 arch/x86/kernel/irq_32.c           |  79 ++++++++++++-------------
 arch/x86/kernel/process_32.c       |   4 ++
 arch/x86/kernel/ptrace.c           |   8 +--
 arch/x86/kernel/smpboot.c          |   2 +-
 9 files changed, 188 insertions(+), 135 deletions(-)
#20 lore raw

PATCH 5/5 x86: Clean up dumpstack_64.c code

From: Steven Rostedt <[email protected]>

The dump_trace() function in dumpstack_64.c is hard to follow. The test for exception stack is processed differently than the test for irq stack, and the normal stack is outside completely.

By restructuring this code to have all the stacks determined by a single function that returns an enum of the following:

STACK_IS_NORMAL
STACK_IS_EXCEPTION
STACK_IS_IRQ
STACK_IS_UNKNOWN

and has the logic of each within a switch statement. This should make the code much easier to read and understand.

Cc Andrew Morton <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 arch/x86/kernel/dumpstack_64.c | 117 +++++++++++++++++++++++++++++------------
 1 file changed, 83 insertions(+), 34 deletions(-)
arch/x86/kernel/dumpstack_64.c +83 -34
index addb207..346b1df 100644
@@ -104,6 +104,45 @@ in_irq_stack(unsigned long *stack, unsigned long *irq_stack,
 	return (stack >= irq_stack && stack < irq_stack_end);
 }
 
+static const unsigned long irq_stack_size =
+	(IRQ_STACK_SIZE - 64) / sizeof(unsigned long);
+
+enum stack_type {
+	STACK_IS_UNKNOWN,
+	STACK_IS_NORMAL,
+	STACK_IS_EXCEPTION,
+	STACK_IS_IRQ,
+};
+
+static enum stack_type
+analyze_stack(int cpu, struct task_struct *task,
+	      unsigned long *stack, unsigned long **stack_end, char **id)
+{
+	unsigned long *irq_stack;
+	unsigned long addr;
+	unsigned used = 0;
+
+	addr = ((unsigned long)stack & (~(THREAD_SIZE - 1)));
+	if ((unsigned long)task_stack_page(task) == addr)
+		return STACK_IS_NORMAL;
+
+	*stack_end = in_exception_stack(cpu, (unsigned long)stack,
+					 &used, id);
+	if (*stack_end)
+		return STACK_IS_EXCEPTION;
+
+	*stack_end = (unsigned long *)per_cpu(irq_stack_ptr, cpu);
+	if (!*stack_end)
+		return STACK_IS_UNKNOWN;
+
+	irq_stack = *stack_end - irq_stack_size;
+
+	if (in_irq_stack(stack, irq_stack, *stack_end))
+		return STACK_IS_IRQ;
+
+	return STACK_IS_UNKNOWN;
+}
+
 /*
  * x86-64 can have up to three kernel stacks:
  * process stack
@@ -116,12 +155,11 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		const struct stacktrace_ops *ops, void *data)
 {
 	const unsigned cpu = get_cpu();
-	unsigned long *irq_stack_end =
-		(unsigned long *)per_cpu(irq_stack_ptr, cpu);
-	unsigned used = 0;
 	struct thread_info *tinfo;
-	int graph = 0;
+	unsigned long *irq_stack;
 	unsigned long dummy;
+	int graph = 0;
+	int done = 0;
 
 	if (!task)
 		task = current;
@@ -143,49 +181,60 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 	 * exceptions
 	 */
 	tinfo = task_thread_info(task);
-	for (;;) {
+	while (!done) {
+		unsigned long *stack_end;
+		enum stack_type stype;
 		char *id;
-		unsigned long *estack_end;
-		estack_end = in_exception_stack(cpu, (unsigned long)stack,
-						&used, &id);
 
-		if (estack_end) {
+		stype = analyze_stack(cpu, task, stack, &stack_end, &id);
+
+		/* Default finish unless specified to continue */
+		done = 1;
+
+		switch (stype) {
+
+		/* Break out early if we are on the thread stack */
+		case STACK_IS_NORMAL:
+			break;
+
+		case STACK_IS_EXCEPTION:
+
 			if (ops->stack(data, id) < 0)
 				break;
 
 			bp = ops->walk_stack(tinfo, stack, bp, ops,
-					     data, estack_end, &graph);
+					     data, stack_end, &graph);
 			ops->stack(data, "<EOE>");
 			/*
 			 * We link to the next stack via the
 			 * second-to-last pointer (index -2 to end) in the
 			 * exception stack:
 			 */
-			stack = (unsigned long *) estack_end[-2];
-			continue;
-		}
-		if (irq_stack_end) {
-			unsigned long *irq_stack;
-			irq_stack = irq_stack_end -
-				(IRQ_STACK_SIZE - 64) / sizeof(*irq_stack);
-
-			if (in_irq_stack(stack, irq_stack, irq_stack_end)) {
-				if (ops->stack(data, "IRQ") < 0)
-					break;
-				bp = ops->walk_stack(tinfo, stack, bp,
-					ops, data, irq_stack_end, &graph);
-				/*
-				 * We link to the next stack (which would be
-				 * the process stack normally) the last
-				 * pointer (index -1 to end) in the IRQ stack:
-				 */
-				stack = (unsigned long *) (irq_stack_end[-1]);
-				irq_stack_end = NULL;
-				ops->stack(data, "EOI");
-				continue;
-			}
+			stack = (unsigned long *) stack_end[-2];
+			done = 0;
+			break;
+
+		case STACK_IS_IRQ:
+
+			if (ops->stack(data, "IRQ") < 0)
+				break;
+			bp = ops->walk_stack(tinfo, stack, bp,
+				     ops, data, stack_end, &graph);
+			/*
+			 * We link to the next stack (which would be
+			 * the process stack normally) the last
+			 * pointer (index -1 to end) in the IRQ stack:
+			 */
+			stack = (unsigned long *) (stack_end[-1]);
+			irq_stack = stack_end - irq_stack_size;
+			ops->stack(data, "EOI");
+			done = 0;
+			break;
+
+		case STACK_IS_UNKNOWN:
+			ops->stack(data, "UNK");
+			break;
 		}
-		break;
 	}
 
 	/*
Signature · 1 line
1.8.4.3
#16 lore raw

PATCH 4/5 x86: Keep thread_info on thread stack in x86_32

In reply to Steven Rostedt

From: Steven Rostedt <[email protected]>

x86_64 uses a per_cpu variable kernel_stack to always point to the thread stack of current. This is where the thread_info is stored and is accessed from this location even when the irq or exception stack is in use. This removes the complexity of having to maintain the thread info on the stack when interrupts are running and having to copy the preempt_count and other fields to the interrupt stack.

x86_32 uses the old method of copying the thread_info from the thread stack to the exception stack just before executing the exception.

Having the two different requires #ifdefs and also the x86_32 way is a bit of a pain to maintain. By converting x86_32 to the same method of x86_64, we can remove #ifdefs, clean up the x86_32 code a little, and remove the overhead of the copy.

Cc Andrew Morton <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc H. Peter Anvin <[email protected]>
Cc Ingo Molnar <[email protected]>
---
 arch/x86/include/asm/processor.h   |  9 +++++
 arch/x86/include/asm/thread_info.h | 48 ++++---------------------
 arch/x86/kernel/cpu/common.c       |  8 ++---
 arch/x86/kernel/dumpstack_32.c     | 41 +++++++++++++++++----
 arch/x86/kernel/irq_32.c           | 74 ++++++++++++++++----------------------
 arch/x86/kernel/process_32.c       |  4 +++
 arch/x86/kernel/ptrace.c           |  2 +-
 arch/x86/kernel/smpboot.c          |  2 +-
 8 files changed, 90 insertions(+), 98 deletions(-)
arch/x86/include/asm/processor.h +9 -0
index 7b034a4..d787f40 100644
@@ -432,6 +432,15 @@ struct stack_canary {
 };
 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
+/*
+ * per-CPU IRQ handling stacks
+ */
+struct irq_stack {
+	u32                     stack[THREAD_SIZE/sizeof(u32)];
+} __aligned(THREAD_SIZE);
+
+DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
+DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
 #endif	/* X86_64 */
 
 extern unsigned int xstate_size;
arch/x86/include/asm/thread_info.h +6 -42

79 lines not on this page · raw message

arch/x86/kernel/cpu/common.c +4 -4
index 6abc172..e45705b 100644
@@ -1072,6 +1072,10 @@ static __init int setup_disablecpuid(char *arg)
 }
 __setup("clearcpuid=", setup_disablecpuid);
 
+DEFINE_PER_CPU(unsigned long, kernel_stack) =
+	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+EXPORT_PER_CPU_SYMBOL(kernel_stack);
+
 #ifdef CONFIG_X86_64
 struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
 struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
@@ -1088,10 +1092,6 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
 	&init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
 
-DEFINE_PER_CPU(unsigned long, kernel_stack) =
-	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
-EXPORT_PER_CPU_SYMBOL(kernel_stack);
-
 DEFINE_PER_CPU(char *, irq_stack_ptr) =
 	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
 
arch/x86/kernel/dumpstack_32.c +34 -7

69 lines not on this page · raw message

arch/x86/kernel/irq_32.c +31 -43

125 lines not on this page · raw message

arch/x86/kernel/process_32.c +4 -0
index 6f1236c..09c8380 100644
@@ -315,6 +315,10 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	 */
 	arch_end_context_switch(next_p);
 
+	this_cpu_write(kernel_stack,
+		  (unsigned long)task_stack_page(next_p) +
+		  THREAD_SIZE - KERNEL_STACK_OFFSET);
+
 	/*
 	 * Restore %gs if needed (which is common)
 	 */
arch/x86/kernel/ptrace.c +1 -1
index f352a7c..678c0ad 100644
@@ -189,7 +189,7 @@ unsigned long kernel_stack_pointer(struct pt_regs *regs)
 	if (context == (sp & ~(THREAD_SIZE - 1)))
 		return sp;
 
-	prev_esp = (u32 *)(context + sizeof(struct thread_info) - sizeof(long));
+	prev_esp = (u32 *)(context);
 	if (prev_esp)
 		return (unsigned long)prev_esp;
 
arch/x86/kernel/smpboot.c +1 -1
index 85dc05a..7027585 100644
@@ -758,10 +758,10 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
 #else
 	clear_tsk_thread_flag(idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
+#endif
 	per_cpu(kernel_stack, cpu) =
 		(unsigned long)task_stack_page(idle) -
 		KERNEL_STACK_OFFSET + THREAD_SIZE;
-#endif
 	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
 	initial_code = (unsigned long)start_secondary;
 	stack_start  = idle->thread.sp;
Signature · 1 line
1.8.4.3
#22 lore raw

Re: [PATCH 0/5] [GIT PULL] x86: Consolidate 32 and 64 bit thead_info (again)

In reply to Steven Rostedt

On Thu, 06 Feb 2014 09:41:27 -0500
Steven Rostedt <[email protected]> wrote:

HPA, Ping?

-- Steve

Quoted 45 lines

Peter,

Back in August 2011, I had a pull request to make the irq stack of i386 more like the x86_64 code. There seemed to be acceptance back then and you said you would even pull it after the merge window. But due to your travels it was forgotten (and I forgot about it too).

http://lkml.kernel.org/r/[email protected]

I stumbled on these changes again and noticed that they were never applied so I forward ported them with a few more adjustments.

Hopefully they are still worth pulling. This time I waited till after the merge window to post :-)

Please pull the latest tip/x86/core tree, which can be found at:

git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git
tip/x86/core

Head SHA1: 3132e107d608f8753240d82d61303c500fd515b4 237dfcadc93749ca5cdd2f683f887c3dcf3a0c04

Steven Rostedt (4):
x86: Nuke the supervisor_stack field in i386 thread_info
x86: Prepare removal of previous_esp from i386 thread_info structure
x86: Keep thread_info on thread stack in x86_32
x86: Clean up dumpstack_64.c code

Steven Rostedt (Red Hat) (1):
x86: Nuke GET_THREAD_INFO_WITH_ESP() macro for i386

----

 arch/x86/include/asm/processor.h   |   9 +++
 arch/x86/include/asm/thread_info.h |  52 ++---------------
 arch/x86/kernel/cpu/common.c       |   8 +--
 arch/x86/kernel/dumpstack_32.c     |  44 ++++++++++++--
 arch/x86/kernel/dumpstack_64.c     | 117 ++++++++++++++++++++++++++-----------
 arch/x86/kernel/irq_32.c           |  79 ++++++++++++-------------
 arch/x86/kernel/process_32.c       |   4 ++
 arch/x86/kernel/ptrace.c           |   8 +--
 arch/x86/kernel/smpboot.c          |   2 +-
 9 files changed, 188 insertions(+), 135 deletions(-)
#13 lore raw

[tip:x86/threadinfo] x86: Nuke GET_THREAD_INFO_WITH_ESP() macro for i386

In reply to Steven Rostedt
x86: Nuke GET_THREAD_INFO_WITH_ESP() macro for i386
Commit-ID:  b807902a88c470eb06d0acdf3b6590f27f5dce81
Gitweb:     http://git.kernel.org/tip/b807902a88c470eb06d0acdf3b6590f27f5dce81
Author:     Steven Rostedt (Red Hat) <[email protected]>
AuthorDate: Thu, 6 Feb 2014 09:41:29 -0500
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 6 Mar 2014 16:56:54 -0800

x86: Nuke GET_THREAD_INFO_WITH_ESP() macro for i386

According to a git log -p, GET_THREAD_INFO_WITH_ESP() has only been defined and never been used. Get rid of it.

---
 arch/x86/include/asm/thread_info.h | 4 ----
 1 file changed, 4 deletions(-)
arch/x86/include/asm/thread_info.h +0 -4
index b7aa975..1cb3501 100644
@@ -182,10 +182,6 @@ static inline struct thread_info *current_thread_info(void)
 	movl $-THREAD_SIZE, reg; \
 	andl %esp, reg
 
-/* use this one if reg already contains %esp */
-#define GET_THREAD_INFO_WITH_ESP(reg) \
-	andl $-THREAD_SIZE, reg
-
 #endif
 
 #else /* X86_32 */
#15 lore raw

[tip:x86/threadinfo] x86: Prepare removal of previous_esp from i386 thread_info structure

In reply to Steven Rostedt
x86: Prepare removal of previous_esp from i386 thread_info structure
Commit-ID:  0788aa6a23cb9d693fc5040ec774b979f1e906cd
Gitweb:     http://git.kernel.org/tip/0788aa6a23cb9d693fc5040ec774b979f1e906cd
Author:     Steven Rostedt <[email protected]>
AuthorDate: Thu, 6 Feb 2014 09:41:30 -0500
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 6 Mar 2014 16:56:54 -0800

x86: Prepare removal of previous_esp from i386 thread_info structure

The i386 thread_info contains a previous_esp field that is used to daisy chain the different stacks for dump_stack() (ie. irq, softirq, thread stacks).

The goal is to eventual make i386 handling of thread_info the same as x86_64, which means that the thread_info will not be in the stack but as a per_cpu variable. We will no longer depend on thread_info being able to daisy chain different stacks as it will only exist in one location (the thread stack).

By moving previous_esp to the end of thread_info and referencing it as an offset instead of using a thread_info field, this becomes a stepping stone to moving the thread_info.

The offset to get to the previous stack is rather ugly in this patch, but this is only temporary and the prev_esp will be changed in the next commit. This commit is more for sanity checks of the change.

---
 arch/x86/include/asm/thread_info.h |  5 +++--
 arch/x86/kernel/dumpstack_32.c     | 11 ++++++++++-
 arch/x86/kernel/irq_32.c           | 15 +++++++++++----
 arch/x86/kernel/ptrace.c           |  8 ++++----
 4 files changed, 28 insertions(+), 11 deletions(-)
arch/x86/include/asm/thread_info.h +3 -2
index 1cb3501..ca2de1b 100644
@@ -32,13 +32,14 @@ struct thread_info {
 	mm_segment_t		addr_limit;
 	struct restart_block    restart_block;
 	void __user		*sysenter_return;
+	unsigned int		sig_on_uaccess_error:1;
+	unsigned int		uaccess_err:1;	/* uaccess failed */
 #ifdef CONFIG_X86_32
 	unsigned long           previous_esp;   /* ESP of the previous stack in
 						   case of nested (IRQ) stacks
+						   (Moved to end, to be removed soon)
 						*/
 #endif
-	unsigned int		sig_on_uaccess_error:1;
-	unsigned int		uaccess_err:1;	/* uaccess failed */
 };
 
 #define INIT_THREAD_INFO(tsk)			\
arch/x86/kernel/dumpstack_32.c +10 -1
index f2a1770..187d6a7 100644
@@ -22,6 +22,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		const struct stacktrace_ops *ops, void *data)
 {
 	int graph = 0;
+	u32 *prev_esp;
 
 	if (!task)
 		task = current;
@@ -44,9 +45,17 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 			((unsigned long)stack & (~(THREAD_SIZE - 1)));
 		bp = ops->walk_stack(context, stack, bp, ops, data, NULL, &graph);
 
-		stack = (unsigned long *)context->previous_esp;
+		/* Stop if not on irq stack */
+		if (task_stack_page(task) == context)
+			break;
+
+		/* The previous esp is just above the context */
+		prev_esp = (u32 *) ((char *)context + sizeof(struct thread_info) -
+				    sizeof(long));
+		stack = (unsigned long *)*prev_esp;
 		if (!stack)
 			break;
+
 		if (ops->stack(data, "IRQ") < 0)
 			break;
 		touch_nmi_watchdog();
arch/x86/kernel/irq_32.c +11 -4
index d7fcbed..f135cc2 100644
@@ -81,7 +81,7 @@ static inline int
 execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 {
 	union irq_ctx *curctx, *irqctx;
-	u32 *isp, arg1, arg2;
+	u32 *isp, *prev_esp, arg1, arg2;
 
 	curctx = (union irq_ctx *) current_thread_info();
 	irqctx = __this_cpu_read(hardirq_ctx);
@@ -98,7 +98,10 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 	/* build the stack frame on the IRQ stack */
 	isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
 	irqctx->tinfo.task = curctx->tinfo.task;
-	irqctx->tinfo.previous_esp = current_stack_pointer;
+	/* Save the next esp after thread_info */
+	prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info) -
+			    sizeof(long));
+	*prev_esp = current_stack_pointer;
 
 	if (unlikely(overflow))
 		call_on_stack(print_stack_overflow, isp);
@@ -149,16 +152,20 @@ void do_softirq_own_stack(void)
 {
 	struct thread_info *curctx;
 	union irq_ctx *irqctx;
-	u32 *isp;
+	u32 *isp, *prev_esp;
 
 	curctx = current_thread_info();
 	irqctx = __this_cpu_read(softirq_ctx);
 	irqctx->tinfo.task = curctx->task;
-	irqctx->tinfo.previous_esp = current_stack_pointer;
 
 	/* build the stack frame on the softirq stack */
 	isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
 
+	/* Push the previous esp onto the stack */
+	prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info) -
+			    sizeof(long));
+	*prev_esp = current_stack_pointer;
+
 	call_on_stack(__do_softirq, isp);
 }
 
arch/x86/kernel/ptrace.c +4 -4
index 7461f50..f352a7c 100644
@@ -184,14 +184,14 @@ unsigned long kernel_stack_pointer(struct pt_regs *regs)
 {
 	unsigned long context = (unsigned long)regs & ~(THREAD_SIZE - 1);
 	unsigned long sp = (unsigned long)&regs->sp;
-	struct thread_info *tinfo;
+	u32 *prev_esp;
 
 	if (context == (sp & ~(THREAD_SIZE - 1)))
 		return sp;
 
-	tinfo = (struct thread_info *)context;
-	if (tinfo->previous_esp)
-		return tinfo->previous_esp;
+	prev_esp = (u32 *)(context + sizeof(struct thread_info) - sizeof(long));
+	if (prev_esp)
+		return (unsigned long)prev_esp;
 
 	return (unsigned long)regs;
 }
#11 lore raw

[tip:x86/threadinfo] x86: Nuke the supervisor_stack field in i386 thread_info

In reply to Steven Rostedt
x86: Nuke the supervisor_stack field in i386 thread_info
Commit-ID:  2432e1364bbee83cb6e63e026c91785031704ba5
Gitweb:     http://git.kernel.org/tip/2432e1364bbee83cb6e63e026c91785031704ba5
Author:     Steven Rostedt <[email protected]>
AuthorDate: Thu, 6 Feb 2014 09:41:28 -0500
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 6 Mar 2014 16:56:54 -0800

x86: Nuke the supervisor_stack field in i386 thread_info

Nothing references the supervisor_stack in the thread_info field, and it does not exist in x86_64. To make the two more the same, it is being removed.

---
 arch/x86/include/asm/thread_info.h | 1 -
 1 file changed, 1 deletion(-)
arch/x86/include/asm/thread_info.h +0 -1
index e1940c0..b7aa975 100644
@@ -36,7 +36,6 @@ struct thread_info {
 	unsigned long           previous_esp;   /* ESP of the previous stack in
 						   case of nested (IRQ) stacks
 						*/
-	__u8			supervisor_stack[0];
 #endif
 	unsigned int		sig_on_uaccess_error:1;
 	unsigned int		uaccess_err:1;	/* uaccess failed */
#17 lore raw

[tip:x86/threadinfo] x86: Keep thread_info on thread stack in x86_32

In reply to Steven Rostedt
x86: Keep thread_info on thread stack in x86_32
Commit-ID:  198d208df4371734ac4728f69cb585c284d20a15
Gitweb:     http://git.kernel.org/tip/198d208df4371734ac4728f69cb585c284d20a15
Author:     Steven Rostedt <[email protected]>
AuthorDate: Thu, 6 Feb 2014 09:41:31 -0500
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 6 Mar 2014 16:56:55 -0800

x86: Keep thread_info on thread stack in x86_32

x86_64 uses a per_cpu variable kernel_stack to always point to the thread stack of current. This is where the thread_info is stored and is accessed from this location even when the irq or exception stack is in use. This removes the complexity of having to maintain the thread info on the stack when interrupts are running and having to copy the preempt_count and other fields to the interrupt stack.

x86_32 uses the old method of copying the thread_info from the thread stack to the exception stack just before executing the exception.

Having the two different requires #ifdefs and also the x86_32 way is a bit of a pain to maintain. By converting x86_32 to the same method of x86_64, we can remove #ifdefs, clean up the x86_32 code a little, and remove the overhead of the copy.

---
 arch/x86/include/asm/processor.h   |  9 +++++
 arch/x86/include/asm/thread_info.h | 49 +++----------------------
 arch/x86/kernel/cpu/common.c       |  8 ++---
 arch/x86/kernel/dumpstack_32.c     | 41 +++++++++++++++++----
 arch/x86/kernel/irq_32.c           | 74 ++++++++++++++++----------------------
 arch/x86/kernel/process_32.c       |  4 +++
 arch/x86/kernel/ptrace.c           |  2 +-
 arch/x86/kernel/smpboot.c          |  2 +-
 8 files changed, 89 insertions(+), 100 deletions(-)
arch/x86/include/asm/processor.h +9 -0
index fdedd38..a4ea023 100644
@@ -449,6 +449,15 @@ struct stack_canary {
 };
 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
+/*
+ * per-CPU IRQ handling stacks
+ */
+struct irq_stack {
+	u32                     stack[THREAD_SIZE/sizeof(u32)];
+} __aligned(THREAD_SIZE);
+
+DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
+DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
 #endif	/* X86_64 */
 
 extern unsigned int xstate_size;
arch/x86/include/asm/thread_info.h +5 -44

86 lines not on this page · raw message

arch/x86/kernel/cpu/common.c +4 -4
index 8e28bf2..29c1944 100644
@@ -1078,6 +1078,10 @@ static __init int setup_disablecpuid(char *arg)
 }
 __setup("clearcpuid=", setup_disablecpuid);
 
+DEFINE_PER_CPU(unsigned long, kernel_stack) =
+	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+EXPORT_PER_CPU_SYMBOL(kernel_stack);
+
 #ifdef CONFIG_X86_64
 struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
 struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
@@ -1094,10 +1098,6 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
 	&init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
 
-DEFINE_PER_CPU(unsigned long, kernel_stack) =
-	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
-EXPORT_PER_CPU_SYMBOL(kernel_stack);
-
 DEFINE_PER_CPU(char *, irq_stack_ptr) =
 	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
 
arch/x86/kernel/dumpstack_32.c +34 -7

69 lines not on this page · raw message

arch/x86/kernel/irq_32.c +31 -43

125 lines not on this page · raw message

arch/x86/kernel/process_32.c +4 -0
index 0de43e9..7bc86bb 100644
@@ -314,6 +314,10 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	 */
 	arch_end_context_switch(next_p);
 
+	this_cpu_write(kernel_stack,
+		  (unsigned long)task_stack_page(next_p) +
+		  THREAD_SIZE - KERNEL_STACK_OFFSET);
+
 	/*
 	 * Restore %gs if needed (which is common)
 	 */
arch/x86/kernel/ptrace.c +1 -1
index f352a7c..678c0ad 100644
@@ -189,7 +189,7 @@ unsigned long kernel_stack_pointer(struct pt_regs *regs)
 	if (context == (sp & ~(THREAD_SIZE - 1)))
 		return sp;
 
-	prev_esp = (u32 *)(context + sizeof(struct thread_info) - sizeof(long));
+	prev_esp = (u32 *)(context);
 	if (prev_esp)
 		return (unsigned long)prev_esp;
 
arch/x86/kernel/smpboot.c +1 -1
index a32da80..867d53e 100644
@@ -758,10 +758,10 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
 #else
 	clear_tsk_thread_flag(idle, TIF_FORK);
 	initial_gs = per_cpu_offset(cpu);
+#endif
 	per_cpu(kernel_stack, cpu) =
 		(unsigned long)task_stack_page(idle) -
 		KERNEL_STACK_OFFSET + THREAD_SIZE;
-#endif
 	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
 	initial_code = (unsigned long)start_secondary;
 	stack_start  = idle->thread.sp;
#21 lore raw

[tip:x86/threadinfo] x86: Clean up dumpstack_64.c code

In reply to Steven Rostedt
x86: Clean up dumpstack_64.c code
Commit-ID:  2223f6f6eeaad6ea0a3acd3f4bc1ae45e8117ddc
Gitweb:     http://git.kernel.org/tip/2223f6f6eeaad6ea0a3acd3f4bc1ae45e8117ddc
Author:     Steven Rostedt <[email protected]>
AuthorDate: Thu, 6 Feb 2014 09:41:32 -0500
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Thu, 6 Mar 2014 16:56:55 -0800

x86: Clean up dumpstack_64.c code

The dump_trace() function in dumpstack_64.c is hard to follow. The test for exception stack is processed differently than the test for irq stack, and the normal stack is outside completely.

By restructuring this code to have all the stacks determined by a single function that returns an enum of the following:

STACK_IS_NORMAL
STACK_IS_EXCEPTION
STACK_IS_IRQ
STACK_IS_UNKNOWN

and has the logic of each within a switch statement. This should make the code much easier to read and understand.

---
 arch/x86/kernel/dumpstack_64.c | 117 +++++++++++++++++++++++++++++------------
 1 file changed, 83 insertions(+), 34 deletions(-)
arch/x86/kernel/dumpstack_64.c +83 -34
index addb207..346b1df 100644
@@ -104,6 +104,45 @@ in_irq_stack(unsigned long *stack, unsigned long *irq_stack,
 	return (stack >= irq_stack && stack < irq_stack_end);
 }
 
+static const unsigned long irq_stack_size =
+	(IRQ_STACK_SIZE - 64) / sizeof(unsigned long);
+
+enum stack_type {
+	STACK_IS_UNKNOWN,
+	STACK_IS_NORMAL,
+	STACK_IS_EXCEPTION,
+	STACK_IS_IRQ,
+};
+
+static enum stack_type
+analyze_stack(int cpu, struct task_struct *task,
+	      unsigned long *stack, unsigned long **stack_end, char **id)
+{
+	unsigned long *irq_stack;
+	unsigned long addr;
+	unsigned used = 0;
+
+	addr = ((unsigned long)stack & (~(THREAD_SIZE - 1)));
+	if ((unsigned long)task_stack_page(task) == addr)
+		return STACK_IS_NORMAL;
+
+	*stack_end = in_exception_stack(cpu, (unsigned long)stack,
+					 &used, id);
+	if (*stack_end)
+		return STACK_IS_EXCEPTION;
+
+	*stack_end = (unsigned long *)per_cpu(irq_stack_ptr, cpu);
+	if (!*stack_end)
+		return STACK_IS_UNKNOWN;
+
+	irq_stack = *stack_end - irq_stack_size;
+
+	if (in_irq_stack(stack, irq_stack, *stack_end))
+		return STACK_IS_IRQ;
+
+	return STACK_IS_UNKNOWN;
+}
+
 /*
  * x86-64 can have up to three kernel stacks:
  * process stack
@@ -116,12 +155,11 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 		const struct stacktrace_ops *ops, void *data)
 {
 	const unsigned cpu = get_cpu();
-	unsigned long *irq_stack_end =
-		(unsigned long *)per_cpu(irq_stack_ptr, cpu);
-	unsigned used = 0;
 	struct thread_info *tinfo;
-	int graph = 0;
+	unsigned long *irq_stack;
 	unsigned long dummy;
+	int graph = 0;
+	int done = 0;
 
 	if (!task)
 		task = current;
@@ -143,49 +181,60 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 	 * exceptions
 	 */
 	tinfo = task_thread_info(task);
-	for (;;) {
+	while (!done) {
+		unsigned long *stack_end;
+		enum stack_type stype;
 		char *id;
-		unsigned long *estack_end;
-		estack_end = in_exception_stack(cpu, (unsigned long)stack,
-						&used, &id);
 
-		if (estack_end) {
+		stype = analyze_stack(cpu, task, stack, &stack_end, &id);
+
+		/* Default finish unless specified to continue */
+		done = 1;
+
+		switch (stype) {
+
+		/* Break out early if we are on the thread stack */
+		case STACK_IS_NORMAL:
+			break;
+
+		case STACK_IS_EXCEPTION:
+
 			if (ops->stack(data, id) < 0)
 				break;
 
 			bp = ops->walk_stack(tinfo, stack, bp, ops,
-					     data, estack_end, &graph);
+					     data, stack_end, &graph);
 			ops->stack(data, "<EOE>");
 			/*
 			 * We link to the next stack via the
 			 * second-to-last pointer (index -2 to end) in the
 			 * exception stack:
 			 */
-			stack = (unsigned long *) estack_end[-2];
-			continue;
-		}
-		if (irq_stack_end) {
-			unsigned long *irq_stack;
-			irq_stack = irq_stack_end -
-				(IRQ_STACK_SIZE - 64) / sizeof(*irq_stack);
-
-			if (in_irq_stack(stack, irq_stack, irq_stack_end)) {
-				if (ops->stack(data, "IRQ") < 0)
-					break;
-				bp = ops->walk_stack(tinfo, stack, bp,
-					ops, data, irq_stack_end, &graph);
-				/*
-				 * We link to the next stack (which would be
-				 * the process stack normally) the last
-				 * pointer (index -1 to end) in the IRQ stack:
-				 */
-				stack = (unsigned long *) (irq_stack_end[-1]);
-				irq_stack_end = NULL;
-				ops->stack(data, "EOI");
-				continue;
-			}
+			stack = (unsigned long *) stack_end[-2];
+			done = 0;
+			break;
+
+		case STACK_IS_IRQ:
+
+			if (ops->stack(data, "IRQ") < 0)
+				break;
+			bp = ops->walk_stack(tinfo, stack, bp,
+				     ops, data, stack_end, &graph);
+			/*
+			 * We link to the next stack (which would be
+			 * the process stack normally) the last
+			 * pointer (index -1 to end) in the IRQ stack:
+			 */
+			stack = (unsigned long *) (stack_end[-1]);
+			irq_stack = stack_end - irq_stack_size;
+			ops->stack(data, "EOI");
+			done = 0;
+			break;
+
+		case STACK_IS_UNKNOWN:
+			ops->stack(data, "UNK");
+			break;
 		}
-		break;
 	}
 
 	/*
#18 lore raw

PATCH 1/1 x86: redo "x86: Use inline assembler to get sp"

In reply to tip-bot for Steven Rostedt

This patch restores the changes of commit dff38e3e93 "x86: Use inline assembler instead of global register variable to get sp". They got lost in commit 198d208df4 "x86: Keep thread_info on thread stack in x86_32" while moving the code to arch/x86/kernel/irq_32.c.

Quoting Andi from commit dff38e3e93:

"""
LTO in gcc 4.6/47. has trouble with global register variables. They were used to read the stack pointer. Use a simple inline assembler statement with a mov instead.

This also helps LLVM/clang, which does not support global register variables.
"""

Cc Steven Rostedt <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Brian Gerst <[email protected]>
Cc Andi Kleen <[email protected]>
Cc H. Peter Anvin <[email protected]>

---

If it's not obvious, this patch should be applied to tip.git on top of tip/x86/threadinfo.

 arch/x86/kernel/irq_32.c |    6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
arch/x86/kernel/irq_32.c +5 -1
index 988dc8bcae..63ce838e5a 100644
@@ -70,7 +70,11 @@ static void call_on_stack(void *func, void *stack)
 }
 
 /* how to get the current stack pointer from C */
-register unsigned long current_stack_pointer asm("esp") __used;
+#define current_stack_pointer ({		\
+	unsigned long sp;			\
+	asm("mov %%esp,%0" : "=g" (sp));	\
+	sp;					\
+})
 
 static inline void *current_stack(void)
 {
Signature · 1 line
1.7.10.4
#19 lore raw

[tip:x86/threadinfo] x86, threadinfo: Redo "x86: Use inline assembler to get sp"

x86, threadinfo: Redo "x86: Use inline assembler to get sp"
Commit-ID:  6cce16f99d7be23cec7cabdf32a8166eec6e5393
Gitweb:     http://git.kernel.org/tip/6cce16f99d7be23cec7cabdf32a8166eec6e5393
Author:     Mathias Krause <[email protected]>
AuthorDate: Fri, 7 Mar 2014 08:52:32 +0100
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Mon, 10 Mar 2014 17:32:01 -0700

x86, threadinfo: Redo "x86: Use inline assembler to get sp"

This patch restores the changes of commit dff38e3e93 "x86: Use inline assembler instead of global register variable to get sp". They got lost in commit 198d208df4 "x86: Keep thread_info on thread stack in x86_32" while moving the code to arch/x86/kernel/irq_32.c.

Quoting Andi from commit dff38e3e93:

"""
LTO in gcc 4.6/47. has trouble with global register variables. They were used to read the stack pointer. Use a simple inline assembler statement with a mov instead.

This also helps LLVM/clang, which does not support global register variables.
"""

---
 arch/x86/kernel/irq_32.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
arch/x86/kernel/irq_32.c +5 -1
index 988dc8b..63ce838 100644
@@ -70,7 +70,11 @@ static void call_on_stack(void *func, void *stack)
 }
 
 /* how to get the current stack pointer from C */
-register unsigned long current_stack_pointer asm("esp") __used;
+#define current_stack_pointer ({		\
+	unsigned long sp;			\
+	asm("mov %%esp,%0" : "=g" (sp));	\
+	sp;					\
+})
 
 static inline void *current_stack(void)
 {