Skip to contentSkip to conversation

x86: use register variable to get stack pointer value

PATCH In review Andrey Ryabinin 4 participants 5 messages Sep 29, 2017

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86: use register variable to get stack pointer value

Currently we use current_stack_pointer() function to get the value of the stack pointer register. Since commit f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang") we have stack register variable declared. It can be used instead of current_stack_pointer() function which allows to optimize away some excessive "mov %rsp, %<dst>" instructions:

-mov    %rsp,%rdx
-sub    %rdx,%rax
-cmp    $0x3fff,%rax
-ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>
+sub    %rsp,%rax
+cmp    $0x3fff,%rax
+ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer and use it instead of removed function.

---
 arch/x86/include/asm/asm.h         |  4 ++--
 arch/x86/include/asm/thread_info.h | 11 -----------
 arch/x86/kernel/irq_32.c           |  6 +++---
 arch/x86/kernel/traps.c            |  2 +-
 arch/x86/mm/tlb.c                  |  2 +-
 5 files changed, 7 insertions(+), 18 deletions(-)
arch/x86/include/asm/asm.h +2 -2
index 30c3c9ac784a..b0dc91f4bedc 100644
@@ -141,8 +141,8 @@
  * gets set up by the containing function.  If you forget to do this, objtool
  * may print a "call without frame pointer save/setup" warning.
  */
-register unsigned long __asm_call_sp asm(_ASM_SP);
-#define ASM_CALL_CONSTRAINT "+r" (__asm_call_sp)
+register unsigned long current_stack_pointer asm(_ASM_SP);
+#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
 #endif
 
 #endif /* _ASM_X86_ASM_H */
arch/x86/include/asm/thread_info.h +0 -11
index 5161da1a0fa0..89e7eeb5cec1 100644
@@ -158,17 +158,6 @@ struct thread_info {
  */
 #ifndef __ASSEMBLY__
 
-static inline unsigned long current_stack_pointer(void)
-{
-	unsigned long sp;
-#ifdef CONFIG_X86_64
-	asm("mov %%rsp,%0" : "=g" (sp));
-#else
-	asm("mov %%esp,%0" : "=g" (sp));
-#endif
-	return sp;
-}
-
 /*
  * Walks up the stack frames to make sure that the specified object is
  * entirely contained by a single stack frame.
arch/x86/kernel/irq_32.c +3 -3
index 1f38d9a4d9de..d4eb450144fd 100644
@@ -64,7 +64,7 @@ static void call_on_stack(void *func, void *stack)
 
 static inline void *current_stack(void)
 {
-	return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
+	return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
 }
 
 static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
@@ -88,7 +88,7 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
 
 	/* Save the next esp at the bottom of the stack */
 	prev_esp = (u32 *)irqstk;
-	*prev_esp = current_stack_pointer();
+	*prev_esp = current_stack_pointer;
 
 	if (unlikely(overflow))
 		call_on_stack(print_stack_overflow, isp);
@@ -139,7 +139,7 @@ void do_softirq_own_stack(void)
 
 	/* Push the previous esp onto the stack */
 	prev_esp = (u32 *)irqstk;
-	*prev_esp = current_stack_pointer();
+	*prev_esp = current_stack_pointer;
 
 	call_on_stack(__do_softirq, isp);
 }
arch/x86/kernel/traps.c +1 -1
index 34ea3651362e..67db4f43309e 100644
@@ -142,7 +142,7 @@ void ist_begin_non_atomic(struct pt_regs *regs)
 	 * from double_fault.
 	 */
 	BUG_ON((unsigned long)(current_top_of_stack() -
-			       current_stack_pointer()) >= THREAD_SIZE);
+			       current_stack_pointer) >= THREAD_SIZE);
 
 	preempt_enable_no_resched();
 }
arch/x86/mm/tlb.c +1 -1
index 93fe97cce581..49d9778376d7 100644
@@ -191,7 +191,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 			 * mapped in the new pgd, we'll double-fault.  Forcibly
 			 * map it.
 			 */
-			unsigned int index = pgd_index(current_stack_pointer());
+			unsigned int index = pgd_index(current_stack_pointer);
 			pgd_t *pgd = next->pgd + index;
 
 			if (unlikely(pgd_none(*pgd)))
Signature · 1 line
2.13.6
Reviewed-by on v1 1/1
#1 lore raw
Quoted 16 lines · Andrey Ryabinin

Currently we use current_stack_pointer() function to get the value of the stack pointer register. Since commit f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang") we have stack register variable declared. It can be used instead of current_stack_pointer() function which allows to optimize away some excessive "mov %rsp, %<dst>" instructions:

(quoted diff) +3 -4
-mov    %rsp,%rdx
-sub    %rdx,%rax
-cmp    $0x3fff,%rax
-ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>
 
+sub    %rsp,%rax
+cmp    $0x3fff,%rax
+ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>
Quoted 4 lines

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer and use it instead of removed function.

Signature · 1 line
Josh
maintainer
#2 lore raw
Quoted 17 lines · Josh Poimboeuf
Quoted 16 lines · Andrey Ryabinin

Currently we use current_stack_pointer() function to get the value of the stack pointer register. Since commit f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang") we have stack register variable declared. It can be used instead of current_stack_pointer() function which allows to optimize away some excessive "mov %rsp, %<dst>" instructions:

(quoted diff) +3 -4
-mov    %rsp,%rdx
-sub    %rdx,%rax
-cmp    $0x3fff,%rax
-ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>
 
+sub    %rsp,%rax
+cmp    $0x3fff,%rax
+ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>
Quoted 6 lines
Quoted 4 lines

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer and use it instead of removed function.

Ok with me. As an alternative, you could leave it as current_stack_pointer(), but either way is fine.

#3 lore raw
Quoted 21 lines · Andy Lutomirski
Quoted 20 lines · Josh Poimboeuf
Quoted 19 lines · Andrey Ryabinin

Currently we use current_stack_pointer() function to get the value of the stack pointer register. Since commit f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang") we have stack register variable declared. It can be used instead of current_stack_pointer() function which allows to optimize away some excessive "mov %rsp, %<dst>" instructions:

(quoted diff) +3 -4
-mov    %rsp,%rdx
-sub    %rdx,%rax
-cmp    $0x3fff,%rax
-ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>
 
+sub    %rsp,%rax
+cmp    $0x3fff,%rax
+ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer and use it instead of removed function.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

Ok with me. As an alternative, you could leave it as current_stack_pointer(), but either way is fine.

Good point, I think changing current_stack_pointer() to return the global variable would be cleaner.

Signature · 1 line
Josh
#4 lore raw

[tip:x86/urgent] x86/asm: Use register variable to get stack pointer value

In reply to Andrey Ryabinin
x86/asm: Use register variable to get stack pointer value
Commit-ID:  196bd485ee4f03ce4c690bfcf38138abfcd0a4bc
Gitweb:     https://git.kernel.org/tip/196bd485ee4f03ce4c690bfcf38138abfcd0a4bc
Author:     Andrey Ryabinin <[email protected]>
AuthorDate: Fri, 29 Sep 2017 17:15:36 +0300
Committer:  Ingo Molnar <[email protected]>
CommitDate: Fri, 29 Sep 2017 19:39:44 +0200

x86/asm: Use register variable to get stack pointer value

Currently we use current_stack_pointer() function to get the value of the stack pointer register. Since commit:

f5caf621ee35 ("x86/asm: Fix inline asm call constraints for Clang")

... we have a stack register variable declared. It can be used instead of current_stack_pointer() function which allows to optimize away some excessive "mov %rsp, %<dst>" instructions:

 -mov    %rsp,%rdx
 -sub    %rdx,%rax
 -cmp    $0x3fff,%rax
 -ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>
 +sub    %rsp,%rax
 +cmp    $0x3fff,%rax
 +ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer and use it instead of the removed function.

---
 arch/x86/include/asm/asm.h         |  4 ++--
 arch/x86/include/asm/thread_info.h | 11 -----------
 arch/x86/kernel/irq_32.c           |  6 +++---
 arch/x86/kernel/traps.c            |  2 +-
 arch/x86/mm/tlb.c                  |  2 +-
 5 files changed, 7 insertions(+), 18 deletions(-)
arch/x86/include/asm/asm.h +2 -2
index 30c3c9a..b0dc91f 100644
@@ -141,8 +141,8 @@
  * gets set up by the containing function.  If you forget to do this, objtool
  * may print a "call without frame pointer save/setup" warning.
  */
-register unsigned long __asm_call_sp asm(_ASM_SP);
-#define ASM_CALL_CONSTRAINT "+r" (__asm_call_sp)
+register unsigned long current_stack_pointer asm(_ASM_SP);
+#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
 #endif
 
 #endif /* _ASM_X86_ASM_H */
arch/x86/include/asm/thread_info.h +0 -11
index 5161da1a..89e7eeb 100644
@@ -158,17 +158,6 @@ struct thread_info {
  */
 #ifndef __ASSEMBLY__
 
-static inline unsigned long current_stack_pointer(void)
-{
-	unsigned long sp;
-#ifdef CONFIG_X86_64
-	asm("mov %%rsp,%0" : "=g" (sp));
-#else
-	asm("mov %%esp,%0" : "=g" (sp));
-#endif
-	return sp;
-}
-
 /*
  * Walks up the stack frames to make sure that the specified object is
  * entirely contained by a single stack frame.
arch/x86/kernel/irq_32.c +3 -3
index 1f38d9a..d4eb450 100644
@@ -64,7 +64,7 @@ static void call_on_stack(void *func, void *stack)
 
 static inline void *current_stack(void)
 {
-	return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
+	return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
 }
 
 static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
@@ -88,7 +88,7 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
 
 	/* Save the next esp at the bottom of the stack */
 	prev_esp = (u32 *)irqstk;
-	*prev_esp = current_stack_pointer();
+	*prev_esp = current_stack_pointer;
 
 	if (unlikely(overflow))
 		call_on_stack(print_stack_overflow, isp);
@@ -139,7 +139,7 @@ void do_softirq_own_stack(void)
 
 	/* Push the previous esp onto the stack */
 	prev_esp = (u32 *)irqstk;
-	*prev_esp = current_stack_pointer();
+	*prev_esp = current_stack_pointer;
 
 	call_on_stack(__do_softirq, isp);
 }
arch/x86/kernel/traps.c +1 -1
index 34ea365..67db4f4 100644
@@ -142,7 +142,7 @@ void ist_begin_non_atomic(struct pt_regs *regs)
 	 * from double_fault.
 	 */
 	BUG_ON((unsigned long)(current_top_of_stack() -
-			       current_stack_pointer()) >= THREAD_SIZE);
+			       current_stack_pointer) >= THREAD_SIZE);
 
 	preempt_enable_no_resched();
 }
arch/x86/mm/tlb.c +1 -1
index 93fe97c..49d9778 100644
@@ -191,7 +191,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 			 * mapped in the new pgd, we'll double-fault.  Forcibly
 			 * map it.
 			 */
-			unsigned int index = pgd_index(current_stack_pointer());
+			unsigned int index = pgd_index(current_stack_pointer);
 			pgd_t *pgd = next->pgd + index;
 
 			if (unlikely(pgd_none(*pgd)))