x86/entry/64: interrupt entry size reduction
Subsystems (2)
cover x86/entry/64: interrupt entry size reduction
This patchset applies on top of the two other tip/pti-related patches I sent out moments ago,[*] and try to implement what Linus suggested a few days ago[+].
[+] http://lkml.kernel.org/r/[email protected]
[*] http://lkml.kernel.org/r/CA+55aFwLTF3EtaQ4OpDv2UM41J=EU7gfemv=eVq+uQi31-usSg@mail.gmail.com .
Overall, these patches provide for a sizeable cutting of up to 4.35k:
text data bss dec hex filename
20987 0 0 20987 51fb entry_64.o-orig 16621 0 0 16621 40ed entry_64.o
They are split up in four small steps to easen the review. Another advantage is that we can decide whether each additional step is really worth it in relation to an increase in code complexity.
NOTE / WARNING: As usual, please be extremely stringent in reviewing these patches.
Thanks,
Dominik
Dominik Brodowski (4):
x86/entry/64: move PUSH_AND_CLEAR_REGS from interrupt macro to helper
function
x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper
function
x86/entry/64: move switch_to_thread_stack to interrupt helper function x86/entry/64: remove interrupt macro
arch/x86/entry/entry_64.S | 99 +++++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 37 deletions(-)
Signature · 1 line
2.16.1
RFC PATCH 1/4 x86/entry/64: move PUSH_AND_CLEAR_REGS from interrupt macro to helper function
The PUSH_AND_CLEAR_REGS macro is able to insert the GP registers "above" the original return address. This allows us to move a sizeable part of the interrupt entry macro to an interrupt entry helper function:
text data bss dec hex filename
20987 0 0 20987 51fb entry_64.o-orig 17905 0 0 17905 45f1 entry_64.o
--- arch/x86/entry/entry_64.S | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
arch/x86/entry/entry_64.S +10 -2
@@ -526,6 +526,15 @@ END(irq_entries_start) * * Entry runs with interrupts off. */ +ENTRY(interrupt_helper) + UNWIND_HINT_FUNC + cld + + PUSH_AND_CLEAR_REGS save_ret=1 + ENCODE_FRAME_POINTER 8 + + ret +END(interrupt_helper) /* 0(%rsp): ~(interrupt number) */ .macro interrupt func
@@ -537,8 +546,7 @@ END(irq_entries_start) call switch_to_thread_stack 1: - PUSH_AND_CLEAR_REGS - ENCODE_FRAME_POINTER + call interrupt_helper testb $3, CS(%rsp) jz 1f
Signature · 1 line
2.16.1
RFC PATCH 2/4 x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper function
In reply to Dominik BrodowskiMoving the switch to IRQ stack from the interrupt macro to the helper function requires some trickery: All ENTER_IRQ_STACK really cares about is where the "original" stack -- meaning the GP registers etc. -- is stored. Therefore, we need to offset the stored RSP value by 8 whenever ENTER_IRQ_STACK is called from within a function. In such cases, and after switching to the IRQ stack, we need to push the "original" return address (i.e. the return address from the call to the interrupt entry function) to the IRQ stack.
This trickery allows us to carve another 1k from the text size:
text data bss dec hex filename
17905 0 0 17905 45f1 entry_64.o-orig 16897 0 0 16897 4201 entry_64.o
--- arch/x86/entry/entry_64.S | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-)
arch/x86/entry/entry_64.S +35 -18
@@ -449,10 +449,18 @@ END(irq_entries_start) * * The invariant is that, if irq_count != -1, then the IRQ stack is in use. */ -.macro ENTER_IRQ_STACK regs=1 old_rsp +.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 DEBUG_ENTRY_ASSERT_IRQS_OFF movq %rsp, \old_rsp + .if \save_ret + /* + * If save_ret is set, the original stack contains one additional + * entry -- the return address. + */ + addq $8, \old_rsp + .endif + .if \regs UNWIND_HINT_REGS base=\old_rsp .endif
@@ -497,6 +505,15 @@ END(irq_entries_start) .if \regs UNWIND_HINT_REGS indirect=1 .endif + + .if \save_ret + /* + * Push the return address to the stack. This return address can + * be found at the "real" original RSP, which was offset by 8 at + * the beginning of this macro. + */ + pushq -8(\old_rsp) + .endif .endm /*
@@ -533,22 +550,7 @@ ENTRY(interrupt_helper) PUSH_AND_CLEAR_REGS save_ret=1 ENCODE_FRAME_POINTER 8 - ret -END(interrupt_helper) - -/* 0(%rsp): ~(interrupt number) */ - .macro interrupt func - cld - - testb $3, CS-ORIG_RAX(%rsp) - jz 1f - SWAPGS - call switch_to_thread_stack -1: - - call interrupt_helper - - testb $3, CS(%rsp) + testb $3, CS+8(%rsp) jz 1f /*
@@ -566,10 +568,25 @@ END(interrupt_helper) CALL_enter_from_user_mode 1: - ENTER_IRQ_STACK old_rsp=%rdi + ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 /* We entered an interrupt context - irqs are off: */ TRACE_IRQS_OFF + ret +END(interrupt_helper) + +/* 0(%rsp): ~(interrupt number) */ + .macro interrupt func + cld + + testb $3, CS-ORIG_RAX(%rsp) + jz 1f + SWAPGS + call switch_to_thread_stack +1: + + call interrupt_helper + call \func /* rdi points to pt_regs */ .endm
Signature · 1 line
2.16.1
Re: [RFC PATCH 2/4] x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper function
Quoted 30 lines · Dominik Brodowski
Moving the switch to IRQ stack from the interrupt macro to the helper function requires some trickery: All
ENTER_IRQ_STACKreally cares about is where the "original" stack -- meaning the GP registers etc. -- is stored. Therefore, we need to offset the stored RSP value by 8 wheneverENTER_IRQ_STACKis called from within a function. In such cases, and after switching to the IRQ stack, we need to push the "original" return address (i.e. the return address from the call to the interrupt entry function) to the IRQ stack.This trickery allows us to carve another 1k from the text size:
text data bss dec hex filename
17905 0 0 17905 45f1 entry_64.o-orig 16897 0 0 16897 4201 entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-)arch/x86/entry/entry_64.S +1
-1@@ -449,10 +449,18 @@ END(irq_entries_start) * * The invariant is that, if irq_count != -1, then the IRQ stack is in use. */ -.macro ENTER_IRQ_STACK regs=1 old_rsp +.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0
arch/x86/entry/entry_64.S END(irq_entries_start) · line 462
Quoted 10 lines
arch/x86/entry/entry_64.S +7
-0DEBUG_ENTRY_ASSERT_IRQS_OFF movq %rsp, \old_rsp + .if \save_ret + /* + * If save_ret is set, the original stack contains one additional + * entry -- the return address. + */ + addq $8, \old_rsp + .endif
Combine the mov and add into leaq 8(%rsp), \old_rsp.
--
Brian Gerst
Andy Lutomirski
maintainerRe: [RFC PATCH 2/4] x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper function
In reply to Dominik BrodowskiQuoted 31 lines · Dominik Brodowski
Moving the switch to IRQ stack from the interrupt macro to the helper function requires some trickery: All
ENTER_IRQ_STACKreally cares about is where the "original" stack -- meaning the GP registers etc. -- is stored. Therefore, we need to offset the stored RSP value by 8 wheneverENTER_IRQ_STACKis called from within a function. In such cases, and after switching to the IRQ stack, we need to push the "original" return address (i.e. the return address from the call to the interrupt entry function) to the IRQ stack.This trickery allows us to carve another 1k from the text size:
text data bss dec hex filename
17905 0 0 17905 45f1 entry_64.o-orig 16897 0 0 16897 4201 entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-)arch/x86/entry/entry_64.S +1
-1@@ -449,10 +449,18 @@ END(irq_entries_start) * * The invariant is that, if irq_count != -1, then the IRQ stack is in use. */ -.macro ENTER_IRQ_STACK regs=1 old_rsp +.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 DEBUG_ENTRY_ASSERT_IRQS_OFF
arch/x86/entry/entry_64.S END(irq_entries_start) · line 463
Quoted 10 lines
arch/x86/entry/entry_64.S +8
-0movq %rsp, \old_rsp + .if \save_ret + /* + * If save_ret is set, the original stack contains one additional + * entry -- the return address. + */ + addq $8, \old_rsp + .endif +
This is a bit alarming in that you now have live data below RSP. For x86_32, this would be a big no-no due to NMI. For x86_64, it might still be bad if there are code paths where NMI is switched to non-IST temporarily, which was the case at some point and might still be the case. (I think it is.) Remember that the x86_64 *kernel* ABI has no red zone.
It also means that, if you manage to hit vmalloc_fault() in here when you touch the IRQ stack, you're dead. IOW you hit:
movq \old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
which gets #PF and eats your return pointer. Debugging this will be quite nasty because you'll only hit it on really huge systems after a thread gets migrated, and even then only if you get unlucky on your stack alignment.
So can you find another way to do this?
--Andy
Re: [RFC PATCH 2/4] x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper function
Quoted 54 lines · Andy Lutomirski
Quoted 31 lines · Dominik Brodowski
Moving the switch to IRQ stack from the interrupt macro to the helper function requires some trickery: All
ENTER_IRQ_STACKreally cares about is where the "original" stack -- meaning the GP registers etc. -- is stored. Therefore, we need to offset the stored RSP value by 8 wheneverENTER_IRQ_STACKis called from within a function. In such cases, and after switching to the IRQ stack, we need to push the "original" return address (i.e. the return address from the call to the interrupt entry function) to the IRQ stack.This trickery allows us to carve another 1k from the text size:
text data bss dec hex filename
17905 0 0 17905 45f1 entry_64.o-orig 16897 0 0 16897 4201 entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-)arch/x86/entry/entry_64.S +1
-1@@ -449,10 +449,18 @@ END(irq_entries_start) * * The invariant is that, if irq_count != -1, then the IRQ stack is in use. */ -.macro ENTER_IRQ_STACK regs=1 old_rsp +.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 DEBUG_ENTRY_ASSERT_IRQS_OFFarch/x86/entry/entry_64.S END(irq_entries_start) · line 463
Quoted 10 lines
arch/x86/entry/entry_64.S +8
-0movq %rsp, \old_rsp + .if \save_ret + /* + * If save_ret is set, the original stack contains one additional + * entry -- the return address. + */ + addq $8, \old_rsp + .endif +This is a bit alarming in that you now have live data below RSP. For
x86_32, this would be a big no-no due to NMI. Forx86_64, it might still be bad if there are code paths where NMI is switched to non-IST temporarily, which was the case at some point and might still be the case. (I think it is.) Remember that thex86_64*kernel* ABI has no red zone.It also means that, if you manage to hit
vmalloc_fault()in here when you touch the IRQ stack, you're dead. IOW you hit:
Quoted 8 lines
movq \old_rsp,
PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)which gets #PF and eats your return pointer. Debugging this will be quite nasty because you'll only hit it on really huge systems after a thread gets migrated, and even then only if you get unlucky on your stack alignment.
So can you find another way to do this?
It's adding 8 to the temp register, not %rsp.
--
Brian Gerst
Andy Lutomirski
maintainerRe: [RFC PATCH 2/4] x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper function
Quoted 57 lines · Brian Gerst
Quoted 56 lines · Andy Lutomirski
Quoted 31 lines · Dominik Brodowski
Moving the switch to IRQ stack from the interrupt macro to the helper function requires some trickery: All
ENTER_IRQ_STACKreally cares about is where the "original" stack -- meaning the GP registers etc. -- is stored. Therefore, we need to offset the stored RSP value by 8 wheneverENTER_IRQ_STACKis called from within a function. In such cases, and after switching to the IRQ stack, we need to push the "original" return address (i.e. the return address from the call to the interrupt entry function) to the IRQ stack.This trickery allows us to carve another 1k from the text size:
text data bss dec hex filename
17905 0 0 17905 45f1 entry_64.o-orig 16897 0 0 16897 4201 entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-)arch/x86/entry/entry_64.S +1
-1@@ -449,10 +449,18 @@ END(irq_entries_start) * * The invariant is that, if irq_count != -1, then the IRQ stack is in use. */ -.macro ENTER_IRQ_STACK regs=1 old_rsp +.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 DEBUG_ENTRY_ASSERT_IRQS_OFFarch/x86/entry/entry_64.S END(irq_entries_start) · line 463
Quoted 10 lines
arch/x86/entry/entry_64.S +8
-0movq %rsp, \old_rsp + .if \save_ret + /* + * If save_ret is set, the original stack contains one additional + * entry -- the return address. + */ + addq $8, \old_rsp + .endif +This is a bit alarming in that you now have live data below RSP. For
x86_32, this would be a big no-no due to NMI. Forx86_64, it might still be bad if there are code paths where NMI is switched to non-IST temporarily, which was the case at some point and might still be the case. (I think it is.) Remember that thex86_64*kernel* ABI has no red zone.It also means that, if you manage to hit
vmalloc_fault()in here when you touch the IRQ stack, you're dead. IOW you hit:movq \old_rsp,
PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
Quoted 8 lines
Quoted 6 lines
which gets #PF and eats your return pointer. Debugging this will be quite nasty because you'll only hit it on really huge systems after a thread gets migrated, and even then only if you get unlucky on your stack alignment.
So can you find another way to do this?
It's adding 8 to the temp register, not %rsp.
Duh.
Re: [RFC PATCH 2/4] x86/entry/64: move ENTER_IRQ_STACK from interrupt macro to helper function
Quoted 63 lines · Andy Lutomirski
Quoted 62 lines · Brian Gerst
Quoted 61 lines · Andy Lutomirski
Quoted 31 lines · Dominik Brodowski
Moving the switch to IRQ stack from the interrupt macro to the helper function requires some trickery: All
ENTER_IRQ_STACKreally cares about is where the "original" stack -- meaning the GP registers etc. -- is stored. Therefore, we need to offset the stored RSP value by 8 wheneverENTER_IRQ_STACKis called from within a function. In such cases, and after switching to the IRQ stack, we need to push the "original" return address (i.e. the return address from the call to the interrupt entry function) to the IRQ stack.This trickery allows us to carve another 1k from the text size:
text data bss dec hex filename
17905 0 0 17905 45f1 entry_64.o-orig 16897 0 0 16897 4201 entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 53 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 18 deletions(-)arch/x86/entry/entry_64.S +1
-1@@ -449,10 +449,18 @@ END(irq_entries_start) * * The invariant is that, if irq_count != -1, then the IRQ stack is in use. */ -.macro ENTER_IRQ_STACK regs=1 old_rsp +.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 DEBUG_ENTRY_ASSERT_IRQS_OFFarch/x86/entry/entry_64.S END(irq_entries_start) · line 463
Quoted 10 lines
arch/x86/entry/entry_64.S +8
-0movq %rsp, \old_rsp + .if \save_ret + /* + * If save_ret is set, the original stack contains one additional + * entry -- the return address. + */ + addq $8, \old_rsp + .endif +This is a bit alarming in that you now have live data below RSP. For
x86_32, this would be a big no-no due to NMI. Forx86_64, it might still be bad if there are code paths where NMI is switched to non-IST temporarily, which was the case at some point and might still be the case. (I think it is.) Remember that thex86_64*kernel* ABI has no red zone.It also means that, if you manage to hit
vmalloc_fault()in here when you touch the IRQ stack, you're dead. IOW you hit:movq \old_rsp,
PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)which gets #PF and eats your return pointer. Debugging this will be quite nasty because you'll only hit it on really huge systems after a thread gets migrated, and even then only if you get unlucky on your stack alignment.
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
So can you find another way to do this?
It's adding 8 to the temp register, not %rsp.
Duh.
Even if you get a #PF when writing to the IRQ stack (which should never happen in the first place since per-cpu memory is mapped at boot time), RSP would still be below the return address and the fault won't overwrite it.
That said, the word it writes to the top of the IRQ stack is vulnerable for a short window between when it switches RSP to the IRQ stack and when it pushes old_rsp. That would only affect the unwinder during a NMI in that window though since it pushes old_rsp again. So I'd say commit 29955909 doesn't work exactly as advertised. But that has nothing to do with this patch.
--
Brian Gerst
RFC PATCH 3/4 x86/entry/64: move switch_to_thread_stack to interrupt helper function
In reply to Dominik BrodowskiWe can also move the SWAPGS and the switch_to_thread_stack to the interrupt helper function. As we do not want call depths of two, convert switch_to_thread_stack to a macro. However, as entry_64_compat.S expects switch_to_thread_stack to be a function, provide a wrapper for that, which leads to some code duplication if CONFIG_IA32_EMULATION is enabled. Therefore, the size reduction differs slightly:
With CONFIG_IA32_EMULATION enabled (-0.13k):
text data bss dec hex filename
16897 0 0 16897 4201 entry_64.o-orig 16767 0 0 16767 417f entry_64.o
With CONFIG_IA32_EMULATION disabled (-0.27k):
text data bss dec hex filename
16897 0 0 16897 4201 entry_64.o-orig 16622 0 0 16622 40ee entry_64.o
--- arch/x86/entry/entry_64.S | 65 ++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-)
arch/x86/entry/entry_64.S +36 -29
@@ -536,6 +536,31 @@ END(irq_entries_start) decl PER_CPU_VAR(irq_count) .endm +/* + * Switch to the thread stack. This is called with the IRET frame and + * orig_ax on the stack. (That is, RDI..R12 are not on the stack and + * space has not been allocated for them.) + */ +.macro DO_SWITCH_TO_THREAD_STACK + pushq %rdi + /* Need to switch before accessing the thread stack. */ + SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi + movq %rsp, %rdi + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp + UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI + + pushq 7*8(%rdi) /* regs->ss */ + pushq 6*8(%rdi) /* regs->rsp */ + pushq 5*8(%rdi) /* regs->eflags */ + pushq 4*8(%rdi) /* regs->cs */ + pushq 3*8(%rdi) /* regs->ip */ + pushq 2*8(%rdi) /* regs->orig_ax */ + pushq 8(%rdi) /* return address */ + UNWIND_HINT_FUNC + + movq (%rdi), %rdi +.endm + /* * Interrupt entry/exit. *
@@ -543,10 +568,17 @@ END(irq_entries_start) * * Entry runs with interrupts off. */ +/* 8(%rsp): ~(interrupt number) */ ENTRY(interrupt_helper) UNWIND_HINT_FUNC cld + testb $3, CS-ORIG_RAX+8(%rsp) + jz 1f + SWAPGS + DO_SWITCH_TO_THREAD_STACK +1: + PUSH_AND_CLEAR_REGS save_ret=1 ENCODE_FRAME_POINTER 8
@@ -579,12 +611,6 @@ END(interrupt_helper) .macro interrupt func cld - testb $3, CS-ORIG_RAX(%rsp) - jz 1f - SWAPGS - call switch_to_thread_stack -1: - call interrupt_helper call \func /* rdi points to pt_regs */
@@ -853,33 +879,14 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) -/* - * Switch to the thread stack. This is called with the IRET frame and - * orig_ax on the stack. (That is, RDI..R12 are not on the stack and - * space has not been allocated for them.) - */ +#if defined(CONFIG_IA32_EMULATION) +/* entry_64_compat.S::entry_INT80_compat expects this to be an ASM function */ ENTRY(switch_to_thread_stack) UNWIND_HINT_FUNC - - pushq %rdi - /* Need to switch before accessing the thread stack. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi - movq %rsp, %rdi - movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp - UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI - - pushq 7*8(%rdi) /* regs->ss */ - pushq 6*8(%rdi) /* regs->rsp */ - pushq 5*8(%rdi) /* regs->eflags */ - pushq 4*8(%rdi) /* regs->cs */ - pushq 3*8(%rdi) /* regs->ip */ - pushq 2*8(%rdi) /* regs->orig_ax */ - pushq 8(%rdi) /* return address */ - UNWIND_HINT_FUNC - - movq (%rdi), %rdi + DO_SWITCH_TO_THREAD_STACK ret END(switch_to_thread_stack) +#endif .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ENTRY(\sym)
Signature · 1 line
2.16.1
Re: [RFC PATCH 3/4] x86/entry/64: move switch_to_thread_stack to interrupt helper function
Quoted 128 lines · Dominik Brodowski
We can also move the SWAPGS and the
switch_to_thread_stackto the interrupt helper function. As we do not want call depths of two, convertswitch_to_thread_stackto a macro. However, asentry_64_compat.Sexpectsswitch_to_thread_stackto be a function, provide a wrapper for that, which leads to some code duplication ifCONFIG_IA32_EMULATIONis enabled. Therefore, the size reduction differs slightly:With
CONFIG_IA32_EMULATIONenabled (-0.13k):
text data bss dec hex filename16897 0 0 16897 4201 entry_64.o-orig 16767 0 0 16767 417f entry_64.oWith
CONFIG_IA32_EMULATIONdisabled (-0.27k):
text data bss dec hex filename16897 0 0 16897 4201 entry_64.o-orig 16622 0 0 16622 40ee entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 65 ++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-)arch/x86/entry/entry_64.S +36
-30@@ -536,6 +536,31 @@ END(irq_entries_start) decl PER_CPU_VAR(irq_count) .endm +/* + * Switch to the thread stack. This is called with the IRET frame and + * orig_ax on the stack. (That is, RDI..R12 are not on the stack and + * space has not been allocated for them.) + */ +.macro DO_SWITCH_TO_THREAD_STACK + pushq %rdi + /* Need to switch before accessing the thread stack. */ + SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi + movq %rsp, %rdi + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp + UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI + + pushq 7*8(%rdi) /* regs->ss */ + pushq 6*8(%rdi) /* regs->rsp */ + pushq 5*8(%rdi) /* regs->eflags */ + pushq 4*8(%rdi) /* regs->cs */ + pushq 3*8(%rdi) /* regs->ip */ + pushq 2*8(%rdi) /* regs->orig_ax */ + pushq 8(%rdi) /* return address */ + UNWIND_HINT_FUNC + + movq (%rdi), %rdi +.endm + /* * Interrupt entry/exit. *@@ -543,10 +568,17 @@ END(irq_entries_start) * * Entry runs with interrupts off. */ +/* 8(%rsp): ~(interrupt number) */ ENTRY(interrupt_helper) UNWIND_HINT_FUNC cld + testb $3, CS-ORIG_RAX+8(%rsp) + jz 1f + SWAPGS + DO_SWITCH_TO_THREAD_STACK +1: + PUSH_AND_CLEAR_REGS save_ret=1 ENCODE_FRAME_POINTER 8@@ -579,12 +611,6 @@ END(interrupt_helper) .macro interrupt func cld - testb $3, CS-ORIG_RAX(%rsp) - jz 1f - SWAPGS - call switch_to_thread_stack -1: - call interrupt_helper call \func /* rdi points to pt_regs */@@ -853,33 +879,14 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) -/* - * Switch to the thread stack. This is called with the IRET frame and - * orig_ax on the stack. (That is, RDI..R12 are not on the stack and - * space has not been allocated for them.) - */ +#if defined(CONFIG_IA32_EMULATION) +/* entry_64_compat.S::entry_INT80_compat expects this to be an ASM function */ ENTRY(switch_to_thread_stack) UNWIND_HINT_FUNC - - pushq %rdi - /* Need to switch before accessing the thread stack. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi - movq %rsp, %rdi - movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp - UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI - - pushq 7*8(%rdi) /* regs->ss */ - pushq 6*8(%rdi) /* regs->rsp */ - pushq 5*8(%rdi) /* regs->eflags */ - pushq 4*8(%rdi) /* regs->cs */ - pushq 3*8(%rdi) /* regs->ip */ - pushq 2*8(%rdi) /* regs->orig_ax */ - pushq 8(%rdi) /* return address */ - UNWIND_HINT_FUNC - - movq (%rdi), %rdi + DO_SWITCH_TO_THREAD_STACK ret END(switch_to_thread_stack) +#endif .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ENTRY(\sym) --
Quoted 2 lines
2.16.1
Move the macro to calling.h, and inline it into the compat entry.
--
Brian Gerst
Re: [RFC PATCH 3/4] x86/entry/64: move switch_to_thread_stack to interrupt helper function
Quoted 127 lines · Brian Gerst
Quoted 125 lines · Dominik Brodowski
We can also move the SWAPGS and the
switch_to_thread_stackto the interrupt helper function. As we do not want call depths of two, convertswitch_to_thread_stackto a macro. However, asentry_64_compat.Sexpectsswitch_to_thread_stackto be a function, provide a wrapper for that, which leads to some code duplication ifCONFIG_IA32_EMULATIONis enabled. Therefore, the size reduction differs slightly:With
CONFIG_IA32_EMULATIONenabled (-0.13k):
text data bss dec hex filename16897 0 0 16897 4201 entry_64.o-orig 16767 0 0 16767 417f entry_64.oWith
CONFIG_IA32_EMULATIONdisabled (-0.27k):
text data bss dec hex filename16897 0 0 16897 4201 entry_64.o-orig 16622 0 0 16622 40ee entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 65 ++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-)arch/x86/entry/entry_64.S +36
-29@@ -536,6 +536,31 @@ END(irq_entries_start) decl PER_CPU_VAR(irq_count) .endm +/* + * Switch to the thread stack. This is called with the IRET frame and + * orig_ax on the stack. (That is, RDI..R12 are not on the stack and + * space has not been allocated for them.) + */ +.macro DO_SWITCH_TO_THREAD_STACK + pushq %rdi + /* Need to switch before accessing the thread stack. */ + SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi + movq %rsp, %rdi + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp + UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI + + pushq 7*8(%rdi) /* regs->ss */ + pushq 6*8(%rdi) /* regs->rsp */ + pushq 5*8(%rdi) /* regs->eflags */ + pushq 4*8(%rdi) /* regs->cs */ + pushq 3*8(%rdi) /* regs->ip */ + pushq 2*8(%rdi) /* regs->orig_ax */ + pushq 8(%rdi) /* return address */ + UNWIND_HINT_FUNC + + movq (%rdi), %rdi +.endm + /* * Interrupt entry/exit. *@@ -543,10 +568,17 @@ END(irq_entries_start) * * Entry runs with interrupts off. */ +/* 8(%rsp): ~(interrupt number) */ ENTRY(interrupt_helper) UNWIND_HINT_FUNC cld + testb $3, CS-ORIG_RAX+8(%rsp) + jz 1f + SWAPGS + DO_SWITCH_TO_THREAD_STACK +1: + PUSH_AND_CLEAR_REGS save_ret=1 ENCODE_FRAME_POINTER 8@@ -579,12 +611,6 @@ END(interrupt_helper) .macro interrupt func cld - testb $3, CS-ORIG_RAX(%rsp) - jz 1f - SWAPGS - call switch_to_thread_stack -1: - call interrupt_helper call \func /* rdi points to pt_regs */@@ -853,33 +879,14 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) -/* - * Switch to the thread stack. This is called with the IRET frame and - * orig_ax on the stack. (That is, RDI..R12 are not on the stack and - * space has not been allocated for them.) - */ +#if defined(CONFIG_IA32_EMULATION) +/* entry_64_compat.S::entry_INT80_compat expects this to be an ASM function */ ENTRY(switch_to_thread_stack) UNWIND_HINT_FUNC - - pushq %rdi - /* Need to switch before accessing the thread stack. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi - movq %rsp, %rdi - movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp - UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI - - pushq 7*8(%rdi) /* regs->ss */ - pushq 6*8(%rdi) /* regs->rsp */ - pushq 5*8(%rdi) /* regs->eflags */ - pushq 4*8(%rdi) /* regs->cs */ - pushq 3*8(%rdi) /* regs->ip */ - pushq 2*8(%rdi) /* regs->orig_ax */ - pushq 8(%rdi) /* return address */ - UNWIND_HINT_FUNC - - movq (%rdi), %rdi + DO_SWITCH_TO_THREAD_STACK ret END(switch_to_thread_stack) +#endif
Quoted 7 lines
Quoted 5 lines
arch/x86/entry/entry_64.S +0
-1.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ENTRY(\sym) --2.16.1
Move the macro to
calling.h, and inline it into the compat entry.
That certainly sounds possible, but makes the macro more complex: Inlining means that the offsets need to be reduced by -8. But we need the current offset for the call from interrupt_helper. So such a change might make the code less readable.
Thanks,
Dominik
Re: [RFC PATCH 3/4] x86/entry/64: move switch_to_thread_stack to interrupt helper function
Quoted 136 lines · Dominik Brodowski
Quoted 127 lines · Brian Gerst
Quoted 125 lines · Dominik Brodowski
We can also move the SWAPGS and the
switch_to_thread_stackto the interrupt helper function. As we do not want call depths of two, convertswitch_to_thread_stackto a macro. However, asentry_64_compat.Sexpectsswitch_to_thread_stackto be a function, provide a wrapper for that, which leads to some code duplication ifCONFIG_IA32_EMULATIONis enabled. Therefore, the size reduction differs slightly:With
CONFIG_IA32_EMULATIONenabled (-0.13k):
text data bss dec hex filename16897 0 0 16897 4201 entry_64.o-orig 16767 0 0 16767 417f entry_64.oWith
CONFIG_IA32_EMULATIONdisabled (-0.27k):
text data bss dec hex filename16897 0 0 16897 4201 entry_64.o-orig 16622 0 0 16622 40ee entry_64.oSigned-off-by Dominik Brodowski <[email protected]>--- arch/x86/entry/entry_64.S | 65 ++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 29 deletions(-)arch/x86/entry/entry_64.S +36
-29@@ -536,6 +536,31 @@ END(irq_entries_start) decl PER_CPU_VAR(irq_count) .endm +/* + * Switch to the thread stack. This is called with the IRET frame and + * orig_ax on the stack. (That is, RDI..R12 are not on the stack and + * space has not been allocated for them.) + */ +.macro DO_SWITCH_TO_THREAD_STACK + pushq %rdi + /* Need to switch before accessing the thread stack. */ + SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi + movq %rsp, %rdi + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp + UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI + + pushq 7*8(%rdi) /* regs->ss */ + pushq 6*8(%rdi) /* regs->rsp */ + pushq 5*8(%rdi) /* regs->eflags */ + pushq 4*8(%rdi) /* regs->cs */ + pushq 3*8(%rdi) /* regs->ip */ + pushq 2*8(%rdi) /* regs->orig_ax */ + pushq 8(%rdi) /* return address */ + UNWIND_HINT_FUNC + + movq (%rdi), %rdi +.endm + /* * Interrupt entry/exit. *@@ -543,10 +568,17 @@ END(irq_entries_start) * * Entry runs with interrupts off. */ +/* 8(%rsp): ~(interrupt number) */ ENTRY(interrupt_helper) UNWIND_HINT_FUNC cld + testb $3, CS-ORIG_RAX+8(%rsp) + jz 1f + SWAPGS + DO_SWITCH_TO_THREAD_STACK +1: + PUSH_AND_CLEAR_REGS save_ret=1 ENCODE_FRAME_POINTER 8@@ -579,12 +611,6 @@ END(interrupt_helper) .macro interrupt func cld - testb $3, CS-ORIG_RAX(%rsp) - jz 1f - SWAPGS - call switch_to_thread_stack -1: - call interrupt_helper call \func /* rdi points to pt_regs */@@ -853,33 +879,14 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt */ #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) -/* - * Switch to the thread stack. This is called with the IRET frame and - * orig_ax on the stack. (That is, RDI..R12 are not on the stack and - * space has not been allocated for them.) - */ +#if defined(CONFIG_IA32_EMULATION) +/* entry_64_compat.S::entry_INT80_compat expects this to be an ASM function */ ENTRY(switch_to_thread_stack) UNWIND_HINT_FUNC - - pushq %rdi - /* Need to switch before accessing the thread stack. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi - movq %rsp, %rdi - movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp - UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI - - pushq 7*8(%rdi) /* regs->ss */ - pushq 6*8(%rdi) /* regs->rsp */ - pushq 5*8(%rdi) /* regs->eflags */ - pushq 4*8(%rdi) /* regs->cs */ - pushq 3*8(%rdi) /* regs->ip */ - pushq 2*8(%rdi) /* regs->orig_ax */ - pushq 8(%rdi) /* return address */ - UNWIND_HINT_FUNC - - movq (%rdi), %rdi + DO_SWITCH_TO_THREAD_STACK ret END(switch_to_thread_stack) +#endifQuoted 7 lines
Quoted 5 lines
arch/x86/entry/entry_64.S +0
-1.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ENTRY(\sym) --2.16.1
Move the macro to
calling.h, and inline it into the compat entry.
Quoted 7 lines
That certainly sounds possible, but makes the macro more complex: Inlining means that the offsets need to be reduced by -8. But we need the current offset for the call from
interrupt_helper. So such a change might make the code less readable.Thanks,
Dominik
It would probably be better to just open code it then, without the return address handling.
--
Brian Gerst
RFC PATCH 4/4 x86/entry/64: remove interrupt macro
In reply to Dominik BrodowskiIt is now trivial to call the interrupt helper function and then the actual worker. Therefore, remove the interrupt macro.
--- arch/x86/entry/entry_64.S | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-)
arch/x86/entry/entry_64.S +4 -11
@@ -607,15 +607,6 @@ ENTRY(interrupt_helper) ret END(interrupt_helper) -/* 0(%rsp): ~(interrupt number) */ - .macro interrupt func - cld - - call interrupt_helper - - call \func /* rdi points to pt_regs */ - .endm - /* * The interrupt stubs push (~vector+0x80) onto the stack and * then jump to common_interrupt.
@@ -624,7 +615,8 @@ END(interrupt_helper) common_interrupt: ASM_CLAC addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ - interrupt do_IRQ + call interrupt_helper + call do_IRQ /* rdi points to pt_regs */ /* 0(%rsp): old RSP */ ret_from_intr: DISABLE_INTERRUPTS(CLBR_ANY)
@@ -816,7 +808,8 @@ ENTRY(\sym) ASM_CLAC pushq $~(\num) .Lcommon_\sym: - interrupt \do_sym + call interrupt_helper + call \do_sym /* rdi points to pt_regs */ jmp ret_from_intr END(\sym) .endm
Signature · 1 line
2.16.1