x86/mm/fault: Inline page fault paths to reduce kernel text size
RFC PATCH 1/1 x86/mm/fault: Inline page fault paths to reduce kernel text size
From: Pekka Enberg <[email protected]>
The commit 92181f190b649f7ef2b79cbf5c00f26ccc66da2a ("x86: optimise x86's do_page_fault (C entry point for the page fault path)") from 2009 shows significant stack savings when infrequent page fault handling paths are moved out of line with the "noinline" annotation, with some increase in kernel text size.
However, a decade of GCC improvements (and changes in the code) has eliminated such wins in stack usage:
With "noinline":
0000000000000b30 <do_user_addr_fault>:
b30: 41 57 push %r15
b32: 41 56 push %r14
b34: 41 55 push %r13
b36: 49 89 d5 mov %rdx,%r13
b39: 41 54 push %r12
b3b: 49 89 fc mov %rdi,%r12
b3e: 55 push %rbp
b3f: 48 89 f5 mov %rsi,%rbp
b42: 53 push %rbx
b43: 65 48 8b 04 25 00 00 mov %gs:0x0,%rax
b4a: 00 00
b4c: 48 83 ec 18 sub $0x18,%rspWith "inline":
00000000000008a0 <do_user_addr_fault>:
8a0: 41 57 push %r15
8a2: 41 56 push %r14
8a4: 41 55 push %r13
8a6: 49 89 d5 mov %rdx,%r13
8a9: 41 54 push %r12
8ab: 49 89 fc mov %rdi,%r12
8ae: 55 push %rbp
8af: 48 89 f5 mov %rsi,%rbp
8b2: 53 push %rbx
8b3: 65 48 8b 04 25 00 00 mov %gs:0x0,%rax
8ba: 00 00
8bc: 48 83 ec 18 sub $0x18,%rspSo all we're left with is the increase in kernel text:
add/remove: 1/5 grow/shrink: 2/1 up/down: 1049/-1144 (-95)
Function old new delta
spurious_kernel_fault.part - 559 +559
do_user_addr_fault 988 1446 +458
do_kern_addr_fault 103 135 +32 bad_area_nosemaphore 13 - -13 bad_area 66 - -66 pgtable_bad 107 - -107 mm_fault_error 199 - -199 bad_area_access_error 226 - -226 spurious_kernel_fault 553 20 -533
Total: Before=7359, After=7264, chg -1.29%
Therefore, let's inline the page fault handling paths to reduce kernel text size.
--- arch/x86/mm/fault.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
arch/x86/mm/fault.c +7 -7
@@ -533,7 +533,7 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code, unsigned long addump_pagetable (address); } -static noinline void +static inline void pgtable_bad (struct pt_regs *regs,unsigned long error_code,unsigned long address) {
@@ -577,7 +577,7 @@ static void set_signal_archinfo(unsigned long address, tsk->thread.cr2 = address; } -static noinline void +static inline void no_context (struct pt_regs *regs,unsigned long error_code,unsigned long address,int signal,int si_code) {
@@ -788,7 +788,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,no_context (regs, error_code, address,SIGSEGV , si_code); } -static noinline void +static inline void bad_area_nosemaphore (struct pt_regs *regs,unsigned long error_code,unsigned long address) {
@@ -809,7 +809,7 @@ __bad_area(struct pt_regs *regs, unsigned long error_code,__bad_area_nosemaphore (regs, error_code, address, pkey, si_code); } -static noinline void +static inline void bad_area (struct pt_regs *regs,unsigned long error_code,unsigned long address) {__bad_area (regs, error_code, address,0 ,SEGV_MAPERR );
@@ -832,7 +832,7 @@ static inline bool bad_area_access_from_pkeys(unsigned long error_code,return false; } -static noinline void +static inline void bad_area_access_error (struct pt_regs *regs,unsigned long error_code,unsigned long address,struct vm_area_struct *vma) {
@@ -905,7 +905,7 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,force_sig_fault (SIGBUS ,BUS_ADRERR , (void __user *)address); } -static noinline void +static inline void mm_fault_error (struct pt_regs *regs,unsigned long error_code,unsigned long address,vm_fault_t fault) {
@@ -971,7 +971,7 @@ static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)* See Intel Developer's Manual Vol 3 Section 4.10.4.3, bullet 3 * (Optional Invalidation). */ -static noinline int +static inline int spurious_kernel_fault (unsigned long error_code,unsigned long address) {pgd_t *pgd;
Signature · 1 line
2.26.2