x86 optimizations
Subsystems (11)
Peter Zijlstra
maintainercover x86 optimizations
Hi,
These are the x86 optimization patches that I've been sitting on.
They shrink the kernel and generate better code.
text data bss dec hex filename
10702123 4530992 843776 16076891 f5505b defconfig-build/vmlinux.0 10682460 4530992 843776 16057228 f5038c defconfig-build/vmlinux.1 10665111 4530096 843776 16038983 f4bc47 defconfig-build/vmlinux.2 10665111 4530096 843776 16038983 f4bc47 defconfig-build/vmlinux.5
As compiled on an x86_64 using GCC-6.3, also boot tested the lot.
Peter Zijlstra
maintainerPATCH 1/5 x86: Implement __WARN using UD0
By using "UD0" for WARNs we remove the function call and its possible __FILE__ and __LINE__ immediate arguments from the instruction stream.
Total image size will not change much, what we win in the instruction stream we'll loose because of the __bug_table entries. Still, saves on I$ footprint and the total image size does go down a bit.
text data bss dec hex filename
10702123 4530992 843776 16076891 f5505b defconfig-build/vmlinux.0 10682460 4530992 843776 16057228 f5038c defconfig-build/vmlinux.1
(um didn't seem to use GENERIC_BUG at all, so remove it)
--- arch/um/Kconfig.common | 5 --- arch/x86/include/asm/bug.h | 68 ++++++++++++++++++++++++++++++----------- arch/x86/kernel/dumpstack.c | 3 - arch/x86/kernel/dumpstack_32.c | 12 ------- arch/x86/kernel/dumpstack_64.c | 10 ------ arch/x86/kernel/traps.c | 50 ++++++++++++++++++++++++++---- arch/x86/um/Makefile | 2 - arch/x86/um/bug.c | 21 ------------ 8 files changed, 96 insertions(+), 75 deletions(-)
arch/um/Kconfig.common +0 -5
@@ -50,11 +50,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config GENERIC_BUG - bool - default y - depends on BUG - config HZ int default 100
arch/x86/include/asm/bug.h +51 -17
87 lines not on this page · raw message
arch/x86/kernel/dumpstack.c +0 -3
@@ -289,9 +289,6 @@ void die(const char *str, struct pt_regsunsigned long flags =oops_begin ();int sig =SIGSEGV ; -if (!user_mode (regs)) -report_bug (regs->ip, regs); -if (__die (str, regs, err)) sig =0 ;oops_end (flags, regs, sig);
arch/x86/kernel/dumpstack_32.c +0 -12
@@ -162,15 +162,3 @@ void show_regs(struct pt_regs *regs) }pr_cont ("\n" ); } - -int is_valid_bugaddr (unsigned long ip) -{ -unsigned short ud2; - -if (ip <PAGE_OFFSET ) -return 0 ; -if (probe_kernel_address ((unsigned short *)ip, ud2)) -return 0 ; - -return ud2 ==0x0b0f ; -}
arch/x86/kernel/dumpstack_64.c +0 -10
@@ -178,13 +178,3 @@ void show_regs(struct pt_regs *regs) }pr_cont ("\n" ); } - -int is_valid_bugaddr (unsigned long ip) -{ -unsigned short ud2; - -if (__copy_from_user (&ud2, (const void __user *) ip,sizeof (ud2))) -return 0 ; - -return ud2 ==0x0b0f ; -}
arch/x86/kernel/traps.c +44 -6
62 lines not on this page · raw message
arch/x86/um/Makefile +1 -1
@@ -8,7 +8,7 @@ else BITS := 64 endif -obj-y = bug.o bugs_$(BITS).o delay.o fault.o ldt.o \ +obj-y = bugs_$(BITS).o delay.o fault.o ldt.o \ ptrace_$(BITS).o ptrace_user.o setjmp_$(BITS).o signal.o \ stub_$(BITS).o stub_segv.o \ sys_call_table_$(BITS).o sysrq_$(BITS).o tls_$(BITS).o \
arch/x86/um/bug.c deleted +0 -21
@@ -1,21 +0,0 @@ -/* -* Copyright (C) 2006 Jeff Dike ([email protected]) -* Licensed under the GPL V2 -*/ - -#include <linux/uaccess.h> - -/* -* Mostly copied from i386/x86_86 - eliminated the eip < PAGE_OFFSET because -* that's not relevant in skas mode. -*/ - -int is_valid_bugaddr (unsigned long eip) -{ -unsigned short ud2; - -if (probe_kernel_address ((unsigned short __user *)eip, ud2)) -return 0 ; - -return ud2 ==0x0b0f ; -}
Re: [PATCH 1/5] x86: Implement __WARN using UD0
Quoted 14 lines · Peter Zijlstra
arch/x86/include/asm/bug.h +5
-1@@ -1,36 +1,70 @@#ifndef _ASM_X86_BUG_H#define _ASM_X86_BUG_H +#include <linux/stringify.h> +#define HAVE_ARCH_BUG -#ifdef CONFIG_DEBUG_BUGVERBOSE +/* +* Since some emulators terminate on UD2, we cannot use it for WARN. +* Since various instruction decoders disagree on the length of UD1,
arch/x86/include/asm/bug.h line 20
Quoted 10 lines
arch/x86/include/asm/bug.h +10
-0+* we cannot use it either. So use UD0 for WARN. +* +* (binutils knows about "ud1" but {en,de}codes it as 2 bytes, whereas +* our kernel decoder thinks it takes a ModRM byte, which seems consistent +* with various things like the Intel SDM instruction encoding rules) +*/ + +#define ASM_UD0 ".byte 0x0f, 0xff" +#define ASM_UD1 ".byte 0x0f, 0xb9"/* + ModRM */ +#define ASM_UD2 ".byte 0x0f, 0x0b"
Thas ASM_UD1 macro isn't used anywhere.
Quoted 7 lines
arch/x86/kernel/traps.c +1
-0@@ -169,6 +169,41 @@ void ist_end_non_atomic(void)preempt_disable (); } +int is_valid_bugaddr (unsigned long addr)
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 182
Quoted 10 lines
arch/x86/kernel/traps.c +10
-0+{ +unsigned short ud; + +#ifdef CONFIG_X86_32 +if (addr <PAGE_OFFSET ) +return 0 ; +#else +if ((long )addr >0 ) +return 0 ; +#endif
I think comparing with TASK_SIZE would be more correct and it wouldn't need an ifdef.
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 187
Quoted 5 lines
arch/x86/kernel/traps.c +5
-0+if (probe_kernel_address ((unsigned short *)addr, ud)) +return 0 ; + +return ud ==0x0b0f || ud ==0xff0f ; +}
This code would be easier to grok if these were defines IMO.
Also, now that some of the BUG-specific functions are now also related to WARN, they should probably be renamed to describe their new purpose, like:
"report_bug" -> "report_bug_or_warning"
"fixup_bug" -> "fixup_bug_or_warning"
On a related note, if warn and bug are going to continue to use two separate ud instructions for the foreseeable future, report_bug() could be cleaned up a bit: e.g., for a ud0 instruction, it doesn't make sense to call find_bug().
Quoted 4 lines
arch/x86/kernel/traps.c +4
-0+ +static int fixup_bug (struct pt_regs *regs,int trapnr) +{ +if (trapnr !=X86_TRAP_UD )
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201
Quoted 10 lines
arch/x86/kernel/traps.c +10
-0+return 0 ; + +switch (report_bug (regs->ip, regs)) { +case BUG_TRAP_TYPE_NONE : +case BUG_TRAP_TYPE_BUG : +break ; + +case BUG_TRAP_TYPE_WARN : + regs->ip +=2 ; +return 1 ;
For self-documentation purposes, maybe use a define for the length of the ud0 instruction?
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 1/5] x86: Implement __WARN using UD0
Quoted 17 lines · Josh Poimboeuf
arch/x86/include/asm/bug.h line 20
Quoted 13 lines · Peter Zijlstra
arch/x86/include/asm/bug.h +13
-0+/* +* Since some emulators terminate on UD2, we cannot use it for WARN. +* Since various instruction decoders disagree on the length of UD1, +* we cannot use it either. So use UD0 for WARN. +* +* (binutils knows about "ud1" but {en,de}codes it as 2 bytes, whereas +* our kernel decoder thinks it takes a ModRM byte, which seems consistent +* with various things like the Intel SDM instruction encoding rules) +*/ + +#define ASM_UD0 ".byte 0x0f, 0xff" +#define ASM_UD1 ".byte 0x0f, 0xb9"/* + ModRM */ +#define ASM_UD2 ".byte 0x0f, 0x0b"Thas
ASM_UD1macro isn't used anywhere.
I have it there for completeness sake, and documentation purposes.
Quoted 6 lines
Quoted 6 lines
arch/x86/kernel/traps.c +0
-0@@ -169,6 +169,41 @@ void ist_end_non_atomic(void)preempt_disable (); }
Quoted 14 lines
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 182
Quoted 11 lines
arch/x86/kernel/traps.c +11
-0+int is_valid_bugaddr (unsigned long addr) +{ +unsigned short ud; + +#ifdef CONFIG_X86_32 +if (addr <PAGE_OFFSET ) +return 0 ; +#else +if ((long )addr >0 ) +return 0 ; +#endif I think comparing with
TASK_SIZEwould be more correct and it wouldn't need an ifdef.
Dunno about more correct (the TIF_ADDR32 case is irrelevant here), but yes, that would do away with the #ifdef.
Quoted 7 lines
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 187
Quoted 5 lines
arch/x86/kernel/traps.c +5
-0+if (probe_kernel_address ((unsigned short *)addr, ud)) +return 0 ; + +return ud ==0x0b0f || ud ==0xff0f ; +}This code would be easier to grok if these were defines IMO.
Would be nice if I could use the same defines; but I can't see how I can make that happen :/ But sure; I suppose I can add more defines.
Quoted 1 line
Also, now that some of the BUG-specific functions are now also related
They already were, its just that x86 only now will use the WARN part of it. I don't feel that should be part of this patch.
Quoted 10 lines
to WARN, they should probably be renamed to describe their new purpose, like:
"
report_bug" -> "report_bug_or_warning"
"fixup_bug" -> "fixup_bug_or_warning"On a related note, if warn and bug are going to continue to use two separate ud instructions for the foreseeable future,
report_bug()could be cleaned up a bit: e.g., for a ud0 instruction, it doesn't make sense to callfind_bug().
I'm sure you'll break $random arch if you go futz with that. Also, I think you mean UD2, since that's BUG. We actually need the bug_entry for WARNs (aka UD0).
Also, you're now optimizing the BUG() code; I don't think anybody cares about saving a few cycles there. It shouldn't happen in the first place.
Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/traps.c +8
-0+static int fixup_bug (struct pt_regs *regs,int trapnr) +{ +if (trapnr !=X86_TRAP_UD ) +return 0 ; + +switch (report_bug (regs->ip, regs)) { +case BUG_TRAP_TYPE_NONE : +case BUG_TRAP_TYPE_BUG :
Quoted 8 lines
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201
Quoted 5 lines
arch/x86/kernel/traps.c +5
-0+break ; + +case BUG_TRAP_TYPE_WARN : + regs->ip +=2 ; +return 1 ;For self-documentation purposes, maybe use a define for the length of the ud0 instruction?
Well, UD0 and UD2 really. LENGTH_UD0_OR_UD2 is a bit of a fail, name wise.
Re: [PATCH 1/5] x86: Implement __WARN using UD0
Quoted 2 lines · Peter Zijlstra
For self-documentation purposes, maybe use a define for the length of the ud0 instruction?
#define TWO 2
;-)
some things make sense as a define, others don't
(adding a comment, maybe)
Re: [PATCH 1/5] x86: Implement __WARN using UD0
In reply to Peter ZijlstraQuoted 11 lines · Peter Zijlstra
Quoted 10 lines
to WARN, they should probably be renamed to describe their new purpose, like:
"
report_bug" -> "report_bug_or_warning"
"fixup_bug" -> "fixup_bug_or_warning"On a related note, if warn and bug are going to continue to use two separate ud instructions for the foreseeable future,
report_bug()could be cleaned up a bit: e.g., for a ud0 instruction, it doesn't make sense to callfind_bug().
Quoted 6 lines
I'm sure you'll break $random arch if you go futz with that. Also, I think you mean UD2, since that's BUG. We actually need the
bug_entryfor WARNs (aka UD0).Also, you're now optimizing the
BUG()code; I don't think anybody cares about saving a few cycles there. It shouldn't happen in the first place.
My thinking was to make report_bug() a little less obtuse, but yeah, that would break other arches, so never mind...
Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/traps.c +8
-0+static int fixup_bug (struct pt_regs *regs,int trapnr) +{ +if (trapnr !=X86_TRAP_UD ) +return 0 ; + +switch (report_bug (regs->ip, regs)) { +case BUG_TRAP_TYPE_NONE : +case BUG_TRAP_TYPE_BUG :Quoted 8 lines
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201
Quoted 5 lines
arch/x86/kernel/traps.c +5
-0+break ; + +case BUG_TRAP_TYPE_WARN : + regs->ip +=2 ; +return 1 ;For self-documentation purposes, maybe use a define for the length of the ud0 instruction?
Quoted 2 lines
Well, UD0 and UD2 really.
LENGTH_UD0_OR_UD2is a bit of a fail, name wise.
Why UD2? Warnings are UD0-only, no? What about UD0_LEN? Or at least a comment would be helpful I think.
Signature · 1 line
Josh
Peter Zijlstra
maintainerRe: [PATCH 1/5] x86: Implement __WARN using UD0
Quoted 18 lines · Josh Poimboeuf
Quoted 17 lines · Peter Zijlstra
Quoted 16 lines
arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201
Quoted 13 lines
arch/x86/kernel/traps.c +13
-0+static int fixup_bug (struct pt_regs *regs,int trapnr) +{ +if (trapnr !=X86_TRAP_UD ) +return 0 ; + +switch (report_bug (regs->ip, regs)) { +case BUG_TRAP_TYPE_NONE : +case BUG_TRAP_TYPE_BUG : +break ; + +case BUG_TRAP_TYPE_WARN : + regs->ip +=2 ; +return 1 ;For self-documentation purposes, maybe use a define for the length of the ud0 instruction?
Quoted 5 lines
Quoted 2 lines
Well, UD0 and UD2 really.
LENGTH_UD0_OR_UD2is a bit of a fail, name wise.Why UD2? Warnings are UD0-only, no? What about
UD0_LEN? Or at least a comment would be helpful I think.
Oh, right, we only resume on WARN,.. Doh.
Peter Zijlstra
maintainerRe: [PATCH 1/5] x86: Implement __WARN using UD0
In reply to Josh PoimboeufA little like so then.
---
Subject: x86: Implement __WARN using UD0
From: Peter Zijlstra <[email protected]>
Date: Thu Feb 2 14:43:51 CET 2017
By using "UD0" for WARNs we remove the function call and its possible __FILE__ and __LINE__ immediate arguments from the instruction stream.
Total image size will not change much, what we win in the instruction stream we'll loose because of the __bug_table entries. Still, saves on I$ footprint and the total image size does go down a bit.
text data bss dec hex filename
10702123 4530992 843776 16076891 f5505b defconfig-build/vmlinux.0 10682460 4530992 843776 16057228 f5038c defconfig-build/vmlinux.1
(um didn't seem to use GENERIC_BUG at all, so remove it)
--- arch/um/Kconfig.common | 5 -- arch/x86/include/asm/bug.h | 73 +++++++++++++++++++++++++++++++---------- arch/x86/kernel/dumpstack.c | 3 - arch/x86/kernel/dumpstack_32.c | 12 ------ arch/x86/kernel/dumpstack_64.c | 10 ----- arch/x86/kernel/traps.c | 46 ++++++++++++++++++++++--- arch/x86/um/Makefile | 2 - arch/x86/um/bug.c | 21 ----------- 8 files changed, 97 insertions(+), 75 deletions(-)
arch/um/Kconfig.common +0 -5
@@ -50,11 +50,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config GENERIC_BUG - bool - default y - depends on BUG - config HZ int default 100
arch/x86/include/asm/bug.h +56 -17
92 lines not on this page · raw message
arch/x86/kernel/dumpstack.c +0 -3
@@ -289,9 +289,6 @@ void die(const char *str, struct pt_regsunsigned long flags =oops_begin ();int sig =SIGSEGV ; -if (!user_mode (regs)) -report_bug (regs->ip, regs); -if (__die (str, regs, err)) sig =0 ;oops_end (flags, regs, sig);
arch/x86/kernel/dumpstack_32.c +0 -12
@@ -162,15 +162,3 @@ void show_regs(struct pt_regs *regs) }pr_cont ("\n" ); } - -int is_valid_bugaddr (unsigned long ip) -{ -unsigned short ud2; - -if (ip <PAGE_OFFSET ) -return 0 ; -if (probe_kernel_address ((unsigned short *)ip, ud2)) -return 0 ; - -return ud2 ==0x0b0f ; -}
arch/x86/kernel/dumpstack_64.c +0 -10
@@ -178,13 +178,3 @@ void show_regs(struct pt_regs *regs) }pr_cont ("\n" ); } - -int is_valid_bugaddr (unsigned long ip) -{ -unsigned short ud2; - -if (__copy_from_user (&ud2, (const void __user *) ip,sizeof (ud2))) -return 0 ; - -return ud2 ==0x0b0f ; -}
arch/x86/kernel/traps.c +40 -6
58 lines not on this page · raw message
arch/x86/um/Makefile +1 -1
@@ -8,7 +8,7 @@ else BITS := 64 endif -obj-y = bug.o bugs_$(BITS).o delay.o fault.o ldt.o \ +obj-y = bugs_$(BITS).o delay.o fault.o ldt.o \ ptrace_$(BITS).o ptrace_user.o setjmp_$(BITS).o signal.o \ stub_$(BITS).o stub_segv.o \ sys_call_table_$(BITS).o sysrq_$(BITS).o tls_$(BITS).o \
arch/x86/um/bug.c deleted +0 -21
@@ -1,21 +0,0 @@ -/* -* Copyright (C) 2006 Jeff Dike ([email protected]) -* Licensed under the GPL V2 -*/ - -#include <linux/uaccess.h> - -/* -* Mostly copied from i386/x86_86 - eliminated the eip < PAGE_OFFSET because -* that's not relevant in skas mode. -*/ - -int is_valid_bugaddr (unsigned long eip) -{ -unsigned short ud2; - -if (probe_kernel_address ((unsigned short __user *)eip, ud2)) -return 0 ; - -return ud2 ==0x0b0f ; -}
Josh Poimboeuf
Reviewed-byRe: [PATCH 1/5] x86: Implement __WARN using UD0
Quoted 21 lines · Peter Zijlstra
A little like so then.
---
Subject: x86: Implement__WARNusing UD0
From: Peter Zijlstra <[email protected]>
Date: Thu Feb 2 14:43:51 CET 2017By using "UD0" for WARNs we remove the function call and its possible
__FILE__and__LINE__immediate arguments from the instruction stream.Total image size will not change much, what we win in the instruction stream we'll loose because of the
__bug_tableentries. Still, saves on I$ footprint and the total image size does go down a bit.text data bss dec hex filename
10702123 4530992 843776 16076891 f5505b defconfig-build/vmlinux.0 10682460 4530992 843776 16057228 f5038c defconfig-build/vmlinux.1(um didn't seem to use
GENERIC_BUGat all, so remove it)
Quoted 7 lines
Cc Linus Torvalds <[email protected]>Cc Richard Weinberger <[email protected]>Cc Arjan van de Ven <[email protected]>Cc Borislav Petkov <[email protected]>Cc Josh Poimboeuf <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Looks good, thanks!
Signature · 1 line
Josh
Peter Zijlstra
maintainerPATCH 2/5 bug: Add _ONCE logic to report_bug()
In reply to Peter ZijlstraJosh suggested moving the _ONCE logic inside the trap handler, using a bit in the bug_entry::flags field, avoiding the need for the extra variable.
Sadly this only works for WARN_ON_ONCE(), since the others have printk() statements prior to triggering the trap.
Still, this saves a fair amount of text and data:
text data bss dec hex filename 10682460 4530992 843776 16057228 f5038c defconfig-build/vmlinux.1 10665111 4530096 843776 16038983 f4bc47 defconfig-build/vmlinux.2
--- arch/arm64/include/asm/bug.h | 2 +- arch/parisc/include/asm/bug.h | 8 ++++---- arch/powerpc/include/asm/bug.h | 4 ++-- arch/s390/include/asm/bug.h | 4 ++-- arch/sh/include/asm/bug.h | 4 ++-- arch/x86/include/asm/bug.h | 2 +- include/asm-generic/bug.h | 18 +++++++++++++++++- include/asm-generic/vmlinux.lds.h | 3 +-- include/linux/bug.h | 2 +- lib/bug.c | 28 ++++++++++++++++++++-------- 10 files changed, 51 insertions(+), 24 deletions(-)
arch/arm64/include/asm/bug.h +1 -1
@@ -55,7 +55,7 @@ _BUGVERBOSE_LOCATION(__FILE__, __LINE__)unreachable (); \ } while (0) -#define __WARN_TAINT (taint) _BUG_FLAGS(BUGFLAG_TAINT(taint)) +#define __WARN_FLAGS (flags) _BUG_FLAGS(BUGFLAG_WARNING|(flags))#endif /* ! CONFIG_GENERIC_BUG */
arch/parisc/include/asm/bug.h +4 -4
@@ -46,7 +46,7 @@#endif #ifdef CONFIG_DEBUG_BUGVERBOSE -#define __WARN_TAINT (taint) \ +#define __WARN_FLAGS (flags) \ do { \ asm volatile("\n" \ "1:\t" PARISC_BUG_BREAK_ASM "\n" \
@@ -56,11 +56,11 @@ "\t.org 2b+%c3\n" \ "\t.popsection" \ : : "i" (__FILE__), "i" (__LINE__), \ -"i" (BUGFLAG_TAINT (taint)), \ +"i" (BUGFLAG_WARNING |(flags)), \ "i" (sizeof(struct bug_entry)) ); \ } while(0) #else -#define __WARN_TAINT (taint) \ +#define __WARN_FLAGS (flags) \ do { \ asm volatile("\n" \ "1:\t" PARISC_BUG_BREAK_ASM "\n" \
@@ -69,7 +69,7 @@ "\t.short %c0\n" \ "\t.org 2b+%c1\n" \ "\t.popsection" \ - : : "i" (BUGFLAG_TAINT(taint)), \ + : : "i" (BUGFLAG_WARNING|(flags)), \ "i" (sizeof(struct bug_entry)) ); \ } while(0)#endif
arch/powerpc/include/asm/bug.h +2 -2
@@ -85,12 +85,12 @@ } \ } while (0) -#define __WARN_TAINT (taint) do { \ +#define __WARN_FLAGS (flags) do { \ __asm__ __volatile__( \ "1: twi 31,0,0\n" \ _EMIT_BUG_ENTRY \ : : "i" (__FILE__), "i" (__LINE__), \ - "i" (BUGFLAG_TAINT(taint)), \ + "i" (BUGFLAG_WARNING|(flags)), \ "i" (sizeof(struct bug_entry))); \ } while (0)
arch/s390/include/asm/bug.h +2 -2
@@ -46,8 +46,8 @@unreachable (); \ } while (0) -#define __WARN_TAINT (taint) do { \ - __EMIT_BUG(BUGFLAG_TAINT(taint)); \ +#define __WARN_FLAGS (flags) do { \ + __EMIT_BUG(BUGFLAG_WARNING|(flags)); \ } while (0)#define WARN_ON (x) ({ \
arch/sh/include/asm/bug.h +2 -2
@@ -50,7 +50,7 @@ do { \ "i" (sizeof(struct bug_entry))); \ } while (0) -#define __WARN_TAINT (taint) \ +#define __WARN_FLAGS (flags) \ do { \ __asm__ __volatile__ ( \ "1:\t.short %O0\n" \
@@ -59,7 +59,7 @@ do { \ : "n" (TRAPA_BUG_OPCODE), \"i" (__FILE__), \"i" (__LINE__), \ -"i" (BUGFLAG_TAINT (taint)), \ +"i" (BUGFLAG_WARNING |(flags)), \ "i" (sizeof(struct bug_entry))); \ }while (0 )
arch/x86/include/asm/bug.h +1 -1
@@ -64,7 +64,7 @@ do { \unreachable (); \ } while (0) -#define __WARN_TAINT (taint) _BUG_FLAGS(ASM_UD0, BUGFLAG_TAINT(taint)) +#define __WARN_FLAGS (flags) _BUG_FLAGS(ASM_UD0, BUGFLAG_WARNING|(flags))#include <asm-generic/bug.h>
include/asm-generic/bug.h +17 -1
@@ -5,6 +5,8 @@#ifdef CONFIG_GENERIC_BUG #define BUGFLAG_WARNING (1 << 0) +#define BUGFLAG_ONCE (1 << 1) +#define BUGFLAG_DONE (1 << 2)#define BUGFLAG_TAINT (taint) (BUGFLAG_WARNING | ((taint) << 8))#define BUG_GET_TAINT (bug) ((bug)->flags >> 8)#endif
@@ -55,6 +57,18 @@ struct bug_entry {#define BUG_ON (condition) do { if (unlikely(condition)) BUG(); } while (0)#endif +#ifdef __WARN_FLAGS +#define __WARN_TAINT (taint) __WARN_FLAGS(BUGFLAG_TAINT(taint)) +#define __WARN_ONCE_TAINT (taint) __WARN_FLAGS(BUGFLAG_ONCE|BUGFLAG_TAINT(taint)) + +#define WARN_ON_ONCE (condition) ({ \ + int __ret_warn_on = !!(condition); \ + if (unlikely(__ret_warn_on)) \ + __WARN_ONCE_TAINT(TAINT_WARN); \ + unlikely(__ret_warn_on); \ +}) +#endif +/* * WARN(), WARN_ON(), WARN_ON_ONCE, and so on can be used to report * significant issues that need prompt attention if they should ever
@@ -97,7 +111,7 @@ void __warn(const char *file, int line,#endif #ifndef WARN -#define WARN(condition, format...) ({ \ +#define WARN(condition, format...) ({ \ int __ret_warn_on = !!(condition); \ if (unlikely(__ret_warn_on)) \ __WARN_printf(format); \
@@ -112,6 +126,7 @@ void __warn(const char *file, int line,unlikely (__ret_warn_on); \ }) +#ifndef WARN_ON_ONCE #define WARN_ON_ONCE (condition) ({ \ static bool __section(.data.unlikely) __warned; \ int __ret_warn_once = !!(condition); \
@@ -122,6 +137,7 @@ void __warn(const char *file, int line, } \unlikely (__ret_warn_once); \ }) +#endif #define WARN_ONCE(condition, format...) ({ \ static bool __section(.data.unlikely) __warned; \
include/asm-generic/vmlinux.lds.h +1 -2
@@ -286,8 +286,6 @@ *(.rodata1) \ } \ \ - BUG_TABLE \ - \ /* PCI quirks */ \ .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \
@@ -855,6 +853,7 @@READ_MOSTLY_DATA (cacheline) \ DATA_DATA \CONSTRUCTORS \ + BUG_TABLE \ }#define INIT_TEXT_SECTION (inittext_align) \
include/linux/bug.h +1 -1
@@ -105,7 +105,7 @@ static inline int is_warning_bug(const sreturn bug->flags &BUGFLAG_WARNING ; } -const struct bug_entry *find_bug (unsigned long bugaddr); +struct bug_entry *find_bug (unsigned long bugaddr);enum bug_trap_type report_bug (unsigned long bug_addr,struct pt_regs *regs);
lib/bug.c +20 -8
@@ -47,7 +47,7 @@#include <linux/sched.h> #include <linux/rculist.h> -extern const struct bug_entry __start___bug_table[], __stop___bug_table[]; +extern struct bug_entry __start___bug_table[], __stop___bug_table[];static inline unsigned long bug_addr (const struct bug_entry *bug) {
@@ -62,10 +62,10 @@ static inline unsigned long bug_addr(con/* Updates are protected by module mutex */ static LIST_HEAD (module_bug_list); -static const struct bug_entry *module_find_bug (unsigned long bugaddr) +static struct bug_entry *module_find_bug (unsigned long bugaddr) {struct module *mod; -const struct bug_entry *bug =NULL ; +struct bug_entry *bug =NULL ;rcu_read_lock_sched ();list_for_each_entry_rcu (mod, &module_bug_list, bug_list) {
@@ -122,15 +122,15 @@ void module_bug_cleanup(struct module *m#else -static inline const struct bug_entry *module_find_bug (unsigned long bugaddr) +static inline struct bug_entry *module_find_bug (unsigned long bugaddr) {return NULL ; }#endif -const struct bug_entry *find_bug (unsigned long bugaddr) +struct bug_entry *find_bug (unsigned long bugaddr) { -const struct bug_entry *bug; +struct bug_entry *bug;for (bug = __start___bug_table; bug < __stop___bug_table; ++bug)if (bugaddr ==bug_addr (bug))
@@ -141,9 +141,9 @@ const struct bug_entry *find_bug(unsigneenum bug_trap_type report_bug (unsigned long bugaddr,struct pt_regs *regs) { -const struct bug_entry *bug; +struct bug_entry *bug;const char *file; -unsigned line, warning; +unsigned line, warning, once, done;if (!is_valid_bugaddr (bugaddr))return BUG_TRAP_TYPE_NONE ;
@@ -164,6 +164,18 @@ enum bug_trap_type report_bug(unsigned l line = bug->line;#endif warning = (bug->flags &BUGFLAG_WARNING ) !=0 ; + once = (bug->flags &BUGFLAG_ONCE ) !=0 ; + done = (bug->flags &BUGFLAG_DONE ) !=0 ; + +if (warning && once) { +if (done) +return BUG_TRAP_TYPE_WARN ; + +/* +* Since this is the only store, concurrency is not an issue. +*/ + bug->flags |=BUGFLAG_DONE ; + } }if (warning) {
Peter Zijlstra
maintainerPATCH 3/5 atomic: Introduce atomic_try_cmpxchg()
In reply to Peter ZijlstraAdd a new cmpxchg interface:
bool try_cmpxchg(u{8,16,32,64} *ptr, u{8,16,32,64} *val, u{8,16,32,64} new);Where the boolean returns the result of the compare; and thus if the exchange happened; and in case of failure, the new value of *ptr is returned in *val.
This allows simplification/improvement of loops like:
for (;;) {
new = val $op $imm;
old = cmpxchg(ptr, val, new);
if (old == val)
break;
val = old;
}into:
do {
new = val $op $imm;
} while (!try_cmpxchg(ptr, &val, new));while also generating better code (GCC6 and onwards).
--- arch/x86/include/asm/atomic.h | 6 +++ arch/x86/include/asm/atomic64_64.h | 6 +++ arch/x86/include/asm/cmpxchg.h | 69 +++++++++++++++++++++++++++++++++++++ include/linux/atomic.h | 42 ++++++++++++++++++++++ 4 files changed, 123 insertions(+)
arch/x86/include/asm/atomic.h +6 -0
@@ -186,6 +186,12 @@ static __always_inline int atomic_cmpxchreturn cmpxchg (&v->counter, old, new); } +#define atomic_try_cmpxchg atomic_try_cmpxchg +static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ +return try_cmpxchg (&v->counter, old, new); +} +static inline int atomic_xchg (atomic_t *v,int new) {return xchg (&v->counter, new);
arch/x86/include/asm/atomic64_64.h +6 -0
@@ -176,6 +176,12 @@ static inline long atomic64_cmpxchg(atomreturn cmpxchg (&v->counter, old, new); } +#define atomic64_try_cmpxchg atomic64_try_cmpxchg +static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, long *old, long new) +{ +return try_cmpxchg (&v->counter, old, new); +} +static inline long atomic64_xchg (atomic64_t *v,long new) {return xchg (&v->counter, new);
arch/x86/include/asm/cmpxchg.h +69 -0
75 lines not on this page · raw message
include/linux/atomic.h +42 -0
54 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH 4/5 refcount: Use atomic_try_cmpxchg()
In reply to Peter ZijlstraGenerates better code (GCC-6.2.1):
text data bss dec hex filename 1576 7 0 1583 62f defconfig-build/lib/refcount.o.pre 1488 7 0 1495 5d7 defconfig-build/lib/refcount.o.post
--- lib/refcount.c | 47 +++++++++++++++-------------------------------- 1 file changed, 15 insertions(+), 32 deletions(-)
lib/refcount.c +15 -32
@@ -57,9 +57,9 @@*/ bool refcount_add_not_zero (unsigned int i,refcount_t *r) { -unsigned int old, new, val =atomic_read (&r->refs); +unsigned int new, val =atomic_read (&r->refs); -for (;;) { + do { if (!val) return false;
@@ -69,12 +69,8 @@ bool refcount_add_not_zero(unsigned int new = val + i;if (new < val) new =UINT_MAX ; - old =atomic_cmpxchg_relaxed (&r->refs, val, new); -if (old == val) -break ; - val = old; - } + }while (!atomic_try_cmpxchg_relaxed (&r->refs, &val, new));WARN_ONCE (new ==UINT_MAX ,"refcount_t: saturated; leaking memory.\n" );
@@ -118,9 +114,9 @@ EXPORT_SYMBOL_GPL(refcount_add);*/ bool refcount_inc_not_zero (refcount_t *r) { -unsigned int old, new, val =atomic_read (&r->refs); +unsigned int new, val =atomic_read (&r->refs); -for (;;) { + do { new = val + 1; if (!val)
@@ -129,12 +125,7 @@ bool refcount_inc_not_zero(refcount_t *rif (unlikely (!new))return true; - old =atomic_cmpxchg_relaxed (&r->refs, val, new); -if (old == val) -break ; - - val = old; - } + }while (!atomic_try_cmpxchg_relaxed (&r->refs, &val, new));WARN_ONCE (new ==UINT_MAX ,"refcount_t: saturated; leaking memory.\n" );
@@ -182,9 +173,9 @@ EXPORT_SYMBOL_GPL(refcount_inc);*/ bool refcount_sub_and_test (unsigned int i,refcount_t *r) { -unsigned int old, new, val =atomic_read (&r->refs); +unsigned int new, val =atomic_read (&r->refs); -for (;;) { + do { if (unlikely(val == UINT_MAX)) return false;
@@ -194,12 +185,7 @@ bool refcount_sub_and_test(unsigned intreturn false; } - old =atomic_cmpxchg_release (&r->refs, val, new); -if (old == val) -break ; - - val = old; - } + }while (!atomic_try_cmpxchg_release (&r->refs, &val, new));return !new; }
@@ -258,7 +244,9 @@ EXPORT_SYMBOL_GPL(refcount_dec);*/ bool refcount_dec_if_one (refcount_t *r) { -return atomic_cmpxchg_release (&r->refs,1 ,0 ) ==1 ; +int val =1 ; + +return atomic_try_cmpxchg_release (&r->refs, &val,0 ); }EXPORT_SYMBOL_GPL (refcount_dec_if_one);
@@ -275,9 +263,9 @@ EXPORT_SYMBOL_GPL(refcount_dec_if_one);*/ bool refcount_dec_not_one (refcount_t *r) { -unsigned int old, new, val =atomic_read (&r->refs); +unsigned int new, val =atomic_read (&r->refs); -for (;;) { + do { if (unlikely(val == UINT_MAX)) return true;
@@ -290,12 +278,7 @@ bool refcount_dec_not_one(refcount_t *r)return true; } - old =atomic_cmpxchg_release (&r->refs, val, new); -if (old == val) -break ; - - val = old; - } + }while (!atomic_try_cmpxchg_release (&r->refs, &val, new));return true; }
Peter Zijlstra
maintainerPATCH 5/5 x86,atomic: Use atomic_try_cmpxchg
In reply to Peter Zijlstratext data bss dec hex filename
10665111 4530096 843776 16038983 f4bc47 defconfig-build/vmlinux.3 10655703 4530096 843776 16029575 f49787 defconfig-build/vmlinux.4
--- arch/x86/include/asm/atomic.h | 27 ++++++++---------------- arch/x86/include/asm/atomic64_64.h | 40 ++++++++++++------------------------- 2 files changed, 22 insertions(+), 45 deletions(-)
arch/x86/include/asm/atomic.h +9 -18
@@ -207,16 +207,12 @@ static inline void atomic_##op(int i, at }#define ATOMIC_FETCH_OP (op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ +static inline int atomic_fetch_##op(int i, atomic_t *v) \ { \ - int old, val = atomic_read(v); \ - for (;;) { \ - old = atomic_cmpxchg(v, val, val c_op i); \ - if (old == val) \ - break; \ - val = old; \ - } \ - return old; \ + int val = atomic_read(v); \ + do { \ + } while (!atomic_try_cmpxchg(v, &val, val c_op i)); \ + return val; \ }#define ATOMIC_OPS (op, c_op) \
@@ -242,16 +238,11 @@ ATOMIC_OPS(xor, ^)*/ static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u) { -int c, old; - c =atomic_read (v); -for (;;) { -if (unlikely (c == (u))) +int c =atomic_read (v); +do { +if (unlikely (c == u))break ; - old =atomic_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } + }while (!atomic_try_cmpxchg (v, &c, c + a));return c; }
arch/x86/include/asm/atomic64_64.h +13 -27
@@ -198,17 +198,12 @@ static inline long atomic64_xchg(atomic6*/ static inline bool atomic64_add_unless (atomic64_t *v,long a,long u) { -long c, old; - c =atomic64_read (v); -for (;;) { -if (unlikely (c == (u))) -break ; - old =atomic64_cmpxchg ((v), c, c + (a)); -if (likely (old == c)) -break ; - c = old; - } -return c != (u); +long c =atomic64_read (v); +do { +if (unlikely (c == u)) +return false; + }while (!atomic64_try_cmpxchg (v, &c, c + a)); +return true; }#define atomic64_inc_not_zero (v) atomic64_add_unless((v), 1, 0)
@@ -222,17 +217,12 @@ static inline bool atomic64_add_unless(a*/ static inline long atomic64_dec_if_positive (atomic64_t *v) { -long c, old, dec; - c =atomic64_read (v); -for (;;) { +long dec, c =atomic64_read (v); +do { dec = c -1 ;if (unlikely (dec <0 ))break ; - old =atomic64_cmpxchg ((v), c, dec); -if (likely (old == c)) -break ; - c = old; - } + }while (!atomic64_try_cmpxchg (v, &c, dec));return dec; }
@@ -248,14 +238,10 @@ static inline void atomic64_##op(long i,#define ATOMIC64_FETCH_OP (op, c_op) \ static inline long atomic64_fetch_##op(long i, atomic64_t *v) \ { \ - long old, val = atomic64_read(v); \ - for (;;) { \ - old = atomic64_cmpxchg(v, val, val c_op i); \ - if (old == val) \ - break; \ - val = old; \ - } \ - return old; \ + long val = atomic64_read(v); \ + do { \ + } while (!atomic64_try_cmpxchg(v, &val, val c_op i)); \ + return val; \ }#define ATOMIC64_OPS (op, c_op) \