Skip to contentSkip to conversation

x86 optimizations

PATCH In review Peter Zijlstra 3 participants 13 messages Mar 17, 2017 – Mar 22, 2017

Subsystems (11)
x86 PowerPC Library code arm64 +7 more subsystems
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover x86 optimizations

Hi,

These are the x86 optimization patches that I've been sitting on.

They shrink the kernel and generate better code.

text data bss dec hex filename

10702123        4530992  843776 16076891         f5505b defconfig-build/vmlinux.0
10682460        4530992  843776 16057228         f5038c defconfig-build/vmlinux.1
10665111        4530096  843776 16038983         f4bc47 defconfig-build/vmlinux.2
10665111        4530096  843776 16038983         f4bc47 defconfig-build/vmlinux.5

As compiled on an x86_64 using GCC-6.3, also boot tested the lot.

author maintainer
#1 lore raw

PATCH 1/5 x86: Implement __WARN using UD0

By using "UD0" for WARNs we remove the function call and its possible __FILE__ and __LINE__ immediate arguments from the instruction stream.

Total image size will not change much, what we win in the instruction stream we'll loose because of the __bug_table entries. Still, saves on I$ footprint and the total image size does go down a bit.

text data bss dec hex filename

10702123        4530992  843776 16076891         f5505b defconfig-build/vmlinux.0
10682460        4530992  843776 16057228         f5038c defconfig-build/vmlinux.1

(um didn't seem to use GENERIC_BUG at all, so remove it)

Cc Richard Weinberger <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Arjan van de Ven <[email protected]>
Cc Josh Poimboeuf <[email protected]>
---
 arch/um/Kconfig.common         |    5 ---
 arch/x86/include/asm/bug.h     |   68 ++++++++++++++++++++++++++++++-----------
 arch/x86/kernel/dumpstack.c    |    3 -
 arch/x86/kernel/dumpstack_32.c |   12 -------
 arch/x86/kernel/dumpstack_64.c |   10 ------
 arch/x86/kernel/traps.c        |   50 ++++++++++++++++++++++++++----
 arch/x86/um/Makefile           |    2 -
 arch/x86/um/bug.c              |   21 ------------
 8 files changed, 96 insertions(+), 75 deletions(-)
arch/um/Kconfig.common +0 -5
@@ -50,11 +50,6 @@ config GENERIC_CALIBRATE_DELAY
 	bool
 	default y
 
-config GENERIC_BUG
-	bool
-	default y
-	depends on BUG
-
 config HZ
 	int
 	default 100
arch/x86/include/asm/bug.h +51 -17

87 lines not on this page · raw message

arch/x86/kernel/dumpstack.c +0 -3
@@ -289,9 +289,6 @@ void die(const char *str, struct pt_regs
 	unsigned long flags = oops_begin();
 	int sig = SIGSEGV;
 
-	if (!user_mode(regs))
-		report_bug(regs->ip, regs);
-
 	if (__die(str, regs, err))
 		sig = 0;
 	oops_end(flags, regs, sig);
arch/x86/kernel/dumpstack_32.c +0 -12
@@ -162,15 +162,3 @@ void show_regs(struct pt_regs *regs)
 	}
 	pr_cont("\n");
 }
-
-int is_valid_bugaddr(unsigned long ip)
-{
-	unsigned short ud2;
-
-	if (ip < PAGE_OFFSET)
-		return 0;
-	if (probe_kernel_address((unsigned short *)ip, ud2))
-		return 0;
-
-	return ud2 == 0x0b0f;
-}
arch/x86/kernel/dumpstack_64.c +0 -10
@@ -178,13 +178,3 @@ void show_regs(struct pt_regs *regs)
 	}
 	pr_cont("\n");
 }
-
-int is_valid_bugaddr(unsigned long ip)
-{
-	unsigned short ud2;
-
-	if (__copy_from_user(&ud2, (const void __user *) ip, sizeof(ud2)))
-		return 0;
-
-	return ud2 == 0x0b0f;
-}
arch/x86/kernel/traps.c +44 -6

62 lines not on this page · raw message

arch/x86/um/Makefile +1 -1
@@ -8,7 +8,7 @@ else
 	BITS := 64
 endif
 
-obj-y = bug.o bugs_$(BITS).o delay.o fault.o ldt.o \
+obj-y = bugs_$(BITS).o delay.o fault.o ldt.o \
 	ptrace_$(BITS).o ptrace_user.o setjmp_$(BITS).o signal.o \
 	stub_$(BITS).o stub_segv.o \
 	sys_call_table_$(BITS).o sysrq_$(BITS).o tls_$(BITS).o \
arch/x86/um/bug.c deleted +0 -21
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2006 Jeff Dike ([email protected])
- * Licensed under the GPL V2
- */
-
-#include <linux/uaccess.h>
-
-/*
- * Mostly copied from i386/x86_86 - eliminated the eip < PAGE_OFFSET because
- * that's not relevant in skas mode.
- */
-
-int is_valid_bugaddr(unsigned long eip)
-{
-	unsigned short ud2;
-
-	if (probe_kernel_address((unsigned short __user *)eip, ud2))
-		return 0;
-
-	return ud2 == 0x0b0f;
-}
#2 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

Quoted 14 lines · Peter Zijlstra
arch/x86/include/asm/bug.h +5 -1
@@ -1,36 +1,70 @@
 #ifndef _ASM_X86_BUG_H
 #define _ASM_X86_BUG_H
 
+#include <linux/stringify.h>
+
 #define HAVE_ARCH_BUG
 
-#ifdef CONFIG_DEBUG_BUGVERBOSE
+/*
+ * Since some emulators terminate on UD2, we cannot use it for WARN.
+ * Since various instruction decoders disagree on the length of UD1,

arch/x86/include/asm/bug.h line 20

Quoted 10 lines
arch/x86/include/asm/bug.h +10 -0
+ * we cannot use it either. So use UD0 for WARN.
+ *
+ * (binutils knows about "ud1" but {en,de}codes it as 2 bytes, whereas
+ *  our kernel decoder thinks it takes a ModRM byte, which seems consistent
+ *  with various things like the Intel SDM instruction encoding rules)
+ */
+
+#define ASM_UD0		".byte 0x0f, 0xff"
+#define ASM_UD1		".byte 0x0f, 0xb9" /* + ModRM */
+#define ASM_UD2		".byte 0x0f, 0x0b"

Thas ASM_UD1 macro isn't used anywhere.

Quoted 7 lines
arch/x86/kernel/traps.c +1 -0
@@ -169,6 +169,41 @@ void ist_end_non_atomic(void)
 	preempt_disable();
 }
 
+int is_valid_bugaddr(unsigned long addr)

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 182

Quoted 10 lines
arch/x86/kernel/traps.c +10 -0
+{
+	unsigned short ud;
+
+#ifdef CONFIG_X86_32
+	if (addr < PAGE_OFFSET)
+		return 0;
+#else
+	if ((long)addr > 0)
+		return 0;
+#endif

I think comparing with TASK_SIZE would be more correct and it wouldn't need an ifdef.

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 187

Quoted 5 lines
arch/x86/kernel/traps.c +5 -0
+	if (probe_kernel_address((unsigned short *)addr, ud))
+		return 0;
+
+	return ud == 0x0b0f || ud == 0xff0f;
+}

This code would be easier to grok if these were defines IMO.

Also, now that some of the BUG-specific functions are now also related to WARN, they should probably be renamed to describe their new purpose, like:

"report_bug" -> "report_bug_or_warning"
"fixup_bug" -> "fixup_bug_or_warning"

On a related note, if warn and bug are going to continue to use two separate ud instructions for the foreseeable future, report_bug() could be cleaned up a bit: e.g., for a ud0 instruction, it doesn't make sense to call find_bug().

Quoted 4 lines
arch/x86/kernel/traps.c +4 -0
+
+static int fixup_bug(struct pt_regs *regs, int trapnr)
+{
+	if (trapnr != X86_TRAP_UD)

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201

Quoted 10 lines
arch/x86/kernel/traps.c +10 -0
+		return 0;
+
+	switch (report_bug(regs->ip, regs)) {
+	case BUG_TRAP_TYPE_NONE:
+	case BUG_TRAP_TYPE_BUG:
+		break;
+
+	case BUG_TRAP_TYPE_WARN:
+		regs->ip += 2;
+		return 1;

For self-documentation purposes, maybe use a define for the length of the ud0 instruction?

Signature · 1 line
Josh
author maintainer
#3 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

Quoted 17 lines · Josh Poimboeuf

arch/x86/include/asm/bug.h line 20

Quoted 13 lines · Peter Zijlstra
arch/x86/include/asm/bug.h +13 -0
+/*
+ * Since some emulators terminate on UD2, we cannot use it for WARN.
+ * Since various instruction decoders disagree on the length of UD1,
+ * we cannot use it either. So use UD0 for WARN.
+ *
+ * (binutils knows about "ud1" but {en,de}codes it as 2 bytes, whereas
+ *  our kernel decoder thinks it takes a ModRM byte, which seems consistent
+ *  with various things like the Intel SDM instruction encoding rules)
+ */
+
+#define ASM_UD0		".byte 0x0f, 0xff"
+#define ASM_UD1		".byte 0x0f, 0xb9" /* + ModRM */
+#define ASM_UD2		".byte 0x0f, 0x0b"

Thas ASM_UD1 macro isn't used anywhere.

I have it there for completeness sake, and documentation purposes.

Quoted 6 lines
Quoted 6 lines
arch/x86/kernel/traps.c +0 -0
@@ -169,6 +169,41 @@ void ist_end_non_atomic(void)
 	preempt_disable();
 }
 
Quoted 14 lines

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 182

Quoted 11 lines
arch/x86/kernel/traps.c +11 -0
+int is_valid_bugaddr(unsigned long addr)
+{
+	unsigned short ud;
+
+#ifdef CONFIG_X86_32
+	if (addr < PAGE_OFFSET)
+		return 0;
+#else
+	if ((long)addr > 0)
+		return 0;
+#endif

I think comparing with TASK_SIZE would be more correct and it wouldn't need an ifdef.

Dunno about more correct (the TIF_ADDR32 case is irrelevant here), but yes, that would do away with the #ifdef.

Quoted 7 lines

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 187

Quoted 5 lines
arch/x86/kernel/traps.c +5 -0
+	if (probe_kernel_address((unsigned short *)addr, ud))
+		return 0;
+
+	return ud == 0x0b0f || ud == 0xff0f;
+}

This code would be easier to grok if these were defines IMO.

Would be nice if I could use the same defines; but I can't see how I can make that happen :/ But sure; I suppose I can add more defines.

Quoted 1 line

Also, now that some of the BUG-specific functions are now also related

They already were, its just that x86 only now will use the WARN part of it. I don't feel that should be part of this patch.

Quoted 10 lines

to WARN, they should probably be renamed to describe their new purpose, like:

"report_bug" -> "report_bug_or_warning"
"fixup_bug" -> "fixup_bug_or_warning"

On a related note, if warn and bug are going to continue to use two separate ud instructions for the foreseeable future, report_bug() could be cleaned up a bit: e.g., for a ud0 instruction, it doesn't make sense to call find_bug().

I'm sure you'll break $random arch if you go futz with that. Also, I think you mean UD2, since that's BUG. We actually need the bug_entry for WARNs (aka UD0).

Also, you're now optimizing the BUG() code; I don't think anybody cares about saving a few cycles there. It shouldn't happen in the first place.

Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/traps.c +8 -0
+static int fixup_bug(struct pt_regs *regs, int trapnr)
+{
+	if (trapnr != X86_TRAP_UD)
+		return 0;
+
+	switch (report_bug(regs->ip, regs)) {
+	case BUG_TRAP_TYPE_NONE:
+	case BUG_TRAP_TYPE_BUG:
Quoted 8 lines

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201

Quoted 5 lines
arch/x86/kernel/traps.c +5 -0
+		break;
+
+	case BUG_TRAP_TYPE_WARN:
+		regs->ip += 2;
+		return 1;

For self-documentation purposes, maybe use a define for the length of the ud0 instruction?

Well, UD0 and UD2 really. LENGTH_UD0_OR_UD2 is a bit of a fail, name wise.

#4 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

Quoted 2 lines · Peter Zijlstra

For self-documentation purposes, maybe use a define for the length of the ud0 instruction?

#define TWO 2

;-)

some things make sense as a define, others don't
(adding a comment, maybe)

#5 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

In reply to Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines

to WARN, they should probably be renamed to describe their new purpose, like:

"report_bug" -> "report_bug_or_warning"
"fixup_bug" -> "fixup_bug_or_warning"

On a related note, if warn and bug are going to continue to use two separate ud instructions for the foreseeable future, report_bug() could be cleaned up a bit: e.g., for a ud0 instruction, it doesn't make sense to call find_bug().

Quoted 6 lines

I'm sure you'll break $random arch if you go futz with that. Also, I think you mean UD2, since that's BUG. We actually need the bug_entry for WARNs (aka UD0).

Also, you're now optimizing the BUG() code; I don't think anybody cares about saving a few cycles there. It shouldn't happen in the first place.

My thinking was to make report_bug() a little less obtuse, but yeah, that would break other arches, so never mind...

Quoted 17 lines
Quoted 8 lines
Quoted 8 lines
arch/x86/kernel/traps.c +8 -0
+static int fixup_bug(struct pt_regs *regs, int trapnr)
+{
+	if (trapnr != X86_TRAP_UD)
+		return 0;
+
+	switch (report_bug(regs->ip, regs)) {
+	case BUG_TRAP_TYPE_NONE:
+	case BUG_TRAP_TYPE_BUG:
Quoted 8 lines

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201

Quoted 5 lines
arch/x86/kernel/traps.c +5 -0
+		break;
+
+	case BUG_TRAP_TYPE_WARN:
+		regs->ip += 2;
+		return 1;

For self-documentation purposes, maybe use a define for the length of the ud0 instruction?

Quoted 2 lines

Well, UD0 and UD2 really. LENGTH_UD0_OR_UD2 is a bit of a fail, name wise.

Why UD2? Warnings are UD0-only, no? What about UD0_LEN? Or at least a comment would be helpful I think.

Signature · 1 line
Josh
author maintainer
#6 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

Quoted 18 lines · Josh Poimboeuf
Quoted 17 lines · Peter Zijlstra
Quoted 16 lines

arch/x86/kernel/traps.c void ist_end_non_atomic(void) · line 201

Quoted 13 lines
arch/x86/kernel/traps.c +13 -0
+static int fixup_bug(struct pt_regs *regs, int trapnr)
+{
+	if (trapnr != X86_TRAP_UD)
+		return 0;
+
+	switch (report_bug(regs->ip, regs)) {
+	case BUG_TRAP_TYPE_NONE:
+	case BUG_TRAP_TYPE_BUG:
+		break;
+
+	case BUG_TRAP_TYPE_WARN:
+		regs->ip += 2;
+		return 1;

For self-documentation purposes, maybe use a define for the length of the ud0 instruction?

Quoted 5 lines
Quoted 2 lines

Well, UD0 and UD2 really. LENGTH_UD0_OR_UD2 is a bit of a fail, name wise.

Why UD2? Warnings are UD0-only, no? What about UD0_LEN? Or at least a comment would be helpful I think.

Oh, right, we only resume on WARN,.. Doh.

author maintainer
#7 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

In reply to Josh Poimboeuf

A little like so then.

---
Subject: x86: Implement __WARN using UD0
From: Peter Zijlstra <[email protected]>
Date: Thu Feb 2 14:43:51 CET 2017

By using "UD0" for WARNs we remove the function call and its possible __FILE__ and __LINE__ immediate arguments from the instruction stream.

Total image size will not change much, what we win in the instruction stream we'll loose because of the __bug_table entries. Still, saves on I$ footprint and the total image size does go down a bit.

text data bss dec hex filename

10702123        4530992  843776 16076891         f5505b defconfig-build/vmlinux.0
10682460        4530992  843776 16057228         f5038c defconfig-build/vmlinux.1

(um didn't seem to use GENERIC_BUG at all, so remove it)

Cc Linus Torvalds <[email protected]>
Cc Richard Weinberger <[email protected]>
Cc Arjan van de Ven <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Josh Poimboeuf <[email protected]>
---
 arch/um/Kconfig.common         |    5 --
 arch/x86/include/asm/bug.h     |   73 +++++++++++++++++++++++++++++++----------
 arch/x86/kernel/dumpstack.c    |    3 -
 arch/x86/kernel/dumpstack_32.c |   12 ------
 arch/x86/kernel/dumpstack_64.c |   10 -----
 arch/x86/kernel/traps.c        |   46 ++++++++++++++++++++++---
 arch/x86/um/Makefile           |    2 -
 arch/x86/um/bug.c              |   21 -----------
 8 files changed, 97 insertions(+), 75 deletions(-)
arch/um/Kconfig.common +0 -5
@@ -50,11 +50,6 @@ config GENERIC_CALIBRATE_DELAY
 	bool
 	default y
 
-config GENERIC_BUG
-	bool
-	default y
-	depends on BUG
-
 config HZ
 	int
 	default 100
arch/x86/include/asm/bug.h +56 -17

92 lines not on this page · raw message

arch/x86/kernel/dumpstack.c +0 -3
@@ -289,9 +289,6 @@ void die(const char *str, struct pt_regs
 	unsigned long flags = oops_begin();
 	int sig = SIGSEGV;
 
-	if (!user_mode(regs))
-		report_bug(regs->ip, regs);
-
 	if (__die(str, regs, err))
 		sig = 0;
 	oops_end(flags, regs, sig);
arch/x86/kernel/dumpstack_32.c +0 -12
@@ -162,15 +162,3 @@ void show_regs(struct pt_regs *regs)
 	}
 	pr_cont("\n");
 }
-
-int is_valid_bugaddr(unsigned long ip)
-{
-	unsigned short ud2;
-
-	if (ip < PAGE_OFFSET)
-		return 0;
-	if (probe_kernel_address((unsigned short *)ip, ud2))
-		return 0;
-
-	return ud2 == 0x0b0f;
-}
arch/x86/kernel/dumpstack_64.c +0 -10
@@ -178,13 +178,3 @@ void show_regs(struct pt_regs *regs)
 	}
 	pr_cont("\n");
 }
-
-int is_valid_bugaddr(unsigned long ip)
-{
-	unsigned short ud2;
-
-	if (__copy_from_user(&ud2, (const void __user *) ip, sizeof(ud2)))
-		return 0;
-
-	return ud2 == 0x0b0f;
-}
arch/x86/kernel/traps.c +40 -6

58 lines not on this page · raw message

arch/x86/um/Makefile +1 -1
@@ -8,7 +8,7 @@ else
 	BITS := 64
 endif
 
-obj-y = bug.o bugs_$(BITS).o delay.o fault.o ldt.o \
+obj-y = bugs_$(BITS).o delay.o fault.o ldt.o \
 	ptrace_$(BITS).o ptrace_user.o setjmp_$(BITS).o signal.o \
 	stub_$(BITS).o stub_segv.o \
 	sys_call_table_$(BITS).o sysrq_$(BITS).o tls_$(BITS).o \
arch/x86/um/bug.c deleted +0 -21
@@ -1,21 +0,0 @@
-/*
- * Copyright (C) 2006 Jeff Dike ([email protected])
- * Licensed under the GPL V2
- */
-
-#include <linux/uaccess.h>
-
-/*
- * Mostly copied from i386/x86_86 - eliminated the eip < PAGE_OFFSET because
- * that's not relevant in skas mode.
- */
-
-int is_valid_bugaddr(unsigned long eip)
-{
-	unsigned short ud2;
-
-	if (probe_kernel_address((unsigned short __user *)eip, ud2))
-		return 0;
-
-	return ud2 == 0x0b0f;
-}
Reviewed-by
#8 lore raw

Re: [PATCH 1/5] x86: Implement __WARN using UD0

Quoted 21 lines · Peter Zijlstra

A little like so then.

---
Subject: x86: Implement __WARN using UD0
From: Peter Zijlstra <[email protected]>
Date: Thu Feb 2 14:43:51 CET 2017

By using "UD0" for WARNs we remove the function call and its possible __FILE__ and __LINE__ immediate arguments from the instruction stream.

Total image size will not change much, what we win in the instruction stream we'll loose because of the __bug_table entries. Still, saves on I$ footprint and the total image size does go down a bit.

text data bss dec hex filename

10702123        4530992  843776 16076891         f5505b defconfig-build/vmlinux.0
10682460        4530992  843776 16057228         f5038c defconfig-build/vmlinux.1

(um didn't seem to use GENERIC_BUG at all, so remove it)

Quoted 7 lines
Cc Linus Torvalds <[email protected]>
Cc Richard Weinberger <[email protected]>
Cc Arjan van de Ven <[email protected]>
Cc Borislav Petkov <[email protected]>
Cc Josh Poimboeuf <[email protected]>

Looks good, thanks!

Signature · 1 line
Josh
author maintainer
#9 lore raw

PATCH 2/5 bug: Add _ONCE logic to report_bug()

In reply to Peter Zijlstra

Josh suggested moving the _ONCE logic inside the trap handler, using a bit in the bug_entry::flags field, avoiding the need for the extra variable.

Sadly this only works for WARN_ON_ONCE(), since the others have printk() statements prior to triggering the trap.

Still, this saves a fair amount of text and data:

  text            data     bss    dec              hex    filename
  10682460        4530992  843776 16057228         f5038c defconfig-build/vmlinux.1
  10665111        4530096  843776 16038983         f4bc47 defconfig-build/vmlinux.2
---
 arch/arm64/include/asm/bug.h      |    2 +-
 arch/parisc/include/asm/bug.h     |    8 ++++----
 arch/powerpc/include/asm/bug.h    |    4 ++--
 arch/s390/include/asm/bug.h       |    4 ++--
 arch/sh/include/asm/bug.h         |    4 ++--
 arch/x86/include/asm/bug.h        |    2 +-
 include/asm-generic/bug.h         |   18 +++++++++++++++++-
 include/asm-generic/vmlinux.lds.h |    3 +--
 include/linux/bug.h               |    2 +-
 lib/bug.c                         |   28 ++++++++++++++++++++--------
 10 files changed, 51 insertions(+), 24 deletions(-)
arch/arm64/include/asm/bug.h +1 -1
@@ -55,7 +55,7 @@ _BUGVERBOSE_LOCATION(__FILE__, __LINE__)
 	unreachable();				\
 } while (0)
 
-#define __WARN_TAINT(taint) _BUG_FLAGS(BUGFLAG_TAINT(taint))
+#define __WARN_FLAGS(flags) _BUG_FLAGS(BUGFLAG_WARNING|(flags))
 
 #endif /* ! CONFIG_GENERIC_BUG */
 
arch/parisc/include/asm/bug.h +4 -4
@@ -46,7 +46,7 @@
 #endif
 
 #ifdef CONFIG_DEBUG_BUGVERBOSE
-#define __WARN_TAINT(taint)						\
+#define __WARN_FLAGS(flags)						\
 	do {								\
 		asm volatile("\n"					\
 			     "1:\t" PARISC_BUG_BREAK_ASM "\n"		\
@@ -56,11 +56,11 @@
 			     "\t.org 2b+%c3\n"				\
 			     "\t.popsection"				\
 			     : : "i" (__FILE__), "i" (__LINE__),	\
-			     "i" (BUGFLAG_TAINT(taint)), 		\
+			     "i" (BUGFLAG_WARNING|(flags)),		\
 			     "i" (sizeof(struct bug_entry)) );		\
 	} while(0)
 #else
-#define __WARN_TAINT(taint)						\
+#define __WARN_FLAGS(flags)						\
 	do {								\
 		asm volatile("\n"					\
 			     "1:\t" PARISC_BUG_BREAK_ASM "\n"		\
@@ -69,7 +69,7 @@
 			     "\t.short %c0\n"				\
 			     "\t.org 2b+%c1\n"				\
 			     "\t.popsection"				\
-			     : : "i" (BUGFLAG_TAINT(taint)),		\
+			     : : "i" (BUGFLAG_WARNING|(flags)),		\
 			     "i" (sizeof(struct bug_entry)) );		\
 	} while(0)
 #endif
arch/powerpc/include/asm/bug.h +2 -2
@@ -85,12 +85,12 @@
 	}							\
 } while (0)
 
-#define __WARN_TAINT(taint) do {				\
+#define __WARN_FLAGS(flags) do {				\
 	__asm__ __volatile__(					\
 		"1:	twi 31,0,0\n"				\
 		_EMIT_BUG_ENTRY					\
 		: : "i" (__FILE__), "i" (__LINE__),		\
-		  "i" (BUGFLAG_TAINT(taint)),			\
+		  "i" (BUGFLAG_WARNING|(flags)),		\
 		  "i" (sizeof(struct bug_entry)));		\
 } while (0)
 
arch/s390/include/asm/bug.h +2 -2
@@ -46,8 +46,8 @@
 	unreachable();					\
 } while (0)
 
-#define __WARN_TAINT(taint) do {			\
-	__EMIT_BUG(BUGFLAG_TAINT(taint));		\
+#define __WARN_FLAGS(flags) do {			\
+	__EMIT_BUG(BUGFLAG_WARNING|(flags));		\
 } while (0)
 
 #define WARN_ON(x) ({					\
arch/sh/include/asm/bug.h +2 -2
@@ -50,7 +50,7 @@ do {							\
 		   "i" (sizeof(struct bug_entry)));	\
 } while (0)
 
-#define __WARN_TAINT(taint)				\
+#define __WARN_FLAGS(flags)				\
 do {							\
 	__asm__ __volatile__ (				\
 		"1:\t.short %O0\n"			\
@@ -59,7 +59,7 @@ do {							\
 		 : "n" (TRAPA_BUG_OPCODE),		\
 		   "i" (__FILE__),			\
 		   "i" (__LINE__),			\
-		   "i" (BUGFLAG_TAINT(taint)),		\
+		   "i" (BUGFLAG_WARNING|(flags)),	\
 		   "i" (sizeof(struct bug_entry)));	\
 } while (0)
 
arch/x86/include/asm/bug.h +1 -1
@@ -64,7 +64,7 @@ do {								\
 	unreachable();						\
 } while (0)
 
-#define __WARN_TAINT(taint)	_BUG_FLAGS(ASM_UD0, BUGFLAG_TAINT(taint))
+#define __WARN_FLAGS(flags)	_BUG_FLAGS(ASM_UD0, BUGFLAG_WARNING|(flags))
 
 #include <asm-generic/bug.h>
 
include/asm-generic/bug.h +17 -1
@@ -5,6 +5,8 @@
 
 #ifdef CONFIG_GENERIC_BUG
 #define BUGFLAG_WARNING		(1 << 0)
+#define BUGFLAG_ONCE		(1 << 1)
+#define BUGFLAG_DONE		(1 << 2)
 #define BUGFLAG_TAINT(taint)	(BUGFLAG_WARNING | ((taint) << 8))
 #define BUG_GET_TAINT(bug)	((bug)->flags >> 8)
 #endif
@@ -55,6 +57,18 @@ struct bug_entry {
 #define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0)
 #endif
 
+#ifdef __WARN_FLAGS
+#define __WARN_TAINT(taint)		__WARN_FLAGS(BUGFLAG_TAINT(taint))
+#define __WARN_ONCE_TAINT(taint)	__WARN_FLAGS(BUGFLAG_ONCE|BUGFLAG_TAINT(taint))
+
+#define WARN_ON_ONCE(condition) ({				\
+	int __ret_warn_on = !!(condition);			\
+	if (unlikely(__ret_warn_on))				\
+		__WARN_ONCE_TAINT(TAINT_WARN);			\
+	unlikely(__ret_warn_on);				\
+})
+#endif
+
 /*
  * WARN(), WARN_ON(), WARN_ON_ONCE, and so on can be used to report
  * significant issues that need prompt attention if they should ever
@@ -97,7 +111,7 @@ void __warn(const char *file, int line,
 #endif
 
 #ifndef WARN
-#define WARN(condition, format...) ({						\
+#define WARN(condition, format...) ({					\
 	int __ret_warn_on = !!(condition);				\
 	if (unlikely(__ret_warn_on))					\
 		__WARN_printf(format);					\
@@ -112,6 +126,7 @@ void __warn(const char *file, int line,
 	unlikely(__ret_warn_on);					\
 })
 
+#ifndef WARN_ON_ONCE
 #define WARN_ON_ONCE(condition)	({				\
 	static bool __section(.data.unlikely) __warned;		\
 	int __ret_warn_once = !!(condition);			\
@@ -122,6 +137,7 @@ void __warn(const char *file, int line,
 	}							\
 	unlikely(__ret_warn_once);				\
 })
+#endif
 
 #define WARN_ONCE(condition, format...)	({			\
 	static bool __section(.data.unlikely) __warned;		\
include/asm-generic/vmlinux.lds.h +1 -2
@@ -286,8 +286,6 @@
 		*(.rodata1)						\
 	}								\
 									\
-	BUG_TABLE							\
-									\
 	/* PCI quirks */						\
 	.pci_fixup        : AT(ADDR(.pci_fixup) - LOAD_OFFSET) {	\
 		VMLINUX_SYMBOL(__start_pci_fixups_early) = .;		\
@@ -855,6 +853,7 @@
 		READ_MOSTLY_DATA(cacheline)				\
 		DATA_DATA						\
 		CONSTRUCTORS						\
+		BUG_TABLE						\
 	}
 
 #define INIT_TEXT_SECTION(inittext_align)				\
include/linux/bug.h +1 -1
@@ -105,7 +105,7 @@ static inline int is_warning_bug(const s
 	return bug->flags & BUGFLAG_WARNING;
 }
 
-const struct bug_entry *find_bug(unsigned long bugaddr);
+struct bug_entry *find_bug(unsigned long bugaddr);
 
 enum bug_trap_type report_bug(unsigned long bug_addr, struct pt_regs *regs);
 
lib/bug.c +20 -8
@@ -47,7 +47,7 @@
 #include <linux/sched.h>
 #include <linux/rculist.h>
 
-extern const struct bug_entry __start___bug_table[], __stop___bug_table[];
+extern struct bug_entry __start___bug_table[], __stop___bug_table[];
 
 static inline unsigned long bug_addr(const struct bug_entry *bug)
 {
@@ -62,10 +62,10 @@ static inline unsigned long bug_addr(con
 /* Updates are protected by module mutex */
 static LIST_HEAD(module_bug_list);
 
-static const struct bug_entry *module_find_bug(unsigned long bugaddr)
+static struct bug_entry *module_find_bug(unsigned long bugaddr)
 {
 	struct module *mod;
-	const struct bug_entry *bug = NULL;
+	struct bug_entry *bug = NULL;
 
 	rcu_read_lock_sched();
 	list_for_each_entry_rcu(mod, &module_bug_list, bug_list) {
@@ -122,15 +122,15 @@ void module_bug_cleanup(struct module *m
 
 #else
 
-static inline const struct bug_entry *module_find_bug(unsigned long bugaddr)
+static inline struct bug_entry *module_find_bug(unsigned long bugaddr)
 {
 	return NULL;
 }
 #endif
 
-const struct bug_entry *find_bug(unsigned long bugaddr)
+struct bug_entry *find_bug(unsigned long bugaddr)
 {
-	const struct bug_entry *bug;
+	struct bug_entry *bug;
 
 	for (bug = __start___bug_table; bug < __stop___bug_table; ++bug)
 		if (bugaddr == bug_addr(bug))
@@ -141,9 +141,9 @@ const struct bug_entry *find_bug(unsigne
 
 enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)
 {
-	const struct bug_entry *bug;
+	struct bug_entry *bug;
 	const char *file;
-	unsigned line, warning;
+	unsigned line, warning, once, done;
 
 	if (!is_valid_bugaddr(bugaddr))
 		return BUG_TRAP_TYPE_NONE;
@@ -164,6 +164,18 @@ enum bug_trap_type report_bug(unsigned l
 		line = bug->line;
 #endif
 		warning = (bug->flags & BUGFLAG_WARNING) != 0;
+		once = (bug->flags & BUGFLAG_ONCE) != 0;
+		done = (bug->flags & BUGFLAG_DONE) != 0;
+
+		if (warning && once) {
+			if (done)
+				return BUG_TRAP_TYPE_WARN;
+
+			/*
+			 * Since this is the only store, concurrency is not an issue.
+			 */
+			bug->flags |= BUGFLAG_DONE;
+		}
 	}
 
 	if (warning) {
author maintainer
#10 lore raw

PATCH 3/5 atomic: Introduce atomic_try_cmpxchg()

In reply to Peter Zijlstra

Add a new cmpxchg interface:

  bool try_cmpxchg(u{8,16,32,64} *ptr, u{8,16,32,64} *val, u{8,16,32,64} new);

Where the boolean returns the result of the compare; and thus if the exchange happened; and in case of failure, the new value of *ptr is returned in *val.

This allows simplification/improvement of loops like:

	for (;;) {
		new = val $op $imm;
		old = cmpxchg(ptr, val, new);
		if (old == val)
			break;
		val = old;
	}

into:

	do {
		new = val $op $imm;
	} while (!try_cmpxchg(ptr, &val, new));

while also generating better code (GCC6 and onwards).

---
 arch/x86/include/asm/atomic.h      |    6 +++
 arch/x86/include/asm/atomic64_64.h |    6 +++
 arch/x86/include/asm/cmpxchg.h     |   69 +++++++++++++++++++++++++++++++++++++
 include/linux/atomic.h             |   42 ++++++++++++++++++++++
 4 files changed, 123 insertions(+)
arch/x86/include/asm/atomic.h +6 -0
@@ -186,6 +186,12 @@ static __always_inline int atomic_cmpxch
 	return cmpxchg(&v->counter, old, new);
 }
 
+#define atomic_try_cmpxchg atomic_try_cmpxchg
+static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new)
+{
+	return try_cmpxchg(&v->counter, old, new);
+}
+
 static inline int atomic_xchg(atomic_t *v, int new)
 {
 	return xchg(&v->counter, new);
arch/x86/include/asm/atomic64_64.h +6 -0
@@ -176,6 +176,12 @@ static inline long atomic64_cmpxchg(atom
 	return cmpxchg(&v->counter, old, new);
 }
 
+#define atomic64_try_cmpxchg atomic64_try_cmpxchg
+static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, long *old, long new)
+{
+	return try_cmpxchg(&v->counter, old, new);
+}
+
 static inline long atomic64_xchg(atomic64_t *v, long new)
 {
 	return xchg(&v->counter, new);
arch/x86/include/asm/cmpxchg.h +69 -0

75 lines not on this page · raw message

include/linux/atomic.h +42 -0

54 lines not on this page · raw message

author maintainer
#11 lore raw

PATCH 4/5 refcount: Use atomic_try_cmpxchg()

In reply to Peter Zijlstra

Generates better code (GCC-6.2.1):

   text    data     bss     dec     hex filename
   1576       7       0    1583     62f defconfig-build/lib/refcount.o.pre
   1488       7       0    1495     5d7 defconfig-build/lib/refcount.o.post
---
 lib/refcount.c |   47 +++++++++++++++--------------------------------
 1 file changed, 15 insertions(+), 32 deletions(-)
lib/refcount.c +15 -32
@@ -57,9 +57,9 @@
  */
 bool refcount_add_not_zero(unsigned int i, refcount_t *r)
 {
-	unsigned int old, new, val = atomic_read(&r->refs);
+	unsigned int new, val = atomic_read(&r->refs);
 
-	for (;;) {
+	do {
 		if (!val)
 			return false;
 
@@ -69,12 +69,8 @@ bool refcount_add_not_zero(unsigned int
 		new = val + i;
 		if (new < val)
 			new = UINT_MAX;
-		old = atomic_cmpxchg_relaxed(&r->refs, val, new);
-		if (old == val)
-			break;
 
-		val = old;
-	}
+	} while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
 
 	WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
 
@@ -118,9 +114,9 @@ EXPORT_SYMBOL_GPL(refcount_add);
  */
 bool refcount_inc_not_zero(refcount_t *r)
 {
-	unsigned int old, new, val = atomic_read(&r->refs);
+	unsigned int new, val = atomic_read(&r->refs);
 
-	for (;;) {
+	do {
 		new = val + 1;
 
 		if (!val)
@@ -129,12 +125,7 @@ bool refcount_inc_not_zero(refcount_t *r
 		if (unlikely(!new))
 			return true;
 
-		old = atomic_cmpxchg_relaxed(&r->refs, val, new);
-		if (old == val)
-			break;
-
-		val = old;
-	}
+	} while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
 
 	WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
 
@@ -182,9 +173,9 @@ EXPORT_SYMBOL_GPL(refcount_inc);
  */
 bool refcount_sub_and_test(unsigned int i, refcount_t *r)
 {
-	unsigned int old, new, val = atomic_read(&r->refs);
+	unsigned int new, val = atomic_read(&r->refs);
 
-	for (;;) {
+	do {
 		if (unlikely(val == UINT_MAX))
 			return false;
 
@@ -194,12 +185,7 @@ bool refcount_sub_and_test(unsigned int
 			return false;
 		}
 
-		old = atomic_cmpxchg_release(&r->refs, val, new);
-		if (old == val)
-			break;
-
-		val = old;
-	}
+	} while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
 
 	return !new;
 }
@@ -258,7 +244,9 @@ EXPORT_SYMBOL_GPL(refcount_dec);
  */
 bool refcount_dec_if_one(refcount_t *r)
 {
-	return atomic_cmpxchg_release(&r->refs, 1, 0) == 1;
+	int val = 1;
+
+	return atomic_try_cmpxchg_release(&r->refs, &val, 0);
 }
 EXPORT_SYMBOL_GPL(refcount_dec_if_one);
 
@@ -275,9 +263,9 @@ EXPORT_SYMBOL_GPL(refcount_dec_if_one);
  */
 bool refcount_dec_not_one(refcount_t *r)
 {
-	unsigned int old, new, val = atomic_read(&r->refs);
+	unsigned int new, val = atomic_read(&r->refs);
 
-	for (;;) {
+	do {
 		if (unlikely(val == UINT_MAX))
 			return true;
 
@@ -290,12 +278,7 @@ bool refcount_dec_not_one(refcount_t *r)
 			return true;
 		}
 
-		old = atomic_cmpxchg_release(&r->refs, val, new);
-		if (old == val)
-			break;
-
-		val = old;
-	}
+	} while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
 
 	return true;
 }
author maintainer
#12 lore raw

PATCH 5/5 x86,atomic: Use atomic_try_cmpxchg

In reply to Peter Zijlstra

text data bss dec hex filename

10665111        4530096  843776 16038983         f4bc47 defconfig-build/vmlinux.3
10655703        4530096  843776 16029575         f49787 defconfig-build/vmlinux.4
---
 arch/x86/include/asm/atomic.h      |   27 ++++++++----------------
 arch/x86/include/asm/atomic64_64.h |   40 ++++++++++++-------------------------
 2 files changed, 22 insertions(+), 45 deletions(-)
arch/x86/include/asm/atomic.h +9 -18
@@ -207,16 +207,12 @@ static inline void atomic_##op(int i, at
 }
 
 #define ATOMIC_FETCH_OP(op, c_op)					\
-static inline int atomic_fetch_##op(int i, atomic_t *v)		\
+static inline int atomic_fetch_##op(int i, atomic_t *v)			\
 {									\
-	int old, val = atomic_read(v);					\
-	for (;;) {							\
-		old = atomic_cmpxchg(v, val, val c_op i);		\
-		if (old == val)						\
-			break;						\
-		val = old;						\
-	}								\
-	return old;							\
+	int val = atomic_read(v);					\
+	do {								\
+	} while (!atomic_try_cmpxchg(v, &val, val c_op i));		\
+	return val;							\
 }
 
 #define ATOMIC_OPS(op, c_op)						\
@@ -242,16 +238,11 @@ ATOMIC_OPS(xor, ^)
  */
 static __always_inline int __atomic_add_unless(atomic_t *v, int a, int u)
 {
-	int c, old;
-	c = atomic_read(v);
-	for (;;) {
-		if (unlikely(c == (u)))
+	int c = atomic_read(v);
+	do {
+		if (unlikely(c == u))
 			break;
-		old = atomic_cmpxchg((v), c, c + (a));
-		if (likely(old == c))
-			break;
-		c = old;
-	}
+	} while (!atomic_try_cmpxchg(v, &c, c + a));
 	return c;
 }
 
arch/x86/include/asm/atomic64_64.h +13 -27
@@ -198,17 +198,12 @@ static inline long atomic64_xchg(atomic6
  */
 static inline bool atomic64_add_unless(atomic64_t *v, long a, long u)
 {
-	long c, old;
-	c = atomic64_read(v);
-	for (;;) {
-		if (unlikely(c == (u)))
-			break;
-		old = atomic64_cmpxchg((v), c, c + (a));
-		if (likely(old == c))
-			break;
-		c = old;
-	}
-	return c != (u);
+	long c = atomic64_read(v);
+	do {
+		if (unlikely(c == u))
+			return false;
+	} while (!atomic64_try_cmpxchg(v, &c, c + a));
+	return true;
 }
 
 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
@@ -222,17 +217,12 @@ static inline bool atomic64_add_unless(a
  */
 static inline long atomic64_dec_if_positive(atomic64_t *v)
 {
-	long c, old, dec;
-	c = atomic64_read(v);
-	for (;;) {
+	long dec, c = atomic64_read(v);
+	do {
 		dec = c - 1;
 		if (unlikely(dec < 0))
 			break;
-		old = atomic64_cmpxchg((v), c, dec);
-		if (likely(old == c))
-			break;
-		c = old;
-	}
+	} while (!atomic64_try_cmpxchg(v, &c, dec));
 	return dec;
 }
 
@@ -248,14 +238,10 @@ static inline void atomic64_##op(long i,
 #define ATOMIC64_FETCH_OP(op, c_op)					\
 static inline long atomic64_fetch_##op(long i, atomic64_t *v)		\
 {									\
-	long old, val = atomic64_read(v);				\
-	for (;;) {							\
-		old = atomic64_cmpxchg(v, val, val c_op i);		\
-		if (old == val)						\
-			break;						\
-		val = old;						\
-	}								\
-	return old;							\
+	long val = atomic64_read(v);					\
+	do {								\
+	} while (!atomic64_try_cmpxchg(v, &val, val c_op i));		\
+	return val;							\
 }
 
 #define ATOMIC64_OPS(op, c_op)						\