Skip to contentSkip to conversation

x86: introduce and use percpu_inc()

PATCH In review Jan Beulich 2 participants 3 messages Apr 21, 2010 – Apr 29, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 x86: introduce and use percpu_inc()

... generating slightly smaller code.

---
 arch/x86/include/asm/hardirq.h   |    2 +-
 arch/x86/include/asm/percpu.h    |   24 ++++++++++++++++++++++++
 arch/x86/kernel/cpu/mcheck/mce.c |    4 ++--
 3 files changed, 27 insertions(+), 3 deletions(-)
arch/x86/include/asm/hardirq.h +1 -1
@@ -35,7 +35,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpust
 
 #define __ARCH_IRQ_STAT
 
-#define inc_irq_stat(member)	percpu_add(irq_stat.member, 1)
+#define inc_irq_stat(member)	percpu_inc(irq_stat.member)
 
 #define local_softirq_pending()	percpu_read(irq_stat.__softirq_pending)
 
arch/x86/include/asm/percpu.h +24 -0
@@ -190,6 +190,29 @@ do {									\
 	pfo_ret__;					\
 })
 
+#define percpu_unary_op(op, var)			\
+({							\
+	switch (sizeof(var)) {				\
+	case 1:						\
+		asm(op "b "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	case 2:						\
+		asm(op "w "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	case 4:						\
+		asm(op "l "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	case 8:						\
+		asm(op "q "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	default: __bad_percpu_size();			\
+	}						\
+})
+
 /*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
@@ -207,6 +230,7 @@ do {									\
 #define percpu_and(var, val)		percpu_to_op("and", var, val)
 #define percpu_or(var, val)		percpu_to_op("or", var, val)
 #define percpu_xor(var, val)		percpu_to_op("xor", var, val)
+#define percpu_inc(var)		percpu_unary_op("inc", var)
 
 #define __this_cpu_read_1(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define __this_cpu_read_2(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
@@ -539,7 +539,7 @@ void machine_check_poll(enum mcp_flags f
 	struct mce m;
 	int i;
 
-	__get_cpu_var(mce_poll_count)++;
+	percpu_inc(mce_poll_count);
 
 	mce_setup(&m);
 
@@ -934,7 +934,7 @@ void do_machine_check(struct pt_regs *re
 
 	atomic_inc(&mce_entry);
 
-	__get_cpu_var(mce_exception_count)++;
+	percpu_inc(mce_exception_count);
 
 	if (notify_die(DIE_NMI, "machine check", regs, error_code,
 			   18, SIGKILL) == NOTIFY_STOP)
#1 lore raw
Quoted 3 lines · Jan Beulich

... generating slightly smaller code.

How much smaller? Keep in mind that although INC is smaller than ADD, the former has flag dependencies that the latter doesn't...

-hpa

#2 lore raw

[tip:x86/asm] x86, asm: Introduce and use percpu_inc()

In reply to Jan Beulich
Commit-ID:  402af0d7c692ddcfa2333e93d3f275ebd0487926
Gitweb:     http://git.kernel.org/tip/402af0d7c692ddcfa2333e93d3f275ebd0487926
Author:     Jan Beulich <[email protected]>
AuthorDate: Wed, 21 Apr 2010 15:21:51 +0100
Committer:  H. Peter Anvin <[email protected]>
CommitDate: Wed, 28 Apr 2010 16:58:49 -0700

x86, asm: Introduce and use percpu_inc()

... generating slightly smaller code.

LKML-Reference: <[email protected]>

---
 arch/x86/include/asm/hardirq.h   |    2 +-
 arch/x86/include/asm/percpu.h    |   24 ++++++++++++++++++++++++
 arch/x86/kernel/cpu/mcheck/mce.c |    4 ++--
 3 files changed, 27 insertions(+), 3 deletions(-)
arch/x86/include/asm/hardirq.h +1 -1
index 0f85764..aeab29a 100644
@@ -35,7 +35,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 
 #define __ARCH_IRQ_STAT
 
-#define inc_irq_stat(member)	percpu_add(irq_stat.member, 1)
+#define inc_irq_stat(member)	percpu_inc(irq_stat.member)
 
 #define local_softirq_pending()	percpu_read(irq_stat.__softirq_pending)
 
arch/x86/include/asm/percpu.h +24 -0
index 66a272d..0ec6d12 100644
@@ -190,6 +190,29 @@ do {									\
 	pfo_ret__;					\
 })
 
+#define percpu_unary_op(op, var)			\
+({							\
+	switch (sizeof(var)) {				\
+	case 1:						\
+		asm(op "b "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	case 2:						\
+		asm(op "w "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	case 4:						\
+		asm(op "l "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	case 8:						\
+		asm(op "q "__percpu_arg(0)		\
+		    : "+m" (var));			\
+		break;					\
+	default: __bad_percpu_size();			\
+	}						\
+})
+
 /*
  * percpu_read() makes gcc load the percpu variable every time it is
  * accessed while percpu_read_stable() allows the value to be cached.
@@ -207,6 +230,7 @@ do {									\
 #define percpu_and(var, val)		percpu_to_op("and", var, val)
 #define percpu_or(var, val)		percpu_to_op("or", var, val)
 #define percpu_xor(var, val)		percpu_to_op("xor", var, val)
+#define percpu_inc(var)		percpu_unary_op("inc", var)
 
 #define __this_cpu_read_1(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
 #define __this_cpu_read_2(pcp)		percpu_from_op("mov", (pcp), "m"(pcp))
arch/x86/kernel/cpu/mcheck/mce.c +2 -2
index 8a6f0af..7a355dd 100644
@@ -539,7 +539,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
 	struct mce m;
 	int i;
 
-	__get_cpu_var(mce_poll_count)++;
+	percpu_inc(mce_poll_count);
 
 	mce_setup(&m);
 
@@ -934,7 +934,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
 
 	atomic_inc(&mce_entry);
 
-	__get_cpu_var(mce_exception_count)++;
+	percpu_inc(mce_exception_count);
 
 	if (notify_die(DIE_NMI, "machine check", regs, error_code,
 			   18, SIGKILL) == NOTIFY_STOP)