Skip to contentSkip to conversation

locking/pvqspinlock: Replace xchg() by the more descriptive set_mb()

tip:locking/core Peter Zijlstra 3 participants 7 messages May 11, 2015 – May 12, 2015 1 message not in the archive

Subsystems (16)
Documentation Arm (32-bit) x86 VFS +12 more subsystems
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

Peter Zijlstra

author maintainer
#1 lore raw

Review

Quoted 2 lines · tip-bot for Waiman Long
(quoted diff) +1 -1
-		(void)xchg(&pn->state, vcpu_halted);
+		set_mb(pn->state, vcpu_halted);

Hmm, so I looked at the set_mb() definitions and I figure we want to do something like the below, right?

---
 arch/arm/include/asm/barrier.h      | 2 +-
 arch/arm64/include/asm/barrier.h    | 2 +-
 arch/ia64/include/asm/barrier.h     | 2 +-
 arch/metag/include/asm/barrier.h    | 2 +-
 arch/mips/include/asm/barrier.h     | 2 +-
 arch/powerpc/include/asm/barrier.h  | 2 +-
 arch/s390/include/asm/barrier.h     | 2 +-
 arch/sparc/include/asm/barrier_64.h | 2 +-
 arch/x86/include/asm/barrier.h      | 2 +-
 include/asm-generic/barrier.h       | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)
arch/arm/include/asm/barrier.h +1 -1
index d2f81e6b8c1c..993150aea681 100644
@@ -81,7 +81,7 @@ do {									\
 #define read_barrier_depends()		do { } while(0)
 #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_mb__before_atomic()	smp_mb()
 #define smp_mb__after_atomic()	smp_mb()
arch/arm64/include/asm/barrier.h +1 -1
index 71f19c4dc0de..ff7de78d01b8 100644
@@ -114,7 +114,7 @@ do {									\
 #define read_barrier_depends()		do { } while(0)
 #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 #define nop()		asm volatile("nop");
 
 #define smp_mb__before_atomic()	smp_mb()
arch/ia64/include/asm/barrier.h +1 -1
index f6769eb2bbf9..03117e7b2ab8 100644
@@ -82,7 +82,7 @@ do {									\
  * acquire vs release semantics but we can't discuss this stuff with
  * Linus just yet.  Grrr...
  */
-#define set_mb(var, value)	do { (var) = (value); mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
 
 /*
  * The group barrier in front of the rsm & ssm are necessary to ensure
arch/metag/include/asm/barrier.h +1 -1
index d703d8e26a65..97eb018a2933 100644
@@ -84,7 +84,7 @@ static inline void fence(void)
 #define read_barrier_depends()		do { } while (0)
 #define smp_read_barrier_depends()	do { } while (0)
 
-#define set_mb(var, value) do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_store_release(p, v)						\
 do {									\
arch/mips/include/asm/barrier.h +1 -1
index 2b8bbbcb9be0..cff1bbdaa74a 100644
@@ -113,7 +113,7 @@
 #endif
 
 #define set_mb(var, value) \
-	do { var = value; smp_mb(); } while (0)
+	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_llsc_mb()	__asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
 
arch/powerpc/include/asm/barrier.h +1 -1
index a3bf5be111ff..2a072e48780d 100644
@@ -34,7 +34,7 @@
 #define rmb()  __asm__ __volatile__ ("sync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("sync" : : : "memory")
 
-#define set_mb(var, value)	do { var = value; mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
 
 #ifdef __SUBARCH_HAS_LWSYNC
 #    define SMPWMB      LWSYNC
arch/s390/include/asm/barrier.h +1 -1
index 8d724718ec21..b66cd53d35fc 100644
@@ -36,7 +36,7 @@
 #define smp_mb__before_atomic()		smp_mb()
 #define smp_mb__after_atomic()		smp_mb()
 
-#define set_mb(var, value)		do { var = value; mb(); } while (0)
+#define set_mb(var, value)		do { WRITE_ONCE(var, value); mb(); } while (0)
 
 #define smp_store_release(p, v)						\
 do {									\
arch/sparc/include/asm/barrier_64.h +1 -1
index 76648941fea7..125fec7512f4 100644
@@ -41,7 +41,7 @@ do {	__asm__ __volatile__("ba,pt	%%xcc, 1f\n\t" \
 #define dma_wmb()	wmb()
 
 #define set_mb(__var, __value) \
-	do { __var = __value; membar_safe("#StoreLoad"); } while(0)
+	do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0)
 
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
arch/x86/include/asm/barrier.h +1 -1
index 959e45b81fe2..9de5cde133a1 100644
@@ -40,7 +40,7 @@
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
-#define set_mb(var, value) do { var = value; barrier(); } while (0)
+#define set_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
 #endif /* SMP */
 
 #define read_barrier_depends()		do { } while (0)
include/asm-generic/barrier.h +1 -1
index f5c40b0fadc2..3938716b44d7 100644
@@ -67,7 +67,7 @@
 #endif
 
 #ifndef set_mb
-#define set_mb(var, value)  do { (var) = (value); mb(); } while (0)
+#define set_mb(var, value)  do { WRITE_ONCE(var, value); mb(); } while (0)
 #endif
 
 #ifndef smp_mb__before_atomic
#2 lore raw
Quoted 5 lines · Peter Zijlstra

Review

Quoted 2 lines · tip-bot for Waiman Long
(quoted diff) +1 -1
-		(void)xchg(&pn->state, vcpu_halted);
+		set_mb(pn->state, vcpu_halted);

Hmm, so I looked at the set_mb() definitions and I figure we want to do something like the below, right?

Yes, I think we should do that just to be safe from unexpected compiler optimization.

Quoted 122 lines
---
  arch/arm/include/asm/barrier.h      | 2 +-
  arch/arm64/include/asm/barrier.h    | 2 +-
  arch/ia64/include/asm/barrier.h     | 2 +-
  arch/metag/include/asm/barrier.h    | 2 +-
  arch/mips/include/asm/barrier.h     | 2 +-
  arch/powerpc/include/asm/barrier.h  | 2 +-
  arch/s390/include/asm/barrier.h     | 2 +-
  arch/sparc/include/asm/barrier_64.h | 2 +-
  arch/x86/include/asm/barrier.h      | 2 +-
  include/asm-generic/barrier.h       | 2 +-
  10 files changed, 10 insertions(+), 10 deletions(-)
arch/arm/include/asm/barrier.h +1 -1
index d2f81e6b8c1c..993150aea681 100644
@@ -81,7 +81,7 @@ do {									\
  #define read_barrier_depends()		do { } while(0)
  #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
  #define smp_mb__before_atomic()	smp_mb()
  #define smp_mb__after_atomic()	smp_mb()
arch/arm64/include/asm/barrier.h +1 -1
index 71f19c4dc0de..ff7de78d01b8 100644
@@ -114,7 +114,7 @@ do {									\
  #define read_barrier_depends()		do { } while(0)
  #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
  #define nop()		asm volatile("nop");
 
  #define smp_mb__before_atomic()	smp_mb()
arch/ia64/include/asm/barrier.h +1 -1
index f6769eb2bbf9..03117e7b2ab8 100644
@@ -82,7 +82,7 @@ do {									\
   * acquire vs release semantics but we can't discuss this stuff with
   * Linus just yet.  Grrr...
   */
-#define set_mb(var, value)	do { (var) = (value); mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
 
  /*
   * The group barrier in front of the rsm&  ssm are necessary to ensure
arch/metag/include/asm/barrier.h +1 -1
index d703d8e26a65..97eb018a2933 100644
@@ -84,7 +84,7 @@ static inline void fence(void)
  #define read_barrier_depends()		do { } while (0)
  #define smp_read_barrier_depends()	do { } while (0)
 
-#define set_mb(var, value) do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
  #define smp_store_release(p, v)						\
  do {									\
arch/mips/include/asm/barrier.h +1 -1
index 2b8bbbcb9be0..cff1bbdaa74a 100644
@@ -113,7 +113,7 @@
  #endif
 
  #define set_mb(var, value) \
-	do { var = value; smp_mb(); } while (0)
+	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
  #define smp_llsc_mb()	__asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
 
arch/powerpc/include/asm/barrier.h +1 -1
index a3bf5be111ff..2a072e48780d 100644
@@ -34,7 +34,7 @@
  #define rmb()  __asm__ __volatile__ ("sync" : : : "memory")
  #define wmb()  __asm__ __volatile__ ("sync" : : : "memory")
 
-#define set_mb(var, value)	do { var = value; mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
 
  #ifdef __SUBARCH_HAS_LWSYNC
  #    define SMPWMB      LWSYNC
arch/s390/include/asm/barrier.h +1 -1
index 8d724718ec21..b66cd53d35fc 100644
@@ -36,7 +36,7 @@
  #define smp_mb__before_atomic()		smp_mb()
  #define smp_mb__after_atomic()		smp_mb()
 
-#define set_mb(var, value)		do { var = value; mb(); } while (0)
+#define set_mb(var, value)		do { WRITE_ONCE(var, value); mb(); } while (0)
 
  #define smp_store_release(p, v)						\
  do {									\
arch/sparc/include/asm/barrier_64.h +1 -1
index 76648941fea7..125fec7512f4 100644
@@ -41,7 +41,7 @@ do {	__asm__ __volatile__("ba,pt	%%xcc, 1f\n\t" \
  #define dma_wmb()	wmb()
 
  #define set_mb(__var, __value) \
-	do { __var = __value; membar_safe("#StoreLoad"); } while(0)
+	do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0)
 
  #ifdef CONFIG_SMP
  #define smp_mb()	mb()
arch/x86/include/asm/barrier.h +0 -0
index 959e45b81fe2..9de5cde133a1 100644

arch/x86/include/asm/barrier.h line 43

Quoted 5 lines
arch/x86/include/asm/barrier.h +1 -1
@@ -40,7 +40,7 @@
  #define smp_mb()	barrier()
  #define smp_rmb()	barrier()
  #define smp_wmb()	barrier()
-#define set_mb(var, value) do { var = value; barrier(); } while (0)
+#define set_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)

That part is in the !CONFIG_SMP portion. I don't think we need to change it.

Cheers,
Longman

#3 lore raw
In reply to Peter Zijlstra
Quoted 3 lines · Peter Zijlstra

Hmm, so I looked at the set_mb() definitions and I figure we want to do something like the below, right?

I don't think you need to do this for the non-smp cases. The whole thing is about smp memory ordering, so on UP you don't even need the WRITE_ONCE(), much less a barrier.

That said, I do wonder if we should make that "it's only an smp barrier" more explicit. We have non-smp barriers for people who do DMA, and while they should probably never use anything like set_mb() anyway (they tend to want *release* semantics, not a full barrier), from a conceptual standpoint the "set_mb()" function really is closer to the "smp_store_release()/smp_load_acquire()" family of macros.

So I wonder if we should change the name to match.

IOW, if we are really cleaning up smp_mb() and changing most of the lines associated with it (we really have very few users, and there seems to be more lines *defining* smp_mb() than there are lines *using* it in the kernel), maybe we should also just rename it "smp_store_mb()" at the same time.

I dunno. Maybe the churn isn't worth it.

Linus

author maintainer
#4 lore raw
Quoted 6 lines · Linus Torvalds
Quoted 3 lines · Peter Zijlstra

Hmm, so I looked at the set_mb() definitions and I figure we want to do something like the below, right?

I don't think you need to do this for the non-smp cases.

Well, its the store tearing thing again, we use WRITE_ONCE() in smp_store_release() for the same reason. We want it to be a single store.

Quoted 3 lines

The whole
thing is about smp memory ordering, so on UP you don't even need the WRITE_ONCE(), much less a barrier.

No, we actually need both still on UP.

Imagine the following sequence:

	for (;;) {
		set_current_state(TASK_KILLABLE);
		if (cond)
			break;

		schedule();
	}
	__set_current_state(TASK_RUNNING);

vs

	<IRQ>
		  wake_up_process(p);

As we know, set_current_state() is set_mb(), and thus will look like:

	current->state = TASK_KILLABLE;
	smp_mb();
	if (cond)
		break;

So without the WRITE_ONCE() we can get store tearing, and suppose our compiler is insane and translates the store into 4 byte stores.

	current->state[0] = TASK_UNINTERRUPTIBLE;
	current->state[1] = TASK_WAKEKILL >> 8;
	current->state[2] = 0;
	current->state[3] = 0;

The obvious fail here is to get the wakeup interrupt between [0] and [1].

	current->state[0] = TASK_UNINTERRUPTIBLE;

	<IRQ>
		wake_up_process(p);
		p->state = TASK_RUNNING;

	current->state[1] = TASK_WAKEKILL >> 8;
	current->state[2] = 0;
	current->state[3] = 0;

With the end result that ->state == TASK_WAKEKILL, from which we'll not wake up unless killed.

Similarly, without the barrier(), our friendly compiler is allowed to do:

	if (cond)
		break
	current->state = TASK_KILLABLE;
	schedule();

Which we all know to be broken.

So no, set_mb() (or smp_store_mb()) very much does need the WRITE_ONCE() and a barrier() on UP.

author maintainer
#5 lore raw
Quoted 15 lines · Peter Zijlstra
Quoted 6 lines · Linus Torvalds
Quoted 3 lines · Peter Zijlstra

Hmm, so I looked at the set_mb() definitions and I figure we want to do something like the below, right?

I don't think you need to do this for the non-smp cases.

Well, its the store tearing thing again, we use WRITE_ONCE() in smp_store_release() for the same reason. We want it to be a single store.

Quoted 3 lines

The whole
thing is about smp memory ordering, so on UP you don't even need the WRITE_ONCE(), much less a barrier.

Ah, you meant the memory barrier; indeed, a compiler barrier is sufficient. I got somewhat confused between Waiman's email and barrier and barrier() (again!).

author maintainer
#6 lore raw
In reply to Linus Torvalds
Quoted 5 lines · Linus Torvalds

from a conceptual standpoint the "set_mb()" function really is closer to the "smp_store_release()/smp_load_acquire()" family of macros.

So I wonder if we should change the name to match.

Quoted 7 lines

IOW, if we are really cleaning up smp_mb() and changing most of the lines associated with it (we really have very few users, and there seems to be more lines *defining* smp_mb() than there are lines *using* it in the kernel), maybe we should also just rename it "smp_store_mb()" at the same time.

I dunno. Maybe the churn isn't worth it.

Can't hurt, and its a trivial patch to generate.

git grep -l "\<set_mb\>" | while read file; do sed -i -e 's/\<set_mb\>/smp_store_mb/g' $file; done

And a manual edit later...

---
 Documentation/memory-barriers.txt   |    6 +++---
 arch/arm/include/asm/barrier.h      |    2 +-
 arch/arm64/include/asm/barrier.h    |    2 +-
 arch/ia64/include/asm/barrier.h     |    7 +------
 arch/metag/include/asm/barrier.h    |    2 +-
 arch/mips/include/asm/barrier.h     |    2 +-
 arch/powerpc/include/asm/barrier.h  |    2 +-
 arch/s390/include/asm/barrier.h     |    2 +-
 arch/sh/include/asm/barrier.h       |    2 +-
 arch/sparc/include/asm/barrier_64.h |    2 +-
 arch/x86/include/asm/barrier.h      |    4 ++--
 arch/x86/um/asm/barrier.h           |    3 ++-
 fs/select.c                         |    6 +++---
 include/asm-generic/barrier.h       |    4 ++--
 include/linux/sched.h               |    8 ++++----
 kernel/futex.c                      |    2 +-
 kernel/locking/qspinlock_paravirt.h |    2 +-
 kernel/sched/wait.c                 |    4 ++--
 18 files changed, 29 insertions(+), 33 deletions(-)
Documentation/memory-barriers.txt +3 -3
@@ -1662,7 +1662,7 @@ CPU from reordering them.
 
 There are some more advanced barrier functions:
 
- (*) set_mb(var, value)
+ (*) smp_store_mb(var, value)
 
      This assigns the value to the variable and then inserts a full memory
      barrier after it, depending on the function.  It isn't guaranteed to
@@ -1975,7 +1975,7 @@ A general memory barrier is interpolated
 	CPU 1
 	===============================
 	set_current_state();
-	  set_mb();
+	  smp_store_mb();
 	    STORE current->state
 	    <general barrier>
 	LOAD event_indicated
@@ -2016,7 +2016,7 @@ something up.  The barrier occurs before
 	CPU 1				CPU 2
 	===============================	===============================
 	set_current_state();		STORE event_indicated
-	  set_mb();			wake_up();
+	  smp_store_mb();		wake_up();
 	    STORE current->state	  <write barrier>
 	    <general barrier>		  STORE current->state
 	LOAD event_indicated
arch/arm/include/asm/barrier.h +1 -1
@@ -81,7 +81,7 @@ do {									\
 #define read_barrier_depends()		do { } while(0)
 #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
+#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_mb__before_atomic()	smp_mb()
 #define smp_mb__after_atomic()	smp_mb()
arch/arm64/include/asm/barrier.h +1 -1
@@ -114,7 +114,7 @@ do {									\
 #define read_barrier_depends()		do { } while(0)
 #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
+#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 #define nop()		asm volatile("nop");
 
 #define smp_mb__before_atomic()	smp_mb()
arch/ia64/include/asm/barrier.h +1 -6
@@ -77,12 +77,7 @@ do {									\
 	___p1;								\
 })
 
-/*
- * XXX check on this ---I suspect what Linus really wants here is
- * acquire vs release semantics but we can't discuss this stuff with
- * Linus just yet.  Grrr...
- */
-#define set_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
 
 /*
  * The group barrier in front of the rsm & ssm are necessary to ensure
arch/metag/include/asm/barrier.h +1 -1
@@ -84,7 +84,7 @@ static inline void fence(void)
 #define read_barrier_depends()		do { } while (0)
 #define smp_read_barrier_depends()	do { } while (0)
 
-#define set_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
+#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_store_release(p, v)						\
 do {									\
arch/mips/include/asm/barrier.h +1 -1
@@ -112,7 +112,7 @@
 #define __WEAK_LLSC_MB		"		\n"
 #endif
 
-#define set_mb(var, value) \
+#define smp_store_mb(var, value) \
 	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_llsc_mb()	__asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
arch/powerpc/include/asm/barrier.h +1 -1
@@ -34,7 +34,7 @@
 #define rmb()  __asm__ __volatile__ ("sync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("sync" : : : "memory")
 
-#define set_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value)	do { WRITE_ONCE(var, value); mb(); } while (0)
 
 #ifdef __SUBARCH_HAS_LWSYNC
 #    define SMPWMB      LWSYNC
arch/s390/include/asm/barrier.h +1 -1
@@ -36,7 +36,7 @@
 #define smp_mb__before_atomic()		smp_mb()
 #define smp_mb__after_atomic()		smp_mb()
 
-#define set_mb(var, value)		do { WRITE_ONCE(var, value); mb(); } while (0)
+#define smp_store_mb(var, value)		do { WRITE_ONCE(var, value); mb(); } while (0)
 
 #define smp_store_release(p, v)						\
 do {									\
arch/sh/include/asm/barrier.h +1 -1
@@ -32,7 +32,7 @@
 #define ctrl_barrier()	__asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop")
 #endif
 
-#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
+#define smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
 
 #include <asm-generic/barrier.h>
 
arch/sparc/include/asm/barrier_64.h +1 -1
@@ -40,7 +40,7 @@ do {	__asm__ __volatile__("ba,pt	%%xcc,
 #define dma_rmb()	rmb()
 #define dma_wmb()	wmb()
 
-#define set_mb(__var, __value) \
+#define smp_store_mb(__var, __value) \
 	do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0)
 
 #ifdef CONFIG_SMP
arch/x86/include/asm/barrier.h +2 -2
@@ -35,12 +35,12 @@
 #define smp_mb()	mb()
 #define smp_rmb()	dma_rmb()
 #define smp_wmb()	barrier()
-#define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
+#define smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
 #else /* !SMP */
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
-#define set_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
+#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
 #endif /* SMP */
 
 #define read_barrier_depends()		do { } while (0)
arch/x86/um/asm/barrier.h +2 -1
@@ -39,7 +39,8 @@
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
-#define set_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
+
+#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
 
 #define read_barrier_depends()		do { } while (0)
 #define smp_read_barrier_depends()	do { } while (0)
fs/select.c +3 -3
@@ -189,7 +189,7 @@ static int __pollwake(wait_queue_t *wait
 	 * doesn't imply write barrier and the users expect write
 	 * barrier semantics on wakeup functions.  The following
 	 * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up()
-	 * and is paired with set_mb() in poll_schedule_timeout.
+	 * and is paired with smp_store_mb() in poll_schedule_timeout.
 	 */
 	smp_wmb();
 	pwq->triggered = 1;
@@ -244,7 +244,7 @@ int poll_schedule_timeout(struct poll_wq
 	/*
 	 * Prepare for the next iteration.
 	 *
-	 * The following set_mb() serves two purposes.  First, it's
+	 * The following smp_store_mb() serves two purposes.  First, it's
 	 * the counterpart rmb of the wmb in pollwake() such that data
 	 * written before wake up is always visible after wake up.
 	 * Second, the full barrier guarantees that triggered clearing
@@ -252,7 +252,7 @@ int poll_schedule_timeout(struct poll_wq
 	 * this problem doesn't exist for the first iteration as
 	 * add_wait_queue() has full barrier semantics.
 	 */
-	set_mb(pwq->triggered, 0);
+	smp_store_mb(pwq->triggered, 0);
 
 	return rc;
 }
include/asm-generic/barrier.h +2 -2
@@ -66,8 +66,8 @@
 #define smp_read_barrier_depends()	do { } while (0)
 #endif
 
-#ifndef set_mb
-#define set_mb(var, value)  do { WRITE_ONCE(var, value); mb(); } while (0)
+#ifndef smp_store_mb
+#define smp_store_mb(var, value)  do { WRITE_ONCE(var, value); mb(); } while (0)
 #endif
 
 #ifndef smp_mb__before_atomic
include/linux/sched.h +4 -4
@@ -257,7 +257,7 @@ extern char ___assert_task_state[1 - 2*!
 #define set_task_state(tsk, state_value)			\
 	do {							\
 		(tsk)->task_state_change = _THIS_IP_;		\
-		set_mb((tsk)->state, (state_value));		\
+		smp_store_mb((tsk)->state, (state_value));		\
 	} while (0)
 
 /*
@@ -279,7 +279,7 @@ extern char ___assert_task_state[1 - 2*!
 #define set_current_state(state_value)				\
 	do {							\
 		current->task_state_change = _THIS_IP_;		\
-		set_mb(current->state, (state_value));		\
+		smp_store_mb(current->state, (state_value));		\
 	} while (0)
 
 #else
@@ -287,7 +287,7 @@ extern char ___assert_task_state[1 - 2*!
 #define __set_task_state(tsk, state_value)		\
 	do { (tsk)->state = (state_value); } while (0)
 #define set_task_state(tsk, state_value)		\
-	set_mb((tsk)->state, (state_value))
+	smp_store_mb((tsk)->state, (state_value))
 
 /*
  * set_current_state() includes a barrier so that the write of current->state
@@ -303,7 +303,7 @@ extern char ___assert_task_state[1 - 2*!
 #define __set_current_state(state_value)		\
 	do { current->state = (state_value); } while (0)
 #define set_current_state(state_value)			\
-	set_mb(current->state, (state_value))
+	smp_store_mb(current->state, (state_value))
 
 #endif
 
kernel/futex.c +1 -1
@@ -2056,7 +2056,7 @@ static void futex_wait_queue_me(struct f
 {
 	/*
 	 * The task state is guaranteed to be set before another task can
-	 * wake it. set_current_state() is implemented using set_mb() and
+	 * wake it. set_current_state() is implemented using smp_store_mb() and
 	 * queue_me() calls spin_unlock() upon completion, both serializing
 	 * access to the hash list and forcing another memory barrier.
 	 */
kernel/locking/qspinlock_paravirt.h +1 -1
@@ -175,7 +175,7 @@ static void pv_wait_node(struct mcs_spin
 		 *
 		 * Matches the xchg() from pv_kick_node().
 		 */
-		set_mb(pn->state, vcpu_halted);
+		smp_store_mb(pn->state, vcpu_halted);
 
 		if (!READ_ONCE(node->locked))
 			pv_wait(&pn->state, vcpu_halted);
kernel/sched/wait.c +2 -2
@@ -341,7 +341,7 @@ long wait_woken(wait_queue_t *wait, unsi
 	 * condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss
 	 * an event.
 	 */
-	set_mb(wait->flags, wait->flags & ~WQ_FLAG_WOKEN); /* B */
+	smp_store_mb(wait->flags, wait->flags & ~WQ_FLAG_WOKEN); /* B */
 
 	return timeout;
 }
@@ -354,7 +354,7 @@ int woken_wake_function(wait_queue_t *wa
 	 * doesn't imply write barrier and the users expects write
 	 * barrier semantics on wakeup functions.  The following
 	 * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up()
-	 * and is paired with set_mb() in wait_woken().
+	 * and is paired with smp_store_mb() in wait_woken().
 	 */
 	smp_wmb(); /* C */
 	wait->flags |= WQ_FLAG_WOKEN;
author maintainer
#7 lore raw
In reply to Peter Zijlstra

Review do { \ · line 87

Quoted 10 lines · Peter Zijlstra

+++ b/arch/arm/include/asm/barrier.h

(quoted diff) +1 -1
@@ -81,7 +81,7 @@ do {									\
 #define read_barrier_depends()		do { } while(0)
 #define smp_read_barrier_depends()	do { } while(0)
 
-#define set_mb(var, value)	do { var = value; smp_mb(); } while (0)
+#define set_mb(var, value)	do { WRITE_ONCE(var, value); smp_mb(); } while (0)
 
 #define smp_mb__before_atomic()	smp_mb()
 #define smp_mb__after_atomic()	smp_mb()

Which triggered a massive compile fail and requires the same union trickery we already had for READ_ONCE().

Added the below bit to the patch to aid compiling.

---

include/linux/compiler.h +1 -1
index a7c0941d10da..03e227ba481c 100644
@@ -250,7 +250,7 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
 	({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; })
 
 #define WRITE_ONCE(x, val) \
-	({ typeof(x) __val = (val); __write_once_size(&(x), &__val, sizeof(__val)); __val; })
+	({ union { typeof(x) __val; char __c[1]; } __u = { .__val = (val) }; __write_once_size(&(x), __u.__c, sizeof(x)); __u.__val; })
 
 #endif /* __KERNEL__ */