locking/pvqspinlock: Replace xchg() by the more descriptive set_mb()
Subsystems (16)
A message that is not in the archive (sent off-list or not yet received): [email protected]
Peter Zijlstra
maintainerReview
Quoted 2 lines · tip-bot for Waiman Long
(quoted diff) +1
-1- (void)xchg(&pn->state, vcpu_halted); + set_mb(pn->state, vcpu_halted);
Hmm, so I looked at the set_mb() definitions and I figure we want to do something like the below, right?
--- arch/arm/include/asm/barrier.h | 2 +- arch/arm64/include/asm/barrier.h | 2 +- arch/ia64/include/asm/barrier.h | 2 +- arch/metag/include/asm/barrier.h | 2 +- arch/mips/include/asm/barrier.h | 2 +- arch/powerpc/include/asm/barrier.h | 2 +- arch/s390/include/asm/barrier.h | 2 +- arch/sparc/include/asm/barrier_64.h | 2 +- arch/x86/include/asm/barrier.h | 2 +- include/asm-generic/barrier.h | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-)
arch/arm/include/asm/barrier.h +1 -1
@@ -81,7 +81,7 @@ do { \#define read_barrier_depends () do { } while(0)#define smp_read_barrier_depends () do { } while(0) -#define set_mb (var, value) do { var = value; smp_mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_mb__before_atomic () smp_mb()#define smp_mb__after_atomic () smp_mb()
arch/arm64/include/asm/barrier.h +1 -1
@@ -114,7 +114,7 @@ do { \#define read_barrier_depends () do { } while(0)#define smp_read_barrier_depends () do { } while(0) -#define set_mb (var, value) do { var = value; smp_mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define nop () asm volatile("nop");#define smp_mb__before_atomic () smp_mb()
arch/ia64/include/asm/barrier.h +1 -1
@@ -82,7 +82,7 @@ do { \* acquire vs release semantics but we can't discuss this stuff with * Linus just yet. Grrr... */ -#define set_mb (var, value) do { (var) = (value); mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)/* * The group barrier in front of the rsm & ssm are necessary to ensure
arch/metag/include/asm/barrier.h +1 -1
@@ -84,7 +84,7 @@ static inline void fence(void)#define read_barrier_depends () do { } while (0)#define smp_read_barrier_depends () do { } while (0) -#define set_mb (var, value) do { var = value; smp_mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_store_release (p, v) \ do { \
arch/mips/include/asm/barrier.h +1 -1
@@ -113,7 +113,7 @@#endif #define set_mb (var, value) \ - do { var = value; smp_mb(); } while (0) + do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_llsc_mb () __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
arch/powerpc/include/asm/barrier.h +1 -1
@@ -34,7 +34,7 @@#define rmb () __asm__ __volatile__ ("sync" : : : "memory")#define wmb () __asm__ __volatile__ ("sync" : : : "memory") -#define set_mb (var, value) do { var = value; mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#ifdef __SUBARCH_HAS_LWSYNC# define SMPWMB LWSYNC
arch/s390/include/asm/barrier.h +1 -1
@@ -36,7 +36,7 @@#define smp_mb__before_atomic () smp_mb()#define smp_mb__after_atomic () smp_mb() -#define set_mb (var, value) do { var = value; mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#define smp_store_release (p, v) \ do { \
arch/sparc/include/asm/barrier_64.h +1 -1
@@ -41,7 +41,7 @@ do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \#define dma_wmb () wmb()#define set_mb (__var, __value) \ - do { __var = __value; membar_safe("#StoreLoad"); } while(0) + do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0)#ifdef CONFIG_SMP #define smp_mb () mb()
arch/x86/include/asm/barrier.h +1 -1
@@ -40,7 +40,7 @@#define smp_mb () barrier()#define smp_rmb () barrier()#define smp_wmb () barrier() -#define set_mb (var, value) do { var = value; barrier(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)#endif /* SMP */ #define read_barrier_depends () do { } while (0)
include/asm-generic/barrier.h +1 -1
@@ -67,7 +67,7 @@#endif #ifndef set_mb -#define set_mb (var, value) do { (var) = (value); mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#endif #ifndef smp_mb__before_atomic
Quoted 5 lines · Peter Zijlstra
Review
Quoted 2 lines · tip-bot for Waiman Long
(quoted diff) +1
-1- (void)xchg(&pn->state, vcpu_halted); + set_mb(pn->state, vcpu_halted);Hmm, so I looked at the
set_mb()definitions and I figure we want to do something like the below, right?
Yes, I think we should do that just to be safe from unexpected compiler optimization.
Quoted 122 lines
--- arch/arm/include/asm/barrier.h | 2 +- arch/arm64/include/asm/barrier.h | 2 +- arch/ia64/include/asm/barrier.h | 2 +- arch/metag/include/asm/barrier.h | 2 +- arch/mips/include/asm/barrier.h | 2 +- arch/powerpc/include/asm/barrier.h | 2 +- arch/s390/include/asm/barrier.h | 2 +- arch/sparc/include/asm/barrier_64.h | 2 +- arch/x86/include/asm/barrier.h | 2 +- include/asm-generic/barrier.h | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-)arch/arm/include/asm/barrier.h +1
-1@@ -81,7 +81,7 @@ do { \#define read_barrier_depends () do { } while(0)#define smp_read_barrier_depends () do { } while(0) -#define set_mb (var, value) do { var = value; smp_mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_mb__before_atomic () smp_mb()#define smp_mb__after_atomic () smp_mb()arch/arm64/include/asm/barrier.h +1
-1@@ -114,7 +114,7 @@ do { \#define read_barrier_depends () do { } while(0)#define smp_read_barrier_depends () do { } while(0) -#define set_mb (var, value) do { var = value; smp_mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define nop () asm volatile("nop");#define smp_mb__before_atomic () smp_mb()arch/ia64/include/asm/barrier.h +1
-1@@ -82,7 +82,7 @@ do { \* acquire vs release semantics but we can't discuss this stuff with * Linus just yet. Grrr... */ -#define set_mb (var, value) do { (var) = (value); mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)/* * The group barrier in front of the rsm& ssm are necessary to ensure arch/metag/include/asm/barrier.h +1
-1@@ -84,7 +84,7 @@ static inline void fence(void)#define read_barrier_depends () do { } while (0)#define smp_read_barrier_depends () do { } while (0) -#define set_mb (var, value) do { var = value; smp_mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_store_release (p, v) \ do { \arch/mips/include/asm/barrier.h +1
-1@@ -113,7 +113,7 @@#endif #define set_mb (var, value) \ - do { var = value; smp_mb(); } while (0) + do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_llsc_mb () __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")arch/powerpc/include/asm/barrier.h +1
-1@@ -34,7 +34,7 @@#define rmb () __asm__ __volatile__ ("sync" : : : "memory")#define wmb () __asm__ __volatile__ ("sync" : : : "memory") -#define set_mb (var, value) do { var = value; mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#ifdef __SUBARCH_HAS_LWSYNC# define SMPWMB LWSYNCarch/s390/include/asm/barrier.h +1
-1@@ -36,7 +36,7 @@#define smp_mb__before_atomic () smp_mb()#define smp_mb__after_atomic () smp_mb() -#define set_mb (var, value) do { var = value; mb(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#define smp_store_release (p, v) \ do { \arch/sparc/include/asm/barrier_64.h +1
-1@@ -41,7 +41,7 @@ do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \#define dma_wmb () wmb()#define set_mb (__var, __value) \ - do { __var = __value; membar_safe("#StoreLoad"); } while(0) + do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0)#ifdef CONFIG_SMP #define smp_mb () mb()arch/x86/include/asm/barrier.h +0
-0
arch/x86/include/asm/barrier.h line 43
Quoted 5 lines
arch/x86/include/asm/barrier.h +1
-1@@ -40,7 +40,7 @@#define smp_mb () barrier()#define smp_rmb () barrier()#define smp_wmb () barrier() -#define set_mb (var, value) do { var = value; barrier(); } while (0) +#define set_mb (var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)
That part is in the !CONFIG_SMP portion. I don't think we need to change it.
Cheers,
Longman
Quoted 3 lines · Peter Zijlstra
Hmm, so I looked at the
set_mb()definitions and I figure we want to do something like the below, right?
I don't think you need to do this for the non-smp cases. The whole thing is about smp memory ordering, so on UP you don't even need the WRITE_ONCE(), much less a barrier.
That said, I do wonder if we should make that "it's only an smp barrier" more explicit. We have non-smp barriers for people who do DMA, and while they should probably never use anything like set_mb() anyway (they tend to want *release* semantics, not a full barrier), from a conceptual standpoint the "set_mb()" function really is closer to the "smp_store_release()/smp_load_acquire()" family of macros.
So I wonder if we should change the name to match.
IOW, if we are really cleaning up smp_mb() and changing most of the lines associated with it (we really have very few users, and there seems to be more lines *defining* smp_mb() than there are lines *using* it in the kernel), maybe we should also just rename it "smp_store_mb()" at the same time.
I dunno. Maybe the churn isn't worth it.
Linus
Peter Zijlstra
maintainerQuoted 6 lines · Linus Torvalds
Quoted 3 lines · Peter Zijlstra
Hmm, so I looked at the
set_mb()definitions and I figure we want to do something like the below, right?I don't think you need to do this for the non-smp cases.
Well, its the store tearing thing again, we use WRITE_ONCE() in smp_store_release() for the same reason. We want it to be a single store.
Quoted 3 lines
The whole
thing is about smp memory ordering, so on UP you don't even need theWRITE_ONCE(), much less a barrier.
No, we actually need both still on UP.
Imagine the following sequence:
for (;;) {
set_current_state(TASK_KILLABLE);
if (cond)
break;
schedule();
}
__set_current_state(TASK_RUNNING);vs
<IRQ> wake_up_process(p);
As we know, set_current_state() is set_mb(), and thus will look like:
current->state = TASK_KILLABLE; smp_mb(); if (cond) break;
So without the WRITE_ONCE() we can get store tearing, and suppose our compiler is insane and translates the store into 4 byte stores.
current->state[0] = TASK_UNINTERRUPTIBLE; current->state[1] = TASK_WAKEKILL >> 8; current->state[2] = 0; current->state[3] = 0;
The obvious fail here is to get the wakeup interrupt between [0] and [1].
current->state[0] = TASK_UNINTERRUPTIBLE; <IRQ> wake_up_process(p); p->state = TASK_RUNNING; current->state[1] = TASK_WAKEKILL >> 8; current->state[2] = 0; current->state[3] = 0;
With the end result that ->state == TASK_WAKEKILL, from which we'll not wake up unless killed.
Similarly, without the barrier(), our friendly compiler is allowed to do:
if (cond) break current->state = TASK_KILLABLE; schedule();
Which we all know to be broken.
So no, set_mb() (or smp_store_mb()) very much does need the WRITE_ONCE() and a barrier() on UP.
Peter Zijlstra
maintainerQuoted 15 lines · Peter Zijlstra
Quoted 6 lines · Linus Torvalds
Quoted 3 lines · Peter Zijlstra
Hmm, so I looked at the
set_mb()definitions and I figure we want to do something like the below, right?I don't think you need to do this for the non-smp cases.
Well, its the store tearing thing again, we use
WRITE_ONCE()insmp_store_release()for the same reason. We want it to be a single store.Quoted 3 lines
The whole
thing is about smp memory ordering, so on UP you don't even need theWRITE_ONCE(), much less a barrier.
Ah, you meant the memory barrier; indeed, a compiler barrier is sufficient. I got somewhat confused between Waiman's email and barrier and barrier() (again!).
Peter Zijlstra
maintainerQuoted 5 lines · Linus Torvalds
from a conceptual standpoint the "
set_mb()" function really is closer to the "smp_store_release()/smp_load_acquire()" family of macros.So I wonder if we should change the name to match.
Quoted 7 lines
IOW, if we are really cleaning up
smp_mb()and changing most of the lines associated with it (we really have very few users, and there seems to be more lines *defining*smp_mb()than there are lines *using* it in the kernel), maybe we should also just rename it "smp_store_mb()" at the same time.I dunno. Maybe the churn isn't worth it.
Can't hurt, and its a trivial patch to generate.
git grep -l "\<set_mb\>" | while read file; do sed -i -e 's/\<set_mb\>/smp_store_mb/g' $file; done
And a manual edit later...
--- Documentation/memory-barriers.txt | 6 +++--- arch/arm/include/asm/barrier.h | 2 +- arch/arm64/include/asm/barrier.h | 2 +- arch/ia64/include/asm/barrier.h | 7 +------ arch/metag/include/asm/barrier.h | 2 +- arch/mips/include/asm/barrier.h | 2 +- arch/powerpc/include/asm/barrier.h | 2 +- arch/s390/include/asm/barrier.h | 2 +- arch/sh/include/asm/barrier.h | 2 +- arch/sparc/include/asm/barrier_64.h | 2 +- arch/x86/include/asm/barrier.h | 4 ++-- arch/x86/um/asm/barrier.h | 3 ++- fs/select.c | 6 +++--- include/asm-generic/barrier.h | 4 ++-- include/linux/sched.h | 8 ++++---- kernel/futex.c | 2 +- kernel/locking/qspinlock_paravirt.h | 2 +- kernel/sched/wait.c | 4 ++-- 18 files changed, 29 insertions(+), 33 deletions(-)
Documentation/memory-barriers.txt +3 -3
@@ -1662,7 +1662,7 @@ CPU from reordering them. There are some more advanced barrier functions: - (*) set_mb(var, value) + (*) smp_store_mb(var, value) This assigns the value to the variable and then inserts a full memory barrier after it, depending on the function. It isn't guaranteed to
@@ -1975,7 +1975,7 @@ A general memory barrier is interpolated CPU 1 =============================== set_current_state(); - set_mb(); + smp_store_mb(); STORE current->state <general barrier> LOAD event_indicated
@@ -2016,7 +2016,7 @@ something up. The barrier occurs before CPU 1 CPU 2 =============================== =============================== set_current_state(); STORE event_indicated - set_mb(); wake_up(); + smp_store_mb(); wake_up(); STORE current->state <write barrier> <general barrier> STORE current->state LOAD event_indicated
arch/arm/include/asm/barrier.h +1 -1
@@ -81,7 +81,7 @@ do { \#define read_barrier_depends () do { } while(0)#define smp_read_barrier_depends () do { } while(0) -#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_mb__before_atomic () smp_mb()#define smp_mb__after_atomic () smp_mb()
arch/arm64/include/asm/barrier.h +1 -1
@@ -114,7 +114,7 @@ do { \#define read_barrier_depends () do { } while(0)#define smp_read_barrier_depends () do { } while(0) -#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define nop () asm volatile("nop");#define smp_mb__before_atomic () smp_mb()
arch/ia64/include/asm/barrier.h +1 -6
@@ -77,12 +77,7 @@ do { \ ___p1; \ }) -/* -* XXX check on this ---I suspect what Linus really wants here is -* acquire vs release semantics but we can't discuss this stuff with -* Linus just yet. Grrr... -*/ -#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)/* * The group barrier in front of the rsm & ssm are necessary to ensure
arch/metag/include/asm/barrier.h +1 -1
@@ -84,7 +84,7 @@ static inline void fence(void)#define read_barrier_depends () do { } while (0)#define smp_read_barrier_depends () do { } while (0) -#define set_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_store_release (p, v) \ do { \
arch/mips/include/asm/barrier.h +1 -1
@@ -112,7 +112,7 @@#define __WEAK_LLSC_MB " \n"#endif -#define set_mb (var, value) \ +#define smp_store_mb (var, value) \ do { WRITE_ONCE(var, value); smp_mb(); } while (0)#define smp_llsc_mb () __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory")
arch/powerpc/include/asm/barrier.h +1 -1
@@ -34,7 +34,7 @@#define rmb () __asm__ __volatile__ ("sync" : : : "memory")#define wmb () __asm__ __volatile__ ("sync" : : : "memory") -#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#ifdef __SUBARCH_HAS_LWSYNC# define SMPWMB LWSYNC
arch/s390/include/asm/barrier.h +1 -1
@@ -36,7 +36,7 @@#define smp_mb__before_atomic () smp_mb()#define smp_mb__after_atomic () smp_mb() -#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#define smp_store_release (p, v) \ do { \
arch/sh/include/asm/barrier.h +1 -1
@@ -32,7 +32,7 @@#define ctrl_barrier () __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop")#endif -#define set_mb (var, value) do { (void)xchg(&var, value); } while (0) +#define smp_store_mb (var, value) do { (void)xchg(&var, value); } while (0)#include <asm-generic/barrier.h>
arch/sparc/include/asm/barrier_64.h +1 -1
@@ -40,7 +40,7 @@ do { __asm__ __volatile__("ba,pt %%xcc,#define dma_rmb () rmb()#define dma_wmb () wmb() -#define set_mb (__var, __value) \ +#define smp_store_mb (__var, __value) \ do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0)#ifdef CONFIG_SMP
arch/x86/include/asm/barrier.h +2 -2
@@ -35,12 +35,12 @@#define smp_mb () mb()#define smp_rmb () dma_rmb()#define smp_wmb () barrier() -#define set_mb (var, value) do { (void)xchg(&var, value); } while (0) +#define smp_store_mb (var, value) do { (void)xchg(&var, value); } while (0)#else /* !SMP */ #define smp_mb () barrier()#define smp_rmb () barrier()#define smp_wmb () barrier() -#define set_mb (var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)#endif /* SMP */ #define read_barrier_depends () do { } while (0)
arch/x86/um/asm/barrier.h +2 -1
@@ -39,7 +39,8 @@#define smp_mb () barrier()#define smp_rmb () barrier()#define smp_wmb () barrier() -#define set_mb (var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) + +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); barrier(); } while (0)#define read_barrier_depends () do { } while (0)#define smp_read_barrier_depends () do { } while (0)
fs/select.c +3 -3
@@ -189,7 +189,7 @@ static int __pollwake(wait_queue_t *wait* doesn't imply write barrier and the users expect write * barrier semantics on wakeup functions. The following * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up() -* and is paired with set_mb() in poll_schedule_timeout. +* and is paired with smp_store_mb() in poll_schedule_timeout. */ smp_wmb (); pwq->triggered =1 ;
@@ -244,7 +244,7 @@ int poll_schedule_timeout(struct poll_wq/* * Prepare for the next iteration. * -* The following set_mb() serves two purposes. First, it's +* The following smp_store_mb() serves two purposes. First, it's * the counterpart rmb of the wmb in pollwake() such that data * written before wake up is always visible after wake up. * Second, the full barrier guarantees that triggered clearing
@@ -252,7 +252,7 @@ int poll_schedule_timeout(struct poll_wq* this problem doesn't exist for the first iteration as * add_wait_queue() has full barrier semantics. */ -set_mb (pwq->triggered,0 ); +smp_store_mb (pwq->triggered,0 );return rc; }
include/asm-generic/barrier.h +2 -2
@@ -66,8 +66,8 @@#define smp_read_barrier_depends () do { } while (0)#endif -#ifndef set_mb -#define set_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0) +#ifndef smp_store_mb +#define smp_store_mb (var, value) do { WRITE_ONCE(var, value); mb(); } while (0)#endif #ifndef smp_mb__before_atomic
include/linux/sched.h +4 -4
@@ -257,7 +257,7 @@ extern char ___assert_task_state[1 - 2*!#define set_task_state (tsk, state_value) \ do { \ (tsk)->task_state_change = _THIS_IP_; \ - set_mb((tsk)->state, (state_value)); \ + smp_store_mb((tsk)->state, (state_value)); \ } while (0)/*
@@ -279,7 +279,7 @@ extern char ___assert_task_state[1 - 2*!#define set_current_state (state_value) \ do { \ current->task_state_change = _THIS_IP_; \ - set_mb(current->state, (state_value)); \ + smp_store_mb(current->state, (state_value)); \ } while (0)#else
@@ -287,7 +287,7 @@ extern char ___assert_task_state[1 - 2*!#define __set_task_state (tsk, state_value) \ do { (tsk)->state = (state_value); } while (0)#define set_task_state (tsk, state_value) \ - set_mb((tsk)->state, (state_value)) + smp_store_mb((tsk)->state, (state_value))/* * set_current_state() includes a barrier so that the write of current->state
@@ -303,7 +303,7 @@ extern char ___assert_task_state[1 - 2*!#define __set_current_state (state_value) \ do { current->state = (state_value); } while (0)#define set_current_state (state_value) \ - set_mb(current->state, (state_value)) + smp_store_mb(current->state, (state_value))#endif
kernel/futex.c +1 -1
@@ -2056,7 +2056,7 @@ static void futex_wait_queue_me(struct f {/* * The task state is guaranteed to be set before another task can -* wake it. set_current_state() is implemented using set_mb() and +* wake it. set_current_state() is implemented using smp_store_mb() and * queue_me() calls spin_unlock() upon completion, both serializing * access to the hash list and forcing another memory barrier. */
kernel/locking/qspinlock_paravirt.h +1 -1
@@ -175,7 +175,7 @@ static void pv_wait_node(struct mcs_spin* * Matches the xchg() from pv_kick_node(). */ -set_mb (pn->state, vcpu_halted); +smp_store_mb (pn->state, vcpu_halted);if (!READ_ONCE (node->locked))pv_wait (&pn->state, vcpu_halted);
kernel/sched/wait.c +2 -2
@@ -341,7 +341,7 @@ long wait_woken(wait_queue_t *wait, unsi* condition being true _OR_ WQ_FLAG_WOKEN such that we will not miss * an event. */ -set_mb (wait->flags, wait->flags & ~WQ_FLAG_WOKEN );/* B */ +smp_store_mb (wait->flags, wait->flags & ~WQ_FLAG_WOKEN );/* B */ return timeout; }
@@ -354,7 +354,7 @@ int woken_wake_function(wait_queue_t *wa* doesn't imply write barrier and the users expects write * barrier semantics on wakeup functions. The following * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up() -* and is paired with set_mb() in wait_woken(). +* and is paired with smp_store_mb() in wait_woken(). */ smp_wmb ();/* C */ wait->flags |=WQ_FLAG_WOKEN ;
Peter Zijlstra
maintainerReview do { \ · line 87
Quoted 10 lines · Peter Zijlstra
+++
b/arch/arm/include/asm/barrier.h(quoted diff) +1
-1@@ -81,7 +81,7 @@ do { \ #define read_barrier_depends() do { } while(0) #define smp_read_barrier_depends() do { } while(0) -#define set_mb(var, value) do { var = value; smp_mb(); } while (0) +#define set_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) #define smp_mb__before_atomic() smp_mb() #define smp_mb__after_atomic() smp_mb()
Which triggered a massive compile fail and requires the same union trickery we already had for READ_ONCE().
Added the below bit to the patch to aid compiling.
---
include/linux/compiler.h +1 -1
@@ -250,7 +250,7 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; })#define WRITE_ONCE (x, val) \ - ({ typeof(x) __val = (val); __write_once_size(&(x), &__val, sizeof(__val)); __val; }) + ({ union { typeof(x) __val; char __c[1]; } __u = { .__val = (val) }; __write_once_size(&(x), __u.__c, sizeof(x)); __u.__val; })#endif /* __KERNEL__ */