Remove spin_unlock_wait()
Subsystems (18)
cover Remove spin_unlock_wait()
Hello!
This series removes spin_unlock_wait():
1. Remove spin_unlock_wait() from net/netfilter/nf_conntrack_core,
courtesy of Manfred Spraul.
2. Replace task_work use of spin_unlock_wait() with lock/unlock
pair, courtesy of Oleg Nesterov.
3. Replace scheduler use of spin_unlock_wait() with lock/unlock pair.
4. Replace completion use of spin_unlock_wait() with lock/unlock
pair.
5. Replace exit() use of spin_unlock_wait() with lock/unlock pair.
6. Replace IPC use of spin_unlock_wait() with lock/unlock pair.
7. Replace ATA use of spin_unlock_wait() with lock/unlock pair.
8. Remove generic definition of spin_unlock_wait().
9. Remove architecture-specific definitions of spin_unlock_wait().
Thanx, Paul
------------------------------------------------------------------------
arch/alpha/include/asm/spinlock.h | 5 - arch/arc/include/asm/spinlock.h | 5 - arch/arm/include/asm/spinlock.h | 16 ---- arch/arm64/include/asm/spinlock.h | 58 +---------------- arch/blackfin/include/asm/spinlock.h | 5 - arch/hexagon/include/asm/spinlock.h | 5 - arch/ia64/include/asm/spinlock.h | 21 ------ arch/m32r/include/asm/spinlock.h | 5 - arch/metag/include/asm/spinlock.h | 5 - arch/mn10300/include/asm/spinlock.h | 5 - arch/parisc/include/asm/spinlock.h | 7 -- arch/powerpc/include/asm/spinlock.h | 33 --------- arch/s390/include/asm/spinlock.h | 7 -- arch/sh/include/asm/spinlock-cas.h | 5 - arch/sh/include/asm/spinlock-llsc.h | 5 - arch/sparc/include/asm/spinlock_32.h | 5 - arch/tile/include/asm/spinlock_32.h | 2 arch/tile/include/asm/spinlock_64.h | 2 arch/tile/lib/spinlock_32.c | 23 ------ arch/tile/lib/spinlock_64.c | 22 ------ arch/xtensa/include/asm/spinlock.h | 5 - drivers/ata/libata-eh.c | 8 -- include/asm-generic/qspinlock.h | 14 ---- include/linux/spinlock.h | 31 --------- include/linux/spinlock_up.h | 6 - ipc/sem.c | 3 kernel/exit.c | 3 kernel/locking/qspinlock.c | 117 ----------------------------------- kernel/sched/completion.c | 9 -- kernel/sched/core.c | 5 - kernel/task_work.c | 8 -- net/netfilter/nf_conntrack_core.c | 52 ++++++++------- 32 files changed, 48 insertions(+), 454 deletions(-)
PATCH 1/9 net/netfilter/nf_conntrack_core: Fix net_conntrack_lock()
From: Manfred Spraul <[email protected]>
As we want to remove spin_unlock_wait() and replace it with explicit spin_lock()/spin_unlock() calls, we can use this to simplify the locking.
In addition:
- Reading nf_conntrack_locks_all needs ACQUIRE memory ordering.
- The new code avoids the backwards loop.
Only slightly tested, I did not manage to trigger calls to nf_conntrack_all_lock().
V2: With improved comments, to clearly show how the barriers
pair.
--- net/netfilter/nf_conntrack_core.c | 52 ++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 23 deletions(-)
net/netfilter/nf_conntrack_core.c +29 -23
@@ -96,19 +96,26 @@ static struct conntrack_gc_work conntrack_gc_work;void nf_conntrack_lock (spinlock_t *lock)__acquires (lock) { +/* 1) Acquire the lock */ spin_lock (lock); -while (unlikely (nf_conntrack_locks_all)) { -spin_unlock (lock); -/* -* Order the 'nf_conntrack_locks_all' load vs. the -* spin_unlock_wait() loads below, to ensure -* that 'nf_conntrack_locks_all_lock' is indeed held: -*/ -smp_rmb ();/* spin_lock(&nf_conntrack_locks_all_lock) */ -spin_unlock_wait (&nf_conntrack_locks_all_lock); -spin_lock (lock); - } +/* 2) read nf_conntrack_locks_all, with ACQUIRE semantics +* It pairs with the smp_store_release() in nf_conntrack_all_unlock() +*/ +if (likely (smp_load_acquire (&nf_conntrack_locks_all) == false)) +return ; + +/* fast path failed, unlock */ +spin_unlock (lock); + +/* Slow path 1) get global lock */ +spin_lock (&nf_conntrack_locks_all_lock); + +/* Slow path 2) get the lock we want */ +spin_lock (lock); + +/* Slow path 3) release the global lock */ +spin_unlock (&nf_conntrack_locks_all_lock); }EXPORT_SYMBOL_GPL (nf_conntrack_lock);
@@ -149,28 +156,27 @@ static void nf_conntrack_all_lock(void)int i;spin_lock (&nf_conntrack_locks_all_lock); - nf_conntrack_locks_all = true; -/* -* Order the above store of 'nf_conntrack_locks_all' against -* the spin_unlock_wait() loads below, such that if -* nf_conntrack_lock() observes 'nf_conntrack_locks_all' -* we must observe nf_conntrack_locks[] held: -*/ -smp_mb ();/* spin_lock(&nf_conntrack_locks_all_lock) */ + nf_conntrack_locks_all = true;for (i =0 ; i <CONNTRACK_LOCKS ; i++) { -spin_unlock_wait (&nf_conntrack_locks[i]); +spin_lock (&nf_conntrack_locks[i]); + +/* This spin_unlock provides the "release" to ensure that +* nf_conntrack_locks_all==true is visible to everyone that +* acquired spin_lock(&nf_conntrack_locks[]). +*/ +spin_unlock (&nf_conntrack_locks[i]); } }static void nf_conntrack_all_unlock (void ) { -/* -* All prior stores must be complete before we clear +/* All prior stores must be complete before we clear * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock() * might observe the false value but not the entire -* critical section: +* critical section. +* It pairs with the smp_load_acquire() in nf_conntrack_lock() */ smp_store_release (&nf_conntrack_locks_all, false);spin_unlock (&nf_conntrack_locks_all_lock);
Signature · 1 line
2.5.2
PATCH 2/9 task_work: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyFrom: Oleg Nesterov <[email protected]>
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in task_work_run() with a spin_lock_irq() and a spin_unlock_irq() aruond the cmpxchg() dequeue loop. This should be safe from a performance perspective because ->pi_lock is local to the task and because calls to the other side of the race, task_work_cancel(), should be rare.
--- kernel/task_work.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
kernel/task_work.c +2 -6
@@ -96,20 +96,16 @@ void task_work_run(void)* work->func() can do task_work_add(), do not set * work_exited unless the list is empty. */ +raw_spin_lock_irq (&task->pi_lock);do { work =READ_ONCE (task->task_works); head = !work && (task->flags &PF_EXITING ) ? &work_exited :NULL ; }while (cmpxchg (&task->task_works, work, head) != work); +raw_spin_unlock_irq (&task->pi_lock);if (!work)break ; -/* -* Synchronize with task_work_cancel(). It can't remove -* the first entry == work, cmpxchg(task_works) should -* fail, but it can play with *work and other entries. -*/ -raw_spin_unlock_wait (&task->pi_lock); do { next = work->next;
Signature · 1 line
2.5.2
PATCH 3/9 sched: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in do_task_dead() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock is this tasks ->pi_lock, and this is called only after the task exits.
[ paulmck: Replace leading smp_mb() with smp_mb__before_spinlock(),
courtesy of Arnd Bergmann's noting its odd location. ]
--- kernel/sched/core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
kernel/sched/core.c +3 -2
@@ -3352,8 +3352,9 @@ void __noreturn do_task_dead(void)* To avoid it, we have to wait for releasing tsk->pi_lock which * is held by try_to_wake_up() */ -smp_mb (); -raw_spin_unlock_wait (¤t->pi_lock); +smp_mb__before_spinlock (); +raw_spin_lock_irq (¤t->pi_lock); +raw_spin_unlock_irq (¤t->pi_lock);/* Causes final put_task_struct in finish_task_switch(): */ __set_current_state (TASK_DEAD );
Signature · 1 line
2.5.2
PATCH 4/9 completion: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in completion_done() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.
--- kernel/sched/completion.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
kernel/sched/completion.c +2 -7
@@ -307,14 +307,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe -* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock); +spin_lock_irq (&x->wait.lock); +spin_unlock_irq (&x->wait.lock);return true; }EXPORT_SYMBOL (completion_done);
Signature · 1 line
2.5.2
PATCH v5 4/9 completion: Replace spin_unlock_wait() with lock/unlock pair
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in completion_done() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.
kernel/sched/completion.c +4 -7
@@ -300,6 +300,8 @@ EXPORT_SYMBOL(try_wait_for_completion);*/ bool completion_done (struct completion *x) { +unsigned long flags; +if (!READ_ONCE (x->done))return false;
@@ -307,14 +309,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe -* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock); +spin_lock_irqsave (&x->wait.lock, flags); +spin_unlock_irqrestore (&x->wait.lock, flags);return true; }EXPORT_SYMBOL (completion_done);
Re: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
Quoted 36 lines · On Tue, 15 Aug 2017 09:16:29 -0700 "Paul E. McKenney"…
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the
spin_unlock_wait()call incompletion_done()withspin_lock()followed immediately byspin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.Signed-off-by Paul E. McKenney <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Will Deacon <[email protected]>Cc Alan Stern <[email protected]>Cc Andrea Parri <[email protected]>Cc Linus Torvalds <[email protected]>[ paulmck: Updated to use irqsave based on 0day Test Robot feedback. ]kernel/sched/completion.c +2
-3@@ -300,6 +300,8 @@ EXPORT_SYMBOL(try_wait_for_completion);*/ bool completion_done (struct completion *x) { +unsigned long flags; +if (!READ_ONCE (x->done))return false;@@ -307,14 +309,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe
kernel/sched/completion.c bool completion_done(struct completion *x) · line 317
Quoted 10 lines
kernel/sched/completion.c +2
-4-* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock); +spin_lock_irqsave (&x->wait.lock, flags); +spin_unlock_irqrestore (&x->wait.lock, flags);return true; }EXPORT_SYMBOL (completion_done);
For this patch:
But I was looking at this function, and it is a little worrisome, as it says it should return false if there are waiters and true otherwise. But it can also return false if there are no waiters and the completion is already done.
Basically we have:
wait_for_completion() {
while (!done)
wait();
done--;
}
complete() {
done++;
wake_up_waiters();
}Thus, completion_done() only returns true if a complete happened and a wait_for_completion has not. It does not return true if the complete has not yet occurred, but there are still waiters.
I looked at a couple of use cases, and this does not appear to be an issue, but the documentation about the completion_done() does not exactly fit the implementation. Should that be addressed?
Also, if complete_all() is called, then reinit_completion() must be called before that completion is used. The reinit_completion() has a comment stating this, but there's no comment by complete_all() stating this, which is where it really should be. I'll send a patch to fix this one.
-- Steve
Paul E. McKenney
AppliedRe: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
Quoted 52 lines · Steven Rostedt
Quoted 36 lines · On Tue, 15 Aug 2017 09:16:29 -0700 "Paul E. McKenney"…
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the
spin_unlock_wait()call incompletion_done()withspin_lock()followed immediately byspin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.Signed-off-by Paul E. McKenney <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Will Deacon <[email protected]>Cc Alan Stern <[email protected]>Cc Andrea Parri <[email protected]>Cc Linus Torvalds <[email protected]>[ paulmck: Updated to use irqsave based on 0day Test Robot feedback. ]kernel/sched/completion.c +2
-3@@ -300,6 +300,8 @@ EXPORT_SYMBOL(try_wait_for_completion);*/ bool completion_done (struct completion *x) { +unsigned long flags; +if (!READ_ONCE (x->done))return false;@@ -307,14 +309,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe kernel/sched/completion.c bool completion_done(struct completion *x) · line 317
Quoted 10 lines
kernel/sched/completion.c +2
-4-* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock); +spin_lock_irqsave (&x->wait.lock, flags); +spin_unlock_irqrestore (&x->wait.lock, flags);return true; }EXPORT_SYMBOL (completion_done);For this patch:
Quoted 1 line
Reviewed-by Steven Rostedt (VMware) <[email protected]>
Applied, thank you!
Quoted 26 lines
But I was looking at this function, and it is a little worrisome, as it says it should return false if there are waiters and true otherwise. But it can also return false if there are no waiters and the completion is already done.
Basically we have:
wait_for_completion() { while (!done) wait(); done--; } complete() { done++; wake_up_waiters(); }Thus,
completion_done()only returns true if a complete happened and await_for_completionhas not. It does not return true if the complete has not yet occurred, but there are still waiters.I looked at a couple of use cases, and this does not appear to be an issue, but the documentation about the
completion_done()does not exactly fit the implementation. Should that be addressed?
Quoted 5 lines
Also, if
complete_all()is called, thenreinit_completion()must be called before that completion is used. Thereinit_completion()has a comment stating this, but there's no comment bycomplete_all()stating this, which is where it really should be. I'll send a patch to fix this one.
But I am too late to return the favor -- good patch, though!
Thanx, Paul
Re: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyQuoted 36 lines · Paul E. McKenney
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the
spin_unlock_wait()call incompletion_done()withspin_lock()followed immediately byspin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.Signed-off-by Paul E. McKenney <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Will Deacon <[email protected]>Cc Alan Stern <[email protected]>Cc Andrea Parri <[email protected]>Cc Linus Torvalds <[email protected]>[ paulmck: Updated to use irqsave based on 0day Test Robot feedback. ]kernel/sched/completion.c +2
-3@@ -300,6 +300,8 @@ EXPORT_SYMBOL(try_wait_for_completion);*/ bool completion_done (struct completion *x) { +unsigned long flags; +if (!READ_ONCE (x->done))return false;@@ -307,14 +309,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe
kernel/sched/completion.c bool completion_done(struct completion *x) · line 317
Quoted 10 lines
kernel/sched/completion.c +2
-4-* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock); +spin_lock_irqsave (&x->wait.lock, flags); +spin_unlock_irqrestore (&x->wait.lock, flags);return true; }EXPORT_SYMBOL (completion_done);
I'm fine with this patch - as long as there are no performance regression reports. (which I suspect there won't be.)
Would you like to carry this in the RCU tree, due to other changes depending on this change - or can I pick this up into the scheduler tree?
Thanks,
Ingo
Re: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
Quoted 44 lines · Ingo Molnar
Quoted 41 lines · Paul E. McKenney
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the
spin_unlock_wait()call incompletion_done()withspin_lock()followed immediately byspin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.Signed-off-by Paul E. McKenney <[email protected]>Cc Ingo Molnar <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Will Deacon <[email protected]>Cc Alan Stern <[email protected]>Cc Andrea Parri <[email protected]>Cc Linus Torvalds <[email protected]>[ paulmck: Updated to use irqsave based on 0day Test Robot feedback. ]kernel/sched/completion.c +2
-7@@ -300,6 +300,8 @@ EXPORT_SYMBOL(try_wait_for_completion);*/ bool completion_done (struct completion *x) { +unsigned long flags; +if (!READ_ONCE (x->done))return false;@@ -307,14 +309,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe -* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock);
Quoted 8 lines
kernel/sched/completion.c bool completion_done(struct completion *x) · line 317
Quoted 5 lines
kernel/sched/completion.c +2
-0+spin_lock_irqsave (&x->wait.lock, flags); +spin_unlock_irqrestore (&x->wait.lock, flags);return true; }EXPORT_SYMBOL (completion_done);I'm fine with this patch - as long as there are no performance regression reports. (which I suspect there won't be.)
If there is, 0day Test Robot hasn't spotted it, nor has anyone else reported it.
Quoted 2 lines
Would you like to carry this in the RCU tree, due to other changes depending on this change - or can I pick this up into the scheduler tree?
Timely question! ;-)
My current plan is to send you a pull request like the following later today, Pacific Time (but rebased adding Steve Rostedt's Reviewed-by). This patch is on one of the branches, currently v4.13-rc2..93d8d7a12090 ("arch: Remove spin_unlock_wait() arch-specific definitions") in my -rcu tree.
Ah, and v4.13-rc2..7391304c4959 ("membarrier: Expedited private command") is mostly outside of RCU as well.
Since I will be rebasing and remerging anyway, if you would prefer that I split the spin_unlock_wait() and/or misc branches out, I am happy to do so. If I don't hear otherwise, though, I will send all seven branches using my usual approach.
So, if you want something different than my usual approach, please just let me know!
Thanx, Paul
PS. At the moment, there are no code changes to be applied, just
Steve's Reviewed-by.
------------------------------------------------------------------------
Prototype pull request
------------------------------------------------------------------------
The following changes since commit 520eccdfe187591a51ea9ab4c1a024ae4d0f68d9:
Linux 4.13-rc2 (2017-07-23 16:15:17 -0700)
are available in the git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git 20f2d43c114bf57d16580a758120cc65aa991bea
for you to fetch changes up to 20f2d43c114bf57d16580a758120cc65aa991bea:
Merge branches 'doc.2017.07.24c', 'fixes.2017.08.09a', 'hotplug.2017.07.25b', 'misc.2017.08.10a', 'spin_unlock_wait_no.2017.08.11a', 'srcu.2017.07.27c' and 'torture.2017.07.24c' into HEAD (2017-08-11 13:15:44 -0700)
----------------------------------------------------------------
Joe Perches (1):
module: Fix pr_fmt() bug for header use of printk
Luis R. Rodriguez (2):
swait: add idle variants which don't contribute to load average
rcu: use idle versions of swait to make idle-hack clear
Manfred Spraul (1):
net/netfilter/nf_conntrack_core: Fix net_conntrack_lock()
Masami Hiramatsu (1):
rcu/tracing: Set disable_rcu_irq_enter on rcu_eqs_exit()
Mathieu Desnoyers (1):
membarrier: Expedited private command
Oleg Nesterov (1):
task_work: Replace spin_unlock_wait() with lock/unlock pair
Paul E. McKenney (56):
documentation: Fix relation between nohz_full and rcu_nocbs
doc: RCU documentation update
doc: Update memory-barriers.txt for read-to-write dependencies
doc: Add RCU files to docbook-generation files
doc: No longer allowed to use rcu_dereference on non-pointers
doc: Set down RCU's scheduling-clock-interrupt needs
init_task: Remove redundant INIT_TASK_RCU_TREE_PREEMPT() macro
srcu: Move rcu_scheduler_starting() from Tiny RCU to Tiny SRCU
rcutorture: Remove obsolete SRCU-C.boot
srcu: Make process_srcu() be static
rcutorture: Move SRCU status printing to SRCU implementations
rcutorture: Print SRCU lock/unlock totals
rcu: Remove CONFIG_TASKS_RCU ifdef from rcuperf.c
rcutorture: Select CONFIG_PROVE_LOCKING for Tiny SRCU scenario
torture: Add --kconfig argument to kvm.sh
rcutorture: Don't wait for kernel when all builds fail
rcutorture: Enable SRCU readers from timer handler
rcutorture: Place event-traced strings into trace buffer
rcutorture: Use nr_cpus rather than maxcpus to limit test size
rcutorture: Add task's CPU for rcutorture writer stalls
rcutorture: Eliminate unused ts_rem local from rcu_trace_clock_local()
rcu: Add last-CPU to GP-kthread starvation messages
rcutorture: Invoke call_rcu() from timer handler
rcu: Use timer as backstop for NOCB deferred wakeups
atomics: Revert addition of comment header to spin_unlock_wait()
rcu: Migrate callbacks earlier in the CPU-offline timeline
rcu: Make expedited GPs correctly handle hardware CPU insertion
torture: Fix typo suppressing CPU-hotplug statistics
rcu: Remove orphan/adopt event-tracing fields
rcu: Check for NOCB CPUs and empty lists earlier in CB migration
rcu: Make NOCB CPUs migrate CBs directly from outgoing CPU
rcu: Advance outgoing CPU's callbacks before migrating them
rcu: Eliminate rcu_state ->orphan_lock
rcu: Advance callbacks after migration
rcu: Localize rcu_state ->orphan_pend and ->orphan_done
rcu: Remove unused RCU list functions
rcu: Move callback-list warning to irq-disable region
srcu: Provide ordering for CPU not involved in grace period
sched,rcu: Make cond_resched() provide RCU quiescent state
rcu: Drive TASKS_RCU directly off of PREEMPT
rcu: Create reasonable API for do_exit() TASKS_RCU processing
rcu: Add TPS() to event-traced strings
rcu: Move rcu.h to new trivial-function style
rcu: Add event tracing to ->gp_tasks update at GP start
rcu: Add TPS() protection for _rcu_barrier_trace strings
rcu: Add assertions verifying blocked-tasks list
rcu: Add warning to rcu_idle_enter() for irqs enabled
rcu: Remove exports from rcu_idle_exit() and rcu_idle_enter()
sched: Replace spin_unlock_wait() with lock/unlock pair
completion: Replace spin_unlock_wait() with lock/unlock pair
exit: Replace spin_unlock_wait() with lock/unlock pair
ipc: Replace spin_unlock_wait() with lock/unlock pair
drivers/ata: Replace spin_unlock_wait() with lock/unlock pair
locking: Remove spin_unlock_wait() generic definitions
arch: Remove spin_unlock_wait() arch-specific definitions
Merge branches 'doc.2017.07.24c', 'fixes.2017.08.09a', 'hotplug.2017.07.25b', 'misc.2017.08.10a', 'spin_unlock_wait_no.2017.08.11a', 'srcu.2017.07.27c' and 'torture.2017.07.24c' into HEAD
Peter Zijlstra (Intel) (1):
rcu: Make rcu_idle_enter() rely on callers disabling irqs
Tejun Heo (1):
sched: Allow migrating kthreads into online but inactive CPUs
.../RCU/Design/Requirements/Requirements.html | 130 +++++++++++ Documentation/RCU/checklist.txt | 121 +++++++---- Documentation/RCU/rcu.txt | 9 +- Documentation/RCU/rcu_dereference.txt | 61 ++---- Documentation/RCU/rcubarrier.txt | 5 + Documentation/RCU/torture.txt | 20 +- Documentation/RCU/whatisRCU.txt | 5 +- Documentation/admin-guide/kernel-parameters.txt | 7 +- Documentation/core-api/kernel-api.rst | 49 +++++ Documentation/memory-barriers.txt | 41 ++-- MAINTAINERS | 2 +- arch/alpha/include/asm/spinlock.h | 5 - arch/arc/include/asm/spinlock.h | 5 - arch/arm/include/asm/spinlock.h | 16 -- arch/arm64/include/asm/spinlock.h | 58 +---- arch/arm64/kernel/process.c | 2 + arch/blackfin/include/asm/spinlock.h | 5 - arch/blackfin/kernel/module.c | 39 ++-- arch/hexagon/include/asm/spinlock.h | 5 - arch/ia64/include/asm/spinlock.h | 21 -- arch/m32r/include/asm/spinlock.h | 5 - arch/metag/include/asm/spinlock.h | 5 - arch/mn10300/include/asm/spinlock.h | 5 - arch/parisc/include/asm/spinlock.h | 7 - arch/powerpc/include/asm/spinlock.h | 33 --- arch/s390/include/asm/spinlock.h | 7 - arch/sh/include/asm/spinlock-cas.h | 5 - arch/sh/include/asm/spinlock-llsc.h | 5 - arch/sparc/include/asm/spinlock_32.h | 5 - arch/tile/include/asm/spinlock_32.h | 2 - arch/tile/include/asm/spinlock_64.h | 2 - arch/tile/lib/spinlock_32.c | 23 -- arch/tile/lib/spinlock_64.c | 22 -- arch/xtensa/include/asm/spinlock.h | 5 - drivers/ata/libata-eh.c | 8 +- include/asm-generic/qspinlock.h | 14 -- include/linux/init_task.h | 8 +- include/linux/rcupdate.h | 15 +- include/linux/rcutiny.h | 8 +- include/linux/sched.h | 8 +- include/linux/spinlock.h | 31 --- include/linux/spinlock_up.h | 6 - include/linux/srcutiny.h | 13 ++ include/linux/srcutree.h | 3 +- include/linux/swait.h | 55 +++++ include/trace/events/rcu.h | 7 +- include/uapi/linux/membarrier.h | 23 +- ipc/sem.c | 3 +- kernel/Makefile | 1 - kernel/cpu.c | 1 + kernel/exit.c | 10 +- kernel/locking/qspinlock.c | 117 ---------- kernel/membarrier.c | 70 ------ kernel/rcu/Kconfig | 3 +- kernel/rcu/rcu.h | 128 ++--------- kernel/rcu/rcu_segcblist.c | 108 +++------- kernel/rcu/rcu_segcblist.h | 28 +-- kernel/rcu/rcuperf.c | 17 +- kernel/rcu/rcutorture.c | 83 +++---- kernel/rcu/srcutiny.c | 8 + kernel/rcu/srcutree.c | 50 ++++- kernel/rcu/tiny.c | 2 - kernel/rcu/tiny_plugin.h | 47 ---- kernel/rcu/tree.c | 238 ++++++++------------- kernel/rcu/tree.h | 15 +- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_plugin.h | 238 ++++++++++++--------- kernel/rcu/update.c | 18 +- kernel/sched/Makefile | 1 + kernel/sched/completion.c | 11 +- kernel/sched/core.c | 39 +++- kernel/sched/membarrier.c | 152 +++++++++++++ kernel/task_work.c | 8 +- kernel/torture.c | 2 +- net/netfilter/nf_conntrack_core.c | 52 +++-- .../selftests/rcutorture/bin/config_override.sh | 61 ++++++ .../testing/selftests/rcutorture/bin/functions.sh | 27 ++- .../testing/selftests/rcutorture/bin/kvm-build.sh | 11 +- .../selftests/rcutorture/bin/kvm-test-1-run.sh | 58 ++--- tools/testing/selftests/rcutorture/bin/kvm.sh | 34 ++- .../selftests/rcutorture/configs/rcu/BUSTED.boot | 2 +- .../selftests/rcutorture/configs/rcu/SRCU-C.boot | 1 - .../selftests/rcutorture/configs/rcu/SRCU-u | 3 +- .../selftests/rcutorture/configs/rcu/TREE01.boot | 2 +- .../selftests/rcutorture/doc/TREE_RCU-kconfig.txt | 2 +- 85 files changed, 1245 insertions(+), 1344 deletions(-) delete mode 100644 kernel/membarrier.c delete mode 100644 kernel/rcu/tiny_plugin.h create mode 100644 kernel/sched/membarrier.c create mode 100755 tools/testing/selftests/rcutorture/bin/config_override.sh delete mode 100644 tools/testing/selftests/rcutorture/configs/rcu/SRCU-C.boot
Re: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
Quoted 13 lines · Paul E. McKenney
Quoted 1 line
this change - or can I pick this up into the scheduler tree?
Timely question! ;-)
My current plan is to send you a pull request like the following later today, Pacific Time (but rebased adding Steve Rostedt's Reviewed-by). This patch is on one of the branches, currently v4.13-rc2..93d8d7a12090 ("arch: Remove
spin_unlock_wait()arch-specific definitions") in my -rcu tree.Ah, and v4.13-rc2..7391304c4959 ("membarrier: Expedited private command") is mostly outside of RCU as well.
Quoted 7 lines
Since I will be rebasing and remerging anyway, if you would prefer that I split the
spin_unlock_wait()and/or misc branches out, I am happy to do so. If I don't hear otherwise, though, I will send all seven branches using my usual approach.So, if you want something different than my usual approach, please just let me know!
No, all branches together sounds good to me!
If you are rebasing anyway, here are some (very minor) commit title nits I noticed:
Quoted 2 lines
swait: add idle variants which don't contribute to load average
rcu: use idle versions of swait to make idle-hack clear
Capitalization.
Quoted 1 line
membarrier: Expedited private command
Should start with a verb.
Quoted 1 line
doc: RCU documentation update
doc: Update RCU documentation
?
Quoted 1 line
doc: No longer allowed to use
rcu_dereferenceon non-pointers
doc: Describe that it is no longer allowed to use rcu_dereference() on non-pointers
?
Quoted 2 lines
torture: Add --kconfig argument to
kvm.sh
rcutorture: Don't wait for kernel when all builds fail
Is there a difference between 'torture: ' and 'rcutorture: ' prefixes?
Thanks,
Ingo
Re: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
Quoted 21 lines · Ingo Molnar
Quoted 18 lines · Paul E. McKenney
Quoted 1 line
this change - or can I pick this up into the scheduler tree?
Timely question! ;-)
My current plan is to send you a pull request like the following later today, Pacific Time (but rebased adding Steve Rostedt's Reviewed-by). This patch is on one of the branches, currently v4.13-rc2..93d8d7a12090 ("arch: Remove
spin_unlock_wait()arch-specific definitions") in my -rcu tree.Ah, and v4.13-rc2..7391304c4959 ("membarrier: Expedited private command") is mostly outside of RCU as well.
Since I will be rebasing and remerging anyway, if you would prefer that I split the
spin_unlock_wait()and/or misc branches out, I am happy to do so. If I don't hear otherwise, though, I will send all seven branches using my usual approach.
Quoted 4 lines
Quoted 2 lines
So, if you want something different than my usual approach, please just let me know!
No, all branches together sounds good to me!
Very good, will do!
Quoted 6 lines
If you are rebasing anyway, here are some (very minor) commit title nits I noticed:
Quoted 2 lines
swait: add idle variants which don't contribute to load average
rcu: use idle versions of swait to make idle-hack clearCapitalization.
Will fix! Believe it or not, I looked for these... :-/
Quoted 3 lines
Quoted 1 line
membarrier: Expedited private command
Should start with a verb.
OK, something like "Provide expedited private command".
Quoted 5 lines
Quoted 1 line
doc: RCU documentation update
doc: Update RCU documentation
?
Works for me!
Quoted 5 lines
Quoted 1 line
doc: No longer allowed to use
rcu_dereferenceon non-pointersdoc: Describe that it is no longer allowed to use
rcu_dereference()on non-pointers?
Will add a real commit log.
Quoted 4 lines
Quoted 2 lines
torture: Add --kconfig argument to
kvm.sh
rcutorture: Don't wait for kernel when all builds failIs there a difference between 'torture: ' and 'rcutorture: ' prefixes?
Yes, rcutorture is specific to RCU, while torture would also affect locktorture.
Ah, and if I am delaying the cond_resched() patch, I need to retest, which means I will send you the pull request tomorrow or Monday, depending on how the testing goes.
Thanx, Paul
Quoted 4 lines
Thanks,
Ingo
Re: [PATCH v5 tip/core/rcu 4/9] completion: Replace spin_unlock_wait() with lock/unlock pair
Quoted 63 lines · Paul E. McKenney
Quoted 21 lines · Ingo Molnar
Quoted 18 lines · Paul E. McKenney
Quoted 1 line
this change - or can I pick this up into the scheduler tree?
Timely question! ;-)
My current plan is to send you a pull request like the following later today, Pacific Time (but rebased adding Steve Rostedt's Reviewed-by). This patch is on one of the branches, currently v4.13-rc2..93d8d7a12090 ("arch: Remove
spin_unlock_wait()arch-specific definitions") in my -rcu tree.Ah, and v4.13-rc2..7391304c4959 ("membarrier: Expedited private command") is mostly outside of RCU as well.
Since I will be rebasing and remerging anyway, if you would prefer that I split the
spin_unlock_wait()and/or misc branches out, I am happy to do so. If I don't hear otherwise, though, I will send all seven branches using my usual approach.Quoted 4 lines
Quoted 2 lines
So, if you want something different than my usual approach, please just let me know!
No, all branches together sounds good to me!
Very good, will do!
Quoted 6 lines
If you are rebasing anyway, here are some (very minor) commit title nits I noticed:
Quoted 2 lines
swait: add idle variants which don't contribute to load average
rcu: use idle versions of swait to make idle-hack clearCapitalization.
Will fix! Believe it or not, I looked for these... :-/
Quoted 3 lines
Quoted 1 line
membarrier: Expedited private command
Should start with a verb.
OK, something like "Provide expedited private command".
Quoted 5 lines
Quoted 1 line
doc: RCU documentation update
doc: Update RCU documentation
?
Works for me!
Quoted 5 lines
Quoted 1 line
doc: No longer allowed to use
rcu_dereferenceon non-pointersdoc: Describe that it is no longer allowed to use
rcu_dereference()on non-pointers?
Will add a real commit log.
Quoted 3 lines
Quoted 2 lines
torture: Add --kconfig argument to
kvm.sh
rcutorture: Don't wait for kernel when all builds fail
Quoted 8 lines
Quoted 1 line
Is there a difference between 'torture: ' and 'rcutorture: ' prefixes?
Yes, rcutorture is specific to RCU, while torture would also affect locktorture.
Ah, and if I am delaying the
cond_resched()patch, I need to retest, which means I will send you the pull request tomorrow or Monday, depending on how the testing goes.
And here is the updated prototype pull request, FYI. Again, moving the cond_resched() patch requires retesting, and if all goes well, I will send you the official pull request tomorrow morning, Pacific Time.
Thanx, Paul
------------------------------------------------------------------------
The following changes since commit 520eccdfe187591a51ea9ab4c1a024ae4d0f68d9:
Linux 4.13-rc2 (2017-07-23 16:15:17 -0700)
are available in the git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git 656e7c0c0a2e8d899f87fd7f081ea7a711146604
for you to fetch changes up to 656e7c0c0a2e8d899f87fd7f081ea7a711146604:
Merge branches 'doc.2017.08.17a', 'fixes.2017.08.17a', 'hotplug.2017.07.25b', 'misc.2017.08.17a', 'spin_unlock_wait_no.2017.08.17a', 'srcu.2017.07.27c' and 'torture.2017.07.24c' into HEAD (2017-08-17 08:10:04 -0700)
----------------------------------------------------------------
Joe Perches (1):
module: Fix pr_fmt() bug for header use of printk
Luis R. Rodriguez (2):
swait: Add idle variants which don't contribute to load average
rcu: Use idle versions of swait to make idle-hack clear
Manfred Spraul (1):
net/netfilter/nf_conntrack_core: Fix net_conntrack_lock()
Masami Hiramatsu (1):
rcu/tracing: Set disable_rcu_irq_enter on rcu_eqs_exit()
Mathieu Desnoyers (1):
membarrier: Provide expedited private command
Oleg Nesterov (1):
task_work: Replace spin_unlock_wait() with lock/unlock pair
Paul E. McKenney (55):
documentation: Fix relation between nohz_full and rcu_nocbs
init_task: Remove redundant INIT_TASK_RCU_TREE_PREEMPT() macro
srcu: Move rcu_scheduler_starting() from Tiny RCU to Tiny SRCU
rcutorture: Remove obsolete SRCU-C.boot
srcu: Make process_srcu() be static
rcutorture: Move SRCU status printing to SRCU implementations
rcutorture: Print SRCU lock/unlock totals
rcu: Remove CONFIG_TASKS_RCU ifdef from rcuperf.c
rcutorture: Select CONFIG_PROVE_LOCKING for Tiny SRCU scenario
torture: Add --kconfig argument to kvm.sh
rcutorture: Don't wait for kernel when all builds fail
rcutorture: Enable SRCU readers from timer handler
rcutorture: Place event-traced strings into trace buffer
rcutorture: Use nr_cpus rather than maxcpus to limit test size
rcutorture: Add task's CPU for rcutorture writer stalls
rcutorture: Eliminate unused ts_rem local from rcu_trace_clock_local()
rcu: Add last-CPU to GP-kthread starvation messages
rcutorture: Invoke call_rcu() from timer handler
rcu: Use timer as backstop for NOCB deferred wakeups
atomics: Revert addition of comment header to spin_unlock_wait()
rcu: Migrate callbacks earlier in the CPU-offline timeline
rcu: Make expedited GPs correctly handle hardware CPU insertion
torture: Fix typo suppressing CPU-hotplug statistics
rcu: Remove orphan/adopt event-tracing fields
rcu: Check for NOCB CPUs and empty lists earlier in CB migration
rcu: Make NOCB CPUs migrate CBs directly from outgoing CPU
rcu: Advance outgoing CPU's callbacks before migrating them
rcu: Eliminate rcu_state ->orphan_lock
rcu: Advance callbacks after migration
rcu: Localize rcu_state ->orphan_pend and ->orphan_done
rcu: Remove unused RCU list functions
rcu: Move callback-list warning to irq-disable region
srcu: Provide ordering for CPU not involved in grace period
sched: Replace spin_unlock_wait() with lock/unlock pair
rcu: Drive TASKS_RCU directly off of PREEMPT
rcu: Create reasonable API for do_exit() TASKS_RCU processing
rcu: Add TPS() to event-traced strings
rcu: Move rcu.h to new trivial-function style
rcu: Add event tracing to ->gp_tasks update at GP start
rcu: Add TPS() protection for _rcu_barrier_trace strings
rcu: Add assertions verifying blocked-tasks list
rcu: Add warning to rcu_idle_enter() for irqs enabled
rcu: Remove exports from rcu_idle_exit() and rcu_idle_enter()
doc: Update RCU documentation
doc: Update memory-barriers.txt for read-to-write dependencies
doc: Add RCU files to docbook-generation files
doc: No longer allowed to use rcu_dereference on non-pointers
doc: Set down RCU's scheduling-clock-interrupt needs
completion: Replace spin_unlock_wait() with lock/unlock pair
exit: Replace spin_unlock_wait() with lock/unlock pair
ipc: Replace spin_unlock_wait() with lock/unlock pair
drivers/ata: Replace spin_unlock_wait() with lock/unlock pair
locking: Remove spin_unlock_wait() generic definitions
arch: Remove spin_unlock_wait() arch-specific definitions
Merge branches 'doc.2017.08.17a', 'fixes.2017.08.17a', 'hotplug.2017.07.25b', 'misc.2017.08.17a', 'spin_unlock_wait_no.2017.08.17a', 'srcu.2017.07.27c' and 'torture.2017.07.24c' into HEAD
Peter Zijlstra (Intel) (1):
rcu: Make rcu_idle_enter() rely on callers disabling irqs
Tejun Heo (1):
sched: Allow migrating kthreads into online but inactive CPUs
.../RCU/Design/Requirements/Requirements.html | 130 +++++++++++ Documentation/RCU/checklist.txt | 121 +++++++---- Documentation/RCU/rcu.txt | 9 +- Documentation/RCU/rcu_dereference.txt | 61 ++---- Documentation/RCU/rcubarrier.txt | 5 + Documentation/RCU/torture.txt | 20 +- Documentation/RCU/whatisRCU.txt | 5 +- Documentation/admin-guide/kernel-parameters.txt | 7 +- Documentation/core-api/kernel-api.rst | 49 +++++ Documentation/memory-barriers.txt | 41 ++-- MAINTAINERS | 2 +- arch/alpha/include/asm/spinlock.h | 5 - arch/arc/include/asm/spinlock.h | 5 - arch/arm/include/asm/spinlock.h | 16 -- arch/arm64/include/asm/spinlock.h | 58 +---- arch/arm64/kernel/process.c | 2 + arch/blackfin/include/asm/spinlock.h | 5 - arch/blackfin/kernel/module.c | 39 ++-- arch/hexagon/include/asm/spinlock.h | 5 - arch/ia64/include/asm/spinlock.h | 21 -- arch/m32r/include/asm/spinlock.h | 5 - arch/metag/include/asm/spinlock.h | 5 - arch/mn10300/include/asm/spinlock.h | 5 - arch/parisc/include/asm/spinlock.h | 7 - arch/powerpc/include/asm/spinlock.h | 33 --- arch/s390/include/asm/spinlock.h | 7 - arch/sh/include/asm/spinlock-cas.h | 5 - arch/sh/include/asm/spinlock-llsc.h | 5 - arch/sparc/include/asm/spinlock_32.h | 5 - arch/tile/include/asm/spinlock_32.h | 2 - arch/tile/include/asm/spinlock_64.h | 2 - arch/tile/lib/spinlock_32.c | 23 -- arch/tile/lib/spinlock_64.c | 22 -- arch/xtensa/include/asm/spinlock.h | 5 - drivers/ata/libata-eh.c | 8 +- include/asm-generic/qspinlock.h | 14 -- include/linux/init_task.h | 8 +- include/linux/rcupdate.h | 15 +- include/linux/rcutiny.h | 8 +- include/linux/sched.h | 5 +- include/linux/spinlock.h | 31 --- include/linux/spinlock_up.h | 6 - include/linux/srcutiny.h | 13 ++ include/linux/srcutree.h | 3 +- include/linux/swait.h | 55 +++++ include/trace/events/rcu.h | 7 +- include/uapi/linux/membarrier.h | 23 +- ipc/sem.c | 3 +- kernel/Makefile | 1 - kernel/cpu.c | 1 + kernel/exit.c | 10 +- kernel/locking/qspinlock.c | 117 ---------- kernel/membarrier.c | 70 ------ kernel/rcu/Kconfig | 3 +- kernel/rcu/rcu.h | 128 ++--------- kernel/rcu/rcu_segcblist.c | 108 +++------- kernel/rcu/rcu_segcblist.h | 28 +-- kernel/rcu/rcuperf.c | 17 +- kernel/rcu/rcutorture.c | 83 +++---- kernel/rcu/srcutiny.c | 8 + kernel/rcu/srcutree.c | 50 ++++- kernel/rcu/tiny.c | 2 - kernel/rcu/tiny_plugin.h | 47 ---- kernel/rcu/tree.c | 213 ++++++++---------- kernel/rcu/tree.h | 15 +- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_plugin.h | 238 ++++++++++++--------- kernel/rcu/update.c | 18 +- kernel/sched/Makefile | 1 + kernel/sched/completion.c | 11 +- kernel/sched/core.c | 38 +++- kernel/sched/membarrier.c | 152 +++++++++++++ kernel/task_work.c | 8 +- kernel/torture.c | 2 +- net/netfilter/nf_conntrack_core.c | 52 +++-- .../selftests/rcutorture/bin/config_override.sh | 61 ++++++ .../testing/selftests/rcutorture/bin/functions.sh | 27 ++- .../testing/selftests/rcutorture/bin/kvm-build.sh | 11 +- .../selftests/rcutorture/bin/kvm-test-1-run.sh | 58 ++--- tools/testing/selftests/rcutorture/bin/kvm.sh | 34 ++- .../selftests/rcutorture/configs/rcu/BUSTED.boot | 2 +- .../selftests/rcutorture/configs/rcu/SRCU-C.boot | 1 - .../selftests/rcutorture/configs/rcu/SRCU-u | 3 +- .../selftests/rcutorture/configs/rcu/TREE01.boot | 2 +- .../selftests/rcutorture/doc/TREE_RCU-kconfig.txt | 2 +- 85 files changed, 1237 insertions(+), 1323 deletions(-) delete mode 100644 kernel/membarrier.c delete mode 100644 kernel/rcu/tiny_plugin.h create mode 100644 kernel/sched/membarrier.c create mode 100755 tools/testing/selftests/rcutorture/bin/config_override.sh delete mode 100644 tools/testing/selftests/rcutorture/configs/rcu/SRCU-C.boot
PATCH 5/9 exit: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in do_exit() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock is a per-task lock, and this is happening only at task-exit time.
--- kernel/exit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/exit.c +2 -1
@@ -819,7 +819,8 @@ void __noreturn do_exit(long code)* Ensure that we must observe the pi_state in exit_mm() -> * mm_release() -> exit_pi_state_list(). */ -raw_spin_unlock_wait (&tsk->pi_lock); +raw_spin_lock_irq (&tsk->pi_lock); +raw_spin_unlock_irq (&tsk->pi_lock);if (unlikely (in_atomic ())) { pr_info("note: %s[%d] exited with preempt_count %d\n",
Signature · 1 line
2.5.2
PATCH 6/9 ipc: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in exit_sem() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because exit_sem() is rarely invoked in production.
--- ipc/sem.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
ipc/sem.c +2 -1
@@ -2091,7 +2091,8 @@ void exit_sem(struct task_struct *tsk)* possibility where we exit while freeary() didn't * finish unlocking sem_undo_list. */ -spin_unlock_wait (&ulp->lock); +spin_lock (&ulp->lock); +spin_unlock (&ulp->lock);rcu_read_unlock ();break ; }
Signature · 1 line
2.5.2
PATCH 7/9 drivers/ata: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore eliminates the spin_unlock_wait() call and associated else-clause and hoists the then-clause's lock and unlock out of the "if" statement. This should be safe from a performance perspective because according to Tejun there should be few if any drivers that don't set their own error handler.
--- drivers/ata/libata-eh.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
drivers/ata/libata-eh.c +3 -5
@@ -645,12 +645,11 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,* completions are honored. A scmd is determined to have * timed out iff its associated qc is active and not failed. */ +spin_lock_irqsave (ap->lock, flags);if (ap->ops->error_handler) {struct scsi_cmnd *scmd, *tmp;int nr_timedout =0 ; -spin_lock_irqsave (ap->lock, flags); -/* This must occur under the ap->lock as we don't want a polled recovery to race the real interrupt handler
@@ -700,12 +699,11 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,if (nr_timedout)__ata_port_freeze (ap); -spin_unlock_irqrestore (ap->lock, flags);/* initialize eh_tries */ ap->eh_tries =ATA_EH_MAX_TRIES ; - }else -spin_unlock_wait (ap->lock); + } +spin_unlock_irqrestore (ap->lock, flags); }EXPORT_SYMBOL (ata_scsi_cmd_error_handler);
Signature · 1 line
2.5.2
PATCH 8/9 locking: Remove spin_unlock_wait() generic definitions
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore removes spin_unlock_wait() and related definitions from core code.
--- include/asm-generic/qspinlock.h | 14 ----- include/linux/spinlock.h | 31 ----------- include/linux/spinlock_up.h | 6 --- kernel/locking/qspinlock.c | 117 ---------------------------------------- 4 files changed, 168 deletions(-)
include/asm-generic/qspinlock.h +0 -14
@@ -22,17 +22,6 @@#include <asm-generic/qspinlock_types.h> /** -* queued_spin_unlock_wait - wait until the _current_ lock holder releases the lock -* @lock : Pointer to queued spinlock structure -* -* There is a very slight possibility of live-lock if the lockers keep coming -* and the waiter is just unfortunate enough to not see any unlock state. -*/ -#ifndef queued_spin_unlock_wait -extern void queued_spin_unlock_wait (struct qspinlock *lock); -#endif - -/** * queued_spin_is_locked - is the spinlock locked? * @lock: Pointer to queued spinlock structure * Return: 1 if it is locked, 0 otherwise
@@ -41,8 +30,6 @@ extern void queued_spin_unlock_wait(struct qspinlock *lock); static __always_inline int queued_spin_is_locked(struct qspinlock *lock) {/* -* See queued_spin_unlock_wait(). -* * Any !0 state indicates it is locked, even if _Q_LOCKED_VAL * isn't immediately observable. */
@@ -135,6 +122,5 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock)#define arch_spin_trylock (l) queued_spin_trylock(l)#define arch_spin_unlock (l) queued_spin_unlock(l)#define arch_spin_lock_flags (l, f) queued_spin_lock(l) -#define arch_spin_unlock_wait (l) queued_spin_unlock_wait(l)#endif /* __ASM_GENERIC_QSPINLOCK_H */
include/linux/spinlock.h +0 -31
@@ -130,12 +130,6 @@ do { \#define smp_mb__before_spinlock () smp_wmb()#endif -/** -* raw_spin_unlock_wait - wait until the spinlock gets unlocked -* @lock: the spinlock in question. -*/ -#define raw_spin_unlock_wait (lock) arch_spin_unlock_wait(&(lock)->raw_lock) -#ifdef CONFIG_DEBUG_SPINLOCK extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);#define do_raw_spin_lock_flags (lock, flags) do_raw_spin_lock(lock)
@@ -369,31 +363,6 @@ static __always_inline int spin_trylock_irq(spinlock_t *lock)raw_spin_trylock_irqsave (spinlock_check (lock), flags); \ }) -/** -* spin_unlock_wait - Interpose between successive critical sections -* @lock: the spinlock whose critical sections are to be interposed. -* -* Semantically this is equivalent to a spin_lock() immediately -* followed by a spin_unlock(). However, most architectures have -* more efficient implementations in which the spin_unlock_wait() -* cannot block concurrent lock acquisition, and in some cases -* where spin_unlock_wait() does not write to the lock variable. -* Nevertheless, spin_unlock_wait() can have high overhead, so if -* you feel the need to use it, please check to see if there is -* a better way to get your job done. -* -* The ordering guarantees provided by spin_unlock_wait() are: -* -* 1. All accesses preceding the spin_unlock_wait() happen before -* any accesses in later critical sections for this same lock. -* 2. All accesses following the spin_unlock_wait() happen after -* any accesses in earlier critical sections for this same lock. -*/ -static __always_inline void spin_unlock_wait(spinlock_t *lock) -{ -raw_spin_unlock_wait (&lock->rlock); -} - static __always_inline int spin_is_locked(spinlock_t *lock) {return raw_spin_is_locked (&lock->rlock);
include/linux/spinlock_up.h +0 -6
@@ -26,11 +26,6 @@#ifdef CONFIG_DEBUG_SPINLOCK #define arch_spin_is_locked (x) ((x)->slock == 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock,VAL ); -} -static inline void arch_spin_lock (arch_spinlock_t *lock) { lock->slock =0 ;
@@ -73,7 +68,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)#else /* DEBUG_SPINLOCK */ #define arch_spin_is_locked (lock) ((void)(lock), 0) -#define arch_spin_unlock_wait (lock) do { barrier(); (void)(lock); } while (0)/* for sched/core.c and kernel_lock.c: */ # define arch_spin_lock (lock) do { barrier(); (void)(lock); } while (0)# define arch_spin_lock_flags (lock, flags) do { barrier(); (void)(lock); } while (0)
kernel/locking/qspinlock.c +0 -117
123 lines not on this page · raw message
Signature · 1 line
2.5.2
PATCH 9/9 arch: Remove spin_unlock_wait() arch-specific definitions
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore removes the underlying arch-specific arch_spin_unlock_wait() for all architectures providing them.
--- arch/alpha/include/asm/spinlock.h | 5 ---- arch/arc/include/asm/spinlock.h | 5 ---- arch/arm/include/asm/spinlock.h | 16 ---------- arch/arm64/include/asm/spinlock.h | 58 ++++-------------------------------- arch/blackfin/include/asm/spinlock.h | 5 ---- arch/hexagon/include/asm/spinlock.h | 5 ---- arch/ia64/include/asm/spinlock.h | 21 ------------- arch/m32r/include/asm/spinlock.h | 5 ---- arch/metag/include/asm/spinlock.h | 5 ---- arch/mn10300/include/asm/spinlock.h | 5 ---- arch/parisc/include/asm/spinlock.h | 7 ----- arch/powerpc/include/asm/spinlock.h | 33 -------------------- arch/s390/include/asm/spinlock.h | 7 ----- arch/sh/include/asm/spinlock-cas.h | 5 ---- arch/sh/include/asm/spinlock-llsc.h | 5 ---- arch/sparc/include/asm/spinlock_32.h | 5 ---- arch/tile/include/asm/spinlock_32.h | 2 -- arch/tile/include/asm/spinlock_64.h | 2 -- arch/tile/lib/spinlock_32.c | 23 -------------- arch/tile/lib/spinlock_64.c | 22 -------------- arch/xtensa/include/asm/spinlock.h | 5 ---- 21 files changed, 5 insertions(+), 241 deletions(-)
arch/alpha/include/asm/spinlock.h +0 -5
@@ -16,11 +16,6 @@#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)#define arch_spin_is_locked (x) ((x)->lock != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -static inline int arch_spin_value_unlocked (arch_spinlock_t lock) {return lock.lock ==0 ;
arch/arc/include/asm/spinlock.h +0 -5
@@ -16,11 +16,6 @@#define arch_spin_is_locked (x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock, !VAL ); -} -#ifdef CONFIG_ARC_HAS_LLSC static inline void arch_spin_lock (arch_spinlock_t *lock)
arch/arm/include/asm/spinlock.h +0 -16
@@ -52,22 +52,6 @@ static inline void dsb_sev(void)* memory. */ -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -u16 owner =READ_ONCE (lock->tickets.owner); - -for (;;) { -arch_spinlock_t tmp =READ_ONCE (*lock); - -if (tmp.tickets.owner == tmp.tickets.next || - tmp.tickets.owner != owner) -break ; - -wfe (); - } -smp_acquire__after_ctrl_dep (); -} -#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)static inline void arch_spin_lock (arch_spinlock_t *lock)
arch/arm64/include/asm/spinlock.h +5 -53
70 lines not on this page · raw message
arch/blackfin/include/asm/spinlock.h +0 -5
@@ -48,11 +48,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)__raw_spin_unlock_asm (&lock->lock); } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -static inline int arch_read_can_lock (arch_rwlock_t *rw) {return __raw_uncached_fetch_asm (&rw->lock) >0 ;
arch/hexagon/include/asm/spinlock.h +0 -5
@@ -179,11 +179,6 @@ static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)*/ #define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -#define arch_spin_is_locked (x) ((x)->lock != 0)#define arch_read_lock_flags (lock, flags) arch_read_lock(lock)
arch/ia64/include/asm/spinlock.h +0 -21
@@ -76,22 +76,6 @@ static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)ACCESS_ONCE (*p) = (tmp +2 ) & ~1 ; } -static __always_inline void __ticket_spin_unlock_wait(arch_spinlock_t *lock) -{ -int *p = (int *)&lock->lock, ticket; - -ia64_invala (); - -for (;;) { - asmvolatile ("ld4.c.nc %0=[%1]" :"=r" (ticket) :"r" (p) :"memory" ); -if (!(((ticket >>TICKET_SHIFT ) ^ ticket) &TICKET_MASK )) -return ; -cpu_relax (); - } - -smp_acquire__after_ctrl_dep (); -} -static inline int __ticket_spin_is_locked (arch_spinlock_t *lock) {long tmp =ACCESS_ONCE (lock->lock);
@@ -143,11 +127,6 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,arch_spin_lock (lock); } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -__ticket_spin_unlock_wait (lock); -} -#define arch_read_can_lock (rw) (*(volatile int *)(rw) >= 0)#define arch_write_can_lock (rw) (*(volatile int *)(rw) == 0)
arch/m32r/include/asm/spinlock.h +0 -5
@@ -30,11 +30,6 @@#define arch_spin_is_locked (x) (*(volatile int *)(&(x)->slock) <= 0)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock,VAL >0 ); -} -/** * arch_spin_trylock - Try spin lock and return a result * @lock: Pointer to the lock variable
arch/metag/include/asm/spinlock.h +0 -5
@@ -15,11 +15,6 @@* locked. */ -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)#define arch_read_lock_flags (lock, flags) arch_read_lock(lock)
arch/mn10300/include/asm/spinlock.h +0 -5
@@ -26,11 +26,6 @@#define arch_spin_is_locked (x) (*(volatile signed char *)(&(x)->slock) != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock, !VAL ); -} -static inline void arch_spin_unlock (arch_spinlock_t *lock) { asm volatile(
arch/parisc/include/asm/spinlock.h +0 -7
@@ -14,13 +14,6 @@ static inline int arch_spin_is_locked(arch_spinlock_t *x)#define arch_spin_lock (lock) arch_spin_lock_flags(lock, 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *x) -{ -volatile unsigned int *a =__ldcw_align (x); - -smp_cond_load_acquire (a,VAL ); -} -static inline void arch_spin_lock_flags (arch_spinlock_t *x,unsigned long flags) {
arch/powerpc/include/asm/spinlock.h +0 -33
@@ -170,39 +170,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->slock =0 ; } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -arch_spinlock_t lock_val; - -smp_mb (); - -/* -* Atomically load and store back the lock value (unchanged). This -* ensures that our observation of the lock value is ordered with -* respect to other lock operations. -*/ - __asm__ __volatile__( -"1: " PPC_LWARX(%0, 0, %2, 0) "\n" -" stwcx. %0, 0, %2\n" -" bne- 1b\n" - : "=&r" (lock_val), "+m" (*lock) - : "r" (lock) - : "cr0", "xer"); - - if (arch_spin_value_unlocked(lock_val)) - goto out; - -while (lock->slock) { -HMT_low (); -if (SHARED_PROCESSOR ) -__spin_yield (lock); - } -HMT_medium (); - -out: -smp_mb (); -} -/* * Read-write spinlocks, allowing multiple readers * but only one writer.
arch/s390/include/asm/spinlock.h +0 -7
@@ -98,13 +98,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lp) : "cc", "memory"); } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -while (arch_spin_is_locked (lock)) -arch_spin_relax (lock); -smp_acquire__after_ctrl_dep (); -} -/* * Read-write spinlocks, allowing multiple readers * but only one writer.
arch/sh/include/asm/spinlock-cas.h +0 -5
@@ -29,11 +29,6 @@ static inline unsigned __sl_cas(volatile unsigned *p, unsigned old, unsigned new#define arch_spin_is_locked (x) ((x)->lock <= 0)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock,VAL >0 ); -} -static inline void arch_spin_lock (arch_spinlock_t *lock) {while (!__sl_cas (&lock->lock,1 ,0 ));
arch/sh/include/asm/spinlock-llsc.h +0 -5
@@ -21,11 +21,6 @@#define arch_spin_is_locked (x) ((x)->lock <= 0)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock,VAL >0 ); -} -/* * Simple spin lock operations. There are two variants, one clears IRQ's * on the local processor, one does not.
arch/sparc/include/asm/spinlock_32.h +0 -5
@@ -14,11 +14,6 @@#define arch_spin_is_locked (lock) (*((volatile unsigned char *)(lock)) != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -static inline void arch_spin_lock (arch_spinlock_t *lock) { __asm__ __volatile__(
arch/tile/include/asm/spinlock_32.h +0 -2
@@ -64,8 +64,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->current_ticket = old_ticket +TICKET_QUANTUM ; } -void arch_spin_unlock_wait (arch_spinlock_t *lock); -/* * Read-write spinlocks, allowing multiple readers * but only one writer.
arch/tile/include/asm/spinlock_64.h +0 -2
@@ -58,8 +58,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)__insn_fetchadd4 (&lock->lock,1U << __ARCH_SPIN_CURRENT_SHIFT); } -void arch_spin_unlock_wait (arch_spinlock_t *lock); -void arch_spin_lock_slow (arch_spinlock_t *lock,u32 val);/* Grab the "next" ticket number and bump it atomically.
arch/tile/lib/spinlock_32.c +0 -23
@@ -62,29 +62,6 @@ int arch_spin_trylock(arch_spinlock_t *lock) }EXPORT_SYMBOL (arch_spin_trylock); -void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -u32 iterations =0 ; -int curr =READ_ONCE (lock->current_ticket); -int next =READ_ONCE (lock->next_ticket); - -/* Return immediately if unlocked. */ -if (next == curr) -return ; - -/* Wait until the current locker has released the lock. */ -do { -delay_backoff (iterations++); - }while (READ_ONCE (lock->current_ticket) == curr); - -/* -* The TILE architecture doesn't do read speculation; therefore -* a control dependency guarantees a LOAD->{LOAD,STORE} order. -*/ -barrier (); -} -EXPORT_SYMBOL (arch_spin_unlock_wait); -/* * The low byte is always reserved to be the marker for a "tns" operation * since the low bit is set to "1" by a tns. The next seven bits are
arch/tile/lib/spinlock_64.c +0 -22
@@ -62,28 +62,6 @@ int arch_spin_trylock(arch_spinlock_t *lock) }EXPORT_SYMBOL (arch_spin_trylock); -void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -u32 iterations =0 ; -u32 val =READ_ONCE (lock->lock); -u32 curr =arch_spin_current (val); - -/* Return immediately if unlocked. */ -if (arch_spin_next (val) == curr) -return ; - -/* Wait until the current locker has released the lock. */ -do { -delay_backoff (iterations++); - }while (arch_spin_current (READ_ONCE (lock->lock)) == curr); - -/* -* The TILE architecture doesn't do read speculation; therefore -* a control dependency guarantees a LOAD->{LOAD,STORE} order. -*/ -barrier (); -} -EXPORT_SYMBOL (arch_spin_unlock_wait);/* * If the read lock fails due to a writer, we retry periodically
arch/xtensa/include/asm/spinlock.h +0 -5
@@ -33,11 +33,6 @@#define arch_spin_is_locked (x) ((x)->slock != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock, !VAL ); -} -#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)static inline void arch_spin_lock (arch_spinlock_t *lock)
Signature · 1 line
2.5.2
v2 cover Remove spin_unlock_wait()
In reply to Paul E. McKenneyHello!
This series removes spin_unlock_wait():
1. Revert over-enthusiastic spin_unlock_wait() header comment.
2. Remove spin_unlock_wait() from net/netfilter/nf_conntrack_core,
courtesy of Manfred Spraul.
3. Replace task_work use of spin_unlock_wait() with lock/unlock
pair, courtesy of Oleg Nesterov.
4. Replace scheduler use of spin_unlock_wait() with lock/unlock pair.
5. Replace completion use of spin_unlock_wait() with lock/unlock
pair.
6. Replace exit() use of spin_unlock_wait() with lock/unlock pair.
7. Replace IPC use of spin_unlock_wait() with lock/unlock pair.
8. Replace ATA use of spin_unlock_wait() with lock/unlock pair.
9. Remove generic definition of spin_unlock_wait().
10. Remove architecture-specific definitions of spin_unlock_wait().
Thanx, Paul
------------------------------------------------------------------------
arch/alpha/include/asm/spinlock.h | 5 - arch/arc/include/asm/spinlock.h | 5 - arch/arm/include/asm/spinlock.h | 16 ---- arch/arm64/include/asm/spinlock.h | 58 +---------------- arch/blackfin/include/asm/spinlock.h | 5 - arch/hexagon/include/asm/spinlock.h | 5 - arch/ia64/include/asm/spinlock.h | 21 ------ arch/m32r/include/asm/spinlock.h | 5 - arch/metag/include/asm/spinlock.h | 5 - arch/mn10300/include/asm/spinlock.h | 5 - arch/parisc/include/asm/spinlock.h | 7 -- arch/powerpc/include/asm/spinlock.h | 33 --------- arch/s390/include/asm/spinlock.h | 7 -- arch/sh/include/asm/spinlock-cas.h | 5 - arch/sh/include/asm/spinlock-llsc.h | 5 - arch/sparc/include/asm/spinlock_32.h | 5 - arch/tile/include/asm/spinlock_32.h | 2 arch/tile/include/asm/spinlock_64.h | 2 arch/tile/lib/spinlock_32.c | 23 ------ arch/tile/lib/spinlock_64.c | 22 ------ arch/xtensa/include/asm/spinlock.h | 5 - drivers/ata/libata-eh.c | 8 -- include/asm-generic/qspinlock.h | 14 ---- include/linux/spinlock.h | 31 --------- include/linux/spinlock_up.h | 6 - ipc/sem.c | 3 kernel/exit.c | 3 kernel/locking/qspinlock.c | 117 ----------------------------------- kernel/sched/completion.c | 9 -- kernel/sched/core.c | 5 - kernel/task_work.c | 8 -- net/netfilter/nf_conntrack_core.c | 52 ++++++++------- 32 files changed, 48 insertions(+), 454 deletions(-)
PATCH v2 1/10 atomics: Revert addition of comment header to spin_unlock_wait()
There is still considerable confusion as to the semantics of spin_unlock_wait(), but there seems to be universal agreement that it is not that of a lock/unlock pair. This commit therefore removes the comment added by 6016ffc3874d ("atomics: Add header comment so spin_unlock_wait()") in order to prevent at least that flavor of confusion.
--- include/linux/spinlock.h | 20 -------------------- 1 file changed, 20 deletions(-)
include/linux/spinlock.h +0 -20
@@ -369,26 +369,6 @@ static __always_inline int spin_trylock_irq(spinlock_t *lock)raw_spin_trylock_irqsave (spinlock_check (lock), flags); \ }) -/** -* spin_unlock_wait - Interpose between successive critical sections -* @lock: the spinlock whose critical sections are to be interposed. -* -* Semantically this is equivalent to a spin_lock() immediately -* followed by a spin_unlock(). However, most architectures have -* more efficient implementations in which the spin_unlock_wait() -* cannot block concurrent lock acquisition, and in some cases -* where spin_unlock_wait() does not write to the lock variable. -* Nevertheless, spin_unlock_wait() can have high overhead, so if -* you feel the need to use it, please check to see if there is -* a better way to get your job done. -* -* The ordering guarantees provided by spin_unlock_wait() are: -* -* 1. All accesses preceding the spin_unlock_wait() happen before -* any accesses in later critical sections for this same lock. -* 2. All accesses following the spin_unlock_wait() happen after -* any accesses in earlier critical sections for this same lock. -*/ static __always_inline void spin_unlock_wait(spinlock_t *lock) {raw_spin_unlock_wait (&lock->rlock);
Signature · 1 line
2.5.2
PATCH v2 2/10 net/netfilter/nf_conntrack_core: Fix net_conntrack_lock()
In reply to Paul E. McKenneyFrom: Manfred Spraul <[email protected]>
As we want to remove spin_unlock_wait() and replace it with explicit spin_lock()/spin_unlock() calls, we can use this to simplify the locking.
In addition:
- Reading nf_conntrack_locks_all needs ACQUIRE memory ordering.
- The new code avoids the backwards loop.
Only slightly tested, I did not manage to trigger calls to nf_conntrack_all_lock().
V2: With improved comments, to clearly show how the barriers
pair.
--- net/netfilter/nf_conntrack_core.c | 52 ++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 23 deletions(-)
net/netfilter/nf_conntrack_core.c +29 -23
@@ -96,19 +96,26 @@ static struct conntrack_gc_work conntrack_gc_work;void nf_conntrack_lock (spinlock_t *lock)__acquires (lock) { +/* 1) Acquire the lock */ spin_lock (lock); -while (unlikely (nf_conntrack_locks_all)) { -spin_unlock (lock); -/* -* Order the 'nf_conntrack_locks_all' load vs. the -* spin_unlock_wait() loads below, to ensure -* that 'nf_conntrack_locks_all_lock' is indeed held: -*/ -smp_rmb ();/* spin_lock(&nf_conntrack_locks_all_lock) */ -spin_unlock_wait (&nf_conntrack_locks_all_lock); -spin_lock (lock); - } +/* 2) read nf_conntrack_locks_all, with ACQUIRE semantics +* It pairs with the smp_store_release() in nf_conntrack_all_unlock() +*/ +if (likely (smp_load_acquire (&nf_conntrack_locks_all) == false)) +return ; + +/* fast path failed, unlock */ +spin_unlock (lock); + +/* Slow path 1) get global lock */ +spin_lock (&nf_conntrack_locks_all_lock); + +/* Slow path 2) get the lock we want */ +spin_lock (lock); + +/* Slow path 3) release the global lock */ +spin_unlock (&nf_conntrack_locks_all_lock); }EXPORT_SYMBOL_GPL (nf_conntrack_lock);
@@ -149,28 +156,27 @@ static void nf_conntrack_all_lock(void)int i;spin_lock (&nf_conntrack_locks_all_lock); - nf_conntrack_locks_all = true; -/* -* Order the above store of 'nf_conntrack_locks_all' against -* the spin_unlock_wait() loads below, such that if -* nf_conntrack_lock() observes 'nf_conntrack_locks_all' -* we must observe nf_conntrack_locks[] held: -*/ -smp_mb ();/* spin_lock(&nf_conntrack_locks_all_lock) */ + nf_conntrack_locks_all = true;for (i =0 ; i <CONNTRACK_LOCKS ; i++) { -spin_unlock_wait (&nf_conntrack_locks[i]); +spin_lock (&nf_conntrack_locks[i]); + +/* This spin_unlock provides the "release" to ensure that +* nf_conntrack_locks_all==true is visible to everyone that +* acquired spin_lock(&nf_conntrack_locks[]). +*/ +spin_unlock (&nf_conntrack_locks[i]); } }static void nf_conntrack_all_unlock (void ) { -/* -* All prior stores must be complete before we clear +/* All prior stores must be complete before we clear * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock() * might observe the false value but not the entire -* critical section: +* critical section. +* It pairs with the smp_load_acquire() in nf_conntrack_lock() */ smp_store_release (&nf_conntrack_locks_all, false);spin_unlock (&nf_conntrack_locks_all_lock);
Signature · 1 line
2.5.2
PATCH v2 3/10 task_work: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyFrom: Oleg Nesterov <[email protected]>
There is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in task_work_run() with a spin_lock_irq() and a spin_unlock_irq() aruond the cmpxchg() dequeue loop. This should be safe from a performance perspective because ->pi_lock is local to the task and because calls to the other side of the race, task_work_cancel(), should be rare.
--- kernel/task_work.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
kernel/task_work.c +2 -6
@@ -96,20 +96,16 @@ void task_work_run(void)* work->func() can do task_work_add(), do not set * work_exited unless the list is empty. */ +raw_spin_lock_irq (&task->pi_lock);do { work =READ_ONCE (task->task_works); head = !work && (task->flags &PF_EXITING ) ? &work_exited :NULL ; }while (cmpxchg (&task->task_works, work, head) != work); +raw_spin_unlock_irq (&task->pi_lock);if (!work)break ; -/* -* Synchronize with task_work_cancel(). It can't remove -* the first entry == work, cmpxchg(task_works) should -* fail, but it can play with *work and other entries. -*/ -raw_spin_unlock_wait (&task->pi_lock); do { next = work->next;
Signature · 1 line
2.5.2
PATCH v2 4/10 sched: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in do_task_dead() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock is this tasks ->pi_lock, and this is called only after the task exits.
[ paulmck: Replace leading smp_mb() with smp_mb__before_spinlock(),
courtesy of Arnd Bergmann's noting its odd location. ]
--- kernel/sched/core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
kernel/sched/core.c +3 -2
@@ -3352,8 +3352,9 @@ void __noreturn do_task_dead(void)* To avoid it, we have to wait for releasing tsk->pi_lock which * is held by try_to_wake_up() */ -smp_mb (); -raw_spin_unlock_wait (¤t->pi_lock); +smp_mb__before_spinlock (); +raw_spin_lock_irq (¤t->pi_lock); +raw_spin_unlock_irq (¤t->pi_lock);/* Causes final put_task_struct in finish_task_switch(): */ __set_current_state (TASK_DEAD );
Signature · 1 line
2.5.2
PATCH v2 5/10 completion: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in completion_done() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock will be held only the wakeup happens really quickly.
--- kernel/sched/completion.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
kernel/sched/completion.c +2 -7
@@ -307,14 +307,9 @@ bool completion_done(struct completion *x)* If ->done, we need to wait for complete() to release ->wait.lock * otherwise we can end up freeing the completion before complete() * is done referencing it. -* -* The RMB pairs with complete()'s RELEASE of ->wait.lock and orders -* the loads of ->done and ->wait.lock such that we cannot observe -* the lock before complete() acquires it while observing the ->done -* after it's acquired the lock. */ -smp_rmb (); -spin_unlock_wait (&x->wait.lock); +spin_lock_irq (&x->wait.lock); +spin_unlock_irq (&x->wait.lock);return true; }EXPORT_SYMBOL (completion_done);
Signature · 1 line
2.5.2
PATCH v2 6/10 exit: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in do_exit() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because the lock is a per-task lock, and this is happening only at task-exit time.
--- kernel/exit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
kernel/exit.c +2 -1
@@ -819,7 +819,8 @@ void __noreturn do_exit(long code)* Ensure that we must observe the pi_state in exit_mm() -> * mm_release() -> exit_pi_state_list(). */ -raw_spin_unlock_wait (&tsk->pi_lock); +raw_spin_lock_irq (&tsk->pi_lock); +raw_spin_unlock_irq (&tsk->pi_lock);if (unlikely (in_atomic ())) { pr_info("note: %s[%d] exited with preempt_count %d\n",
Signature · 1 line
2.5.2
PATCH v2 7/10 ipc: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore replaces the spin_unlock_wait() call in exit_sem() with spin_lock() followed immediately by spin_unlock(). This should be safe from a performance perspective because exit_sem() is rarely invoked in production.
--- ipc/sem.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
ipc/sem.c +2 -1
@@ -2091,7 +2091,8 @@ void exit_sem(struct task_struct *tsk)* possibility where we exit while freeary() didn't * finish unlocking sem_undo_list. */ -spin_unlock_wait (&ulp->lock); +spin_lock (&ulp->lock); +spin_unlock (&ulp->lock);rcu_read_unlock ();break ; }
Signature · 1 line
2.5.2
PATCH v2 8/10 drivers/ata: Replace spin_unlock_wait() with lock/unlock pair
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore eliminates the spin_unlock_wait() call and associated else-clause and hoists the then-clause's lock and unlock out of the "if" statement. This should be safe from a performance perspective because according to Tejun there should be few if any drivers that don't set their own error handler.
--- drivers/ata/libata-eh.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
drivers/ata/libata-eh.c +3 -5
@@ -645,12 +645,11 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,* completions are honored. A scmd is determined to have * timed out iff its associated qc is active and not failed. */ +spin_lock_irqsave (ap->lock, flags);if (ap->ops->error_handler) {struct scsi_cmnd *scmd, *tmp;int nr_timedout =0 ; -spin_lock_irqsave (ap->lock, flags); -/* This must occur under the ap->lock as we don't want a polled recovery to race the real interrupt handler
@@ -700,12 +699,11 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,if (nr_timedout)__ata_port_freeze (ap); -spin_unlock_irqrestore (ap->lock, flags);/* initialize eh_tries */ ap->eh_tries =ATA_EH_MAX_TRIES ; - }else -spin_unlock_wait (ap->lock); + } +spin_unlock_irqrestore (ap->lock, flags); }EXPORT_SYMBOL (ata_scsi_cmd_error_handler);
Signature · 1 line
2.5.2
PATCH v2 9/10 locking: Remove spin_unlock_wait() generic definitions
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore removes spin_unlock_wait() and related definitions from core code.
--- include/asm-generic/qspinlock.h | 14 ----- include/linux/spinlock.h | 11 ---- include/linux/spinlock_up.h | 6 --- kernel/locking/qspinlock.c | 117 ---------------------------------------- 4 files changed, 148 deletions(-)
include/asm-generic/qspinlock.h +0 -14
@@ -22,17 +22,6 @@#include <asm-generic/qspinlock_types.h> /** -* queued_spin_unlock_wait - wait until the _current_ lock holder releases the lock -* @lock : Pointer to queued spinlock structure -* -* There is a very slight possibility of live-lock if the lockers keep coming -* and the waiter is just unfortunate enough to not see any unlock state. -*/ -#ifndef queued_spin_unlock_wait -extern void queued_spin_unlock_wait (struct qspinlock *lock); -#endif - -/** * queued_spin_is_locked - is the spinlock locked? * @lock: Pointer to queued spinlock structure * Return: 1 if it is locked, 0 otherwise
@@ -41,8 +30,6 @@ extern void queued_spin_unlock_wait(struct qspinlock *lock); static __always_inline int queued_spin_is_locked(struct qspinlock *lock) {/* -* See queued_spin_unlock_wait(). -* * Any !0 state indicates it is locked, even if _Q_LOCKED_VAL * isn't immediately observable. */
@@ -135,6 +122,5 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock)#define arch_spin_trylock (l) queued_spin_trylock(l)#define arch_spin_unlock (l) queued_spin_unlock(l)#define arch_spin_lock_flags (l, f) queued_spin_lock(l) -#define arch_spin_unlock_wait (l) queued_spin_unlock_wait(l)#endif /* __ASM_GENERIC_QSPINLOCK_H */
include/linux/spinlock.h +0 -11
@@ -130,12 +130,6 @@ do { \#define smp_mb__before_spinlock () smp_wmb()#endif -/** -* raw_spin_unlock_wait - wait until the spinlock gets unlocked -* @lock: the spinlock in question. -*/ -#define raw_spin_unlock_wait (lock) arch_spin_unlock_wait(&(lock)->raw_lock) -#ifdef CONFIG_DEBUG_SPINLOCK extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);#define do_raw_spin_lock_flags (lock, flags) do_raw_spin_lock(lock)
@@ -369,11 +363,6 @@ static __always_inline int spin_trylock_irq(spinlock_t *lock)raw_spin_trylock_irqsave (spinlock_check (lock), flags); \ }) -static __always_inline void spin_unlock_wait(spinlock_t *lock) -{ -raw_spin_unlock_wait (&lock->rlock); -} - static __always_inline int spin_is_locked(spinlock_t *lock) {return raw_spin_is_locked (&lock->rlock);
include/linux/spinlock_up.h +0 -6
@@ -26,11 +26,6 @@#ifdef CONFIG_DEBUG_SPINLOCK #define arch_spin_is_locked (x) ((x)->slock == 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock,VAL ); -} -static inline void arch_spin_lock (arch_spinlock_t *lock) { lock->slock =0 ;
@@ -73,7 +68,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)#else /* DEBUG_SPINLOCK */ #define arch_spin_is_locked (lock) ((void)(lock), 0) -#define arch_spin_unlock_wait (lock) do { barrier(); (void)(lock); } while (0)/* for sched/core.c and kernel_lock.c: */ # define arch_spin_lock (lock) do { barrier(); (void)(lock); } while (0)# define arch_spin_lock_flags (lock, flags) do { barrier(); (void)(lock); } while (0)
kernel/locking/qspinlock.c +0 -117
123 lines not on this page · raw message
Signature · 1 line
2.5.2
PATCH v2 10/10 arch: Remove spin_unlock_wait() arch-specific definitions
In reply to Paul E. McKenneyThere is no agreed-upon definition of spin_unlock_wait()'s semantics, and it appears that all callers could do just as well with a lock/unlock pair. This commit therefore removes the underlying arch-specific arch_spin_unlock_wait() for all architectures providing them.
--- arch/alpha/include/asm/spinlock.h | 5 ---- arch/arc/include/asm/spinlock.h | 5 ---- arch/arm/include/asm/spinlock.h | 16 ---------- arch/arm64/include/asm/spinlock.h | 58 ++++-------------------------------- arch/blackfin/include/asm/spinlock.h | 5 ---- arch/hexagon/include/asm/spinlock.h | 5 ---- arch/ia64/include/asm/spinlock.h | 21 ------------- arch/m32r/include/asm/spinlock.h | 5 ---- arch/metag/include/asm/spinlock.h | 5 ---- arch/mn10300/include/asm/spinlock.h | 5 ---- arch/parisc/include/asm/spinlock.h | 7 ----- arch/powerpc/include/asm/spinlock.h | 33 -------------------- arch/s390/include/asm/spinlock.h | 7 ----- arch/sh/include/asm/spinlock-cas.h | 5 ---- arch/sh/include/asm/spinlock-llsc.h | 5 ---- arch/sparc/include/asm/spinlock_32.h | 5 ---- arch/tile/include/asm/spinlock_32.h | 2 -- arch/tile/include/asm/spinlock_64.h | 2 -- arch/tile/lib/spinlock_32.c | 23 -------------- arch/tile/lib/spinlock_64.c | 22 -------------- arch/xtensa/include/asm/spinlock.h | 5 ---- 21 files changed, 5 insertions(+), 241 deletions(-)
arch/alpha/include/asm/spinlock.h +0 -5
@@ -16,11 +16,6 @@#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)#define arch_spin_is_locked (x) ((x)->lock != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -static inline int arch_spin_value_unlocked (arch_spinlock_t lock) {return lock.lock ==0 ;
arch/arc/include/asm/spinlock.h +0 -5
@@ -16,11 +16,6 @@#define arch_spin_is_locked (x) ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock, !VAL ); -} -#ifdef CONFIG_ARC_HAS_LLSC static inline void arch_spin_lock (arch_spinlock_t *lock)
arch/arm/include/asm/spinlock.h +0 -16
@@ -52,22 +52,6 @@ static inline void dsb_sev(void)* memory. */ -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -u16 owner =READ_ONCE (lock->tickets.owner); - -for (;;) { -arch_spinlock_t tmp =READ_ONCE (*lock); - -if (tmp.tickets.owner == tmp.tickets.next || - tmp.tickets.owner != owner) -break ; - -wfe (); - } -smp_acquire__after_ctrl_dep (); -} -#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)static inline void arch_spin_lock (arch_spinlock_t *lock)
arch/arm64/include/asm/spinlock.h +5 -53
70 lines not on this page · raw message
arch/blackfin/include/asm/spinlock.h +0 -5
@@ -48,11 +48,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)__raw_spin_unlock_asm (&lock->lock); } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -static inline int arch_read_can_lock (arch_rwlock_t *rw) {return __raw_uncached_fetch_asm (&rw->lock) >0 ;
arch/hexagon/include/asm/spinlock.h +0 -5
@@ -179,11 +179,6 @@ static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)*/ #define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -#define arch_spin_is_locked (x) ((x)->lock != 0)#define arch_read_lock_flags (lock, flags) arch_read_lock(lock)
arch/ia64/include/asm/spinlock.h +0 -21
@@ -76,22 +76,6 @@ static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)ACCESS_ONCE (*p) = (tmp +2 ) & ~1 ; } -static __always_inline void __ticket_spin_unlock_wait(arch_spinlock_t *lock) -{ -int *p = (int *)&lock->lock, ticket; - -ia64_invala (); - -for (;;) { - asmvolatile ("ld4.c.nc %0=[%1]" :"=r" (ticket) :"r" (p) :"memory" ); -if (!(((ticket >>TICKET_SHIFT ) ^ ticket) &TICKET_MASK )) -return ; -cpu_relax (); - } - -smp_acquire__after_ctrl_dep (); -} -static inline int __ticket_spin_is_locked (arch_spinlock_t *lock) {long tmp =ACCESS_ONCE (lock->lock);
@@ -143,11 +127,6 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,arch_spin_lock (lock); } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -__ticket_spin_unlock_wait (lock); -} -#define arch_read_can_lock (rw) (*(volatile int *)(rw) >= 0)#define arch_write_can_lock (rw) (*(volatile int *)(rw) == 0)
arch/m32r/include/asm/spinlock.h +0 -5
@@ -30,11 +30,6 @@#define arch_spin_is_locked (x) (*(volatile int *)(&(x)->slock) <= 0)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock,VAL >0 ); -} -/** * arch_spin_trylock - Try spin lock and return a result * @lock: Pointer to the lock variable
arch/metag/include/asm/spinlock.h +0 -5
@@ -15,11 +15,6 @@* locked. */ -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)#define arch_read_lock_flags (lock, flags) arch_read_lock(lock)
arch/mn10300/include/asm/spinlock.h +0 -5
@@ -26,11 +26,6 @@#define arch_spin_is_locked (x) (*(volatile signed char *)(&(x)->slock) != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock, !VAL ); -} -static inline void arch_spin_unlock (arch_spinlock_t *lock) { asm volatile(
arch/parisc/include/asm/spinlock.h +0 -7
@@ -14,13 +14,6 @@ static inline int arch_spin_is_locked(arch_spinlock_t *x)#define arch_spin_lock (lock) arch_spin_lock_flags(lock, 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *x) -{ -volatile unsigned int *a =__ldcw_align (x); - -smp_cond_load_acquire (a,VAL ); -} -static inline void arch_spin_lock_flags (arch_spinlock_t *x,unsigned long flags) {
arch/powerpc/include/asm/spinlock.h +0 -33
@@ -170,39 +170,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->slock =0 ; } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -arch_spinlock_t lock_val; - -smp_mb (); - -/* -* Atomically load and store back the lock value (unchanged). This -* ensures that our observation of the lock value is ordered with -* respect to other lock operations. -*/ - __asm__ __volatile__( -"1: " PPC_LWARX(%0, 0, %2, 0) "\n" -" stwcx. %0, 0, %2\n" -" bne- 1b\n" - : "=&r" (lock_val), "+m" (*lock) - : "r" (lock) - : "cr0", "xer"); - - if (arch_spin_value_unlocked(lock_val)) - goto out; - -while (lock->slock) { -HMT_low (); -if (SHARED_PROCESSOR ) -__spin_yield (lock); - } -HMT_medium (); - -out: -smp_mb (); -} -/* * Read-write spinlocks, allowing multiple readers * but only one writer.
arch/s390/include/asm/spinlock.h +0 -7
@@ -98,13 +98,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lp) : "cc", "memory"); } -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -while (arch_spin_is_locked (lock)) -arch_spin_relax (lock); -smp_acquire__after_ctrl_dep (); -} -/* * Read-write spinlocks, allowing multiple readers * but only one writer.
arch/sh/include/asm/spinlock-cas.h +0 -5
@@ -29,11 +29,6 @@ static inline unsigned __sl_cas(volatile unsigned *p, unsigned old, unsigned new#define arch_spin_is_locked (x) ((x)->lock <= 0)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock,VAL >0 ); -} -static inline void arch_spin_lock (arch_spinlock_t *lock) {while (!__sl_cas (&lock->lock,1 ,0 ));
arch/sh/include/asm/spinlock-llsc.h +0 -5
@@ -21,11 +21,6 @@#define arch_spin_is_locked (x) ((x)->lock <= 0)#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock,VAL >0 ); -} -/* * Simple spin lock operations. There are two variants, one clears IRQ's * on the local processor, one does not.
arch/sparc/include/asm/spinlock_32.h +0 -5
@@ -14,11 +14,6 @@#define arch_spin_is_locked (lock) (*((volatile unsigned char *)(lock)) != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->lock, !VAL ); -} -static inline void arch_spin_lock (arch_spinlock_t *lock) { __asm__ __volatile__(
arch/tile/include/asm/spinlock_32.h +0 -2
@@ -64,8 +64,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->current_ticket = old_ticket +TICKET_QUANTUM ; } -void arch_spin_unlock_wait (arch_spinlock_t *lock); -/* * Read-write spinlocks, allowing multiple readers * but only one writer.
arch/tile/include/asm/spinlock_64.h +0 -2
@@ -58,8 +58,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock)__insn_fetchadd4 (&lock->lock,1U << __ARCH_SPIN_CURRENT_SHIFT); } -void arch_spin_unlock_wait (arch_spinlock_t *lock); -void arch_spin_lock_slow (arch_spinlock_t *lock,u32 val);/* Grab the "next" ticket number and bump it atomically.
arch/tile/lib/spinlock_32.c +0 -23
@@ -62,29 +62,6 @@ int arch_spin_trylock(arch_spinlock_t *lock) }EXPORT_SYMBOL (arch_spin_trylock); -void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -u32 iterations =0 ; -int curr =READ_ONCE (lock->current_ticket); -int next =READ_ONCE (lock->next_ticket); - -/* Return immediately if unlocked. */ -if (next == curr) -return ; - -/* Wait until the current locker has released the lock. */ -do { -delay_backoff (iterations++); - }while (READ_ONCE (lock->current_ticket) == curr); - -/* -* The TILE architecture doesn't do read speculation; therefore -* a control dependency guarantees a LOAD->{LOAD,STORE} order. -*/ -barrier (); -} -EXPORT_SYMBOL (arch_spin_unlock_wait); -/* * The low byte is always reserved to be the marker for a "tns" operation * since the low bit is set to "1" by a tns. The next seven bits are
arch/tile/lib/spinlock_64.c +0 -22
@@ -62,28 +62,6 @@ int arch_spin_trylock(arch_spinlock_t *lock) }EXPORT_SYMBOL (arch_spin_trylock); -void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -u32 iterations =0 ; -u32 val =READ_ONCE (lock->lock); -u32 curr =arch_spin_current (val); - -/* Return immediately if unlocked. */ -if (arch_spin_next (val) == curr) -return ; - -/* Wait until the current locker has released the lock. */ -do { -delay_backoff (iterations++); - }while (arch_spin_current (READ_ONCE (lock->lock)) == curr); - -/* -* The TILE architecture doesn't do read speculation; therefore -* a control dependency guarantees a LOAD->{LOAD,STORE} order. -*/ -barrier (); -} -EXPORT_SYMBOL (arch_spin_unlock_wait);/* * If the read lock fails due to a writer, we retry periodically
arch/xtensa/include/asm/spinlock.h +0 -5
@@ -33,11 +33,6 @@#define arch_spin_is_locked (x) ((x)->slock != 0) -static inline void arch_spin_unlock_wait (arch_spinlock_t *lock) -{ -smp_cond_load_acquire (&lock->slock, !VAL ); -} -#define arch_spin_lock_flags (lock, flags) arch_spin_lock(lock)static inline void arch_spin_lock (arch_spinlock_t *lock)
Signature · 1 line
2.5.2