locking/qrwlock: Change "queue rwlock" to "queued rwlock"
Subsystems (2)
Waiman Long
reviewerPATCH 1/2 locking/qrwlock: Change "queue rwlock" to "queued rwlock"
Queued rwlock was originally named "queue rwlock" which wasn't quite grammatically correct. However there are still some "queue rwlock" references in the code. Change those to "queued rwlock" for consistency.
--- include/asm-generic/qrwlock.h | 28 ++++++++++++++-------------- include/asm-generic/qrwlock_types.h | 2 +- kernel/locking/qrwlock.c | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-)
include/asm-generic/qrwlock.h +14 -14
@@ -33,8 +33,8 @@ extern void queued_read_lock_slowpath(struct qrwlock *lock);extern void queued_write_lock_slowpath (struct qrwlock *lock);/** -* queued_read_trylock - try to acquire read lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_read_trylock - try to acquire read lock of a queued rwlock +* @lock : Pointer to queued rwlock structure * Return: 1 if lock acquired, 0 if failed */ static inline int queued_read_trylock (struct qrwlock *lock)
@@ -52,8 +52,8 @@ static inline int queued_read_trylock(struct qrwlock *lock) }/** -* queued_write_trylock - try to acquire write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_trylock - try to acquire write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure * Return: 1 if lock acquired, 0 if failed */ static inline int queued_write_trylock (struct qrwlock *lock)
@@ -68,8 +68,8 @@ static inline int queued_write_trylock(struct qrwlock *lock) _QW_LOCKED)); }/** -* queued_read_lock - acquire read lock of a queue rwlock -* @lock: Pointer to queue rwlock structure +* queued_read_lock - acquire read lock of a queued rwlock +* @lock: Pointer to queued rwlock structure */ static inline void queued_read_lock (struct qrwlock *lock) {
@@ -84,8 +84,8 @@ static inline void queued_read_lock(struct qrwlock *lock) }/** -* queued_write_lock - acquire write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_lock - acquire write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ static inline void queued_write_lock (struct qrwlock *lock) {
@@ -98,8 +98,8 @@ static inline void queued_write_lock(struct qrwlock *lock) }/** -* queued_read_unlock - release read lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_read_unlock - release read lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ static inline void queued_read_unlock (struct qrwlock *lock) {
@@ -110,8 +110,8 @@ static inline void queued_read_unlock(struct qrwlock *lock) }/** -* queued_write_unlock - release write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_unlock - release write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ static inline void queued_write_unlock (struct qrwlock *lock) {
@@ -120,7 +120,7 @@ static inline void queued_write_unlock(struct qrwlock *lock)/** * queued_rwlock_is_contended - check if the lock is contended -* @lock : Pointer to queue rwlock structure +* @lock : Pointer to queued rwlock structure * Return: 1 if lock contended, 0 otherwise */ static inline int queued_rwlock_is_contended (struct qrwlock *lock)
@@ -130,7 +130,7 @@ static inline int queued_rwlock_is_contended(struct qrwlock *lock)/* * Remapping rwlock architecture specific functions to the corresponding -* queue rwlock functions. +* queued rwlock functions. */ #define arch_read_lock (l) queued_read_lock(l)#define arch_write_lock (l) queued_write_lock(l)
include/asm-generic/qrwlock_types.h +1 -1
@@ -7,7 +7,7 @@#include <asm/spinlock_types.h> /* -* The queue read/write lock data structure +* The queued read/write lock data structure */ typedef struct qrwlock {
kernel/locking/qrwlock.c +4 -4
@@ -15,8 +15,8 @@#include <trace/events/lock.h> /** -* queued_read_lock_slowpath - acquire read lock of a queue rwlock -* @lock: Pointer to queue rwlock structure +* queued_read_lock_slowpath - acquire read lock of a queued rwlock +* @lock: Pointer to queued rwlock structure */ void queued_read_lock_slowpath (struct qrwlock *lock) {
@@ -60,8 +60,8 @@ void queued_read_lock_slowpath(struct qrwlock *lock)EXPORT_SYMBOL (queued_read_lock_slowpath);/** -* queued_write_lock_slowpath - acquire write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_lock_slowpath - acquire write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ void queued_write_lock_slowpath (struct qrwlock *lock) {
Signature · 1 line
2.27.0
Waiman Long
reviewerPATCH 2/2 locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
Even though qrwlock is supposed to be a fair lock, it does allow readers from interrupt context to spin on the lock until it can acquire it making it not as fair. This exception was added due to the requirement to allow recursive read lock in interrupt context. This can also be achieved by just ignoring the writer waiting bit without spinning on the lock.
By making this change, we make qrwlock a bit more fair and eliminating the problem of cacheline bouncing for rwlocks that are used heavily in interrupt context, like the networking stack. This should also reduce the chance of lock starvation for those interrupt context rwlocks.
--- include/asm-generic/qrwlock.h | 6 +++--- kernel/locking/qrwlock.c | 17 ++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-)
include/asm-generic/qrwlock.h +3 -3
@@ -29,7 +29,7 @@/* * External function declarations */ -extern void queued_read_lock_slowpath (struct qrwlock *lock); +extern void queued_read_lock_slowpath (struct qrwlock *lock,int cnts);extern void queued_write_lock_slowpath (struct qrwlock *lock);/**
@@ -80,7 +80,7 @@ static inline void queued_read_lock(struct qrwlock *lock)return ;/* The slowpath will decrement the reader count, if necessary. */ -queued_read_lock_slowpath (lock); +queued_read_lock_slowpath (lock, cnts); }/**
@@ -90,7 +90,7 @@ static inline void queued_read_lock(struct qrwlock *lock)static inline void queued_write_lock (struct qrwlock *lock) {int cnts =0 ; -/* Optimize for the unfair lock case where the fair flag is 0. */ +if (likely (atomic_try_cmpxchg_acquire (&lock->cnts, &cnts, _QW_LOCKED)))return ;
kernel/locking/qrwlock.c +6 -11
@@ -18,21 +18,16 @@* queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath (struct qrwlock *lock) +void queued_read_lock_slowpath (struct qrwlock *lock,int cnts) {/* -* Readers come here when they cannot get the lock without waiting +* Readers come here when they cannot get the lock without waiting. +* Readers in interrupt context can steal the lock immediately +* if the writer is just waiting (not holding the lock yet). */ -if (unlikely (in_interrupt ())) { -/* -* Readers in interrupt context will get the lock immediately -* if the writer is just waiting (not holding the lock yet), -* so spin with ACQUIRE semantics until the lock is available -* without waiting in the queue. -*/ -atomic_cond_read_acquire (&lock->cnts, !(VAL & _QW_LOCKED)); +if (unlikely (!(cnts & _QW_LOCKED) &&in_interrupt ()))return ; - } +atomic_sub (_QR_BIAS, &lock->cnts);trace_contention_begin (lock,LCB_F_SPIN |LCB_F_READ );
Signature · 1 line
2.27.0
Re: [PATCH 2/2] locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
Quoted 12 lines · Waiman Long
Even though qrwlock is supposed to be a fair lock, it does allow readers from interrupt context to spin on the lock until it can acquire it making it not as fair. This exception was added due to the requirement to allow recursive read lock in interrupt context. This can also be achieved by just ignoring the writer waiting bit without spinning on the lock.
By making this change, we make qrwlock a bit more fair and eliminating the problem of cacheline bouncing for rwlocks that are used heavily in interrupt context, like the networking stack. This should also reduce the chance of lock starvation for those interrupt context rwlocks.
Quoted 5 lines
Signed-off-by Waiman Long <[email protected]>--- include/asm-generic/qrwlock.h | 6 +++--- kernel/locking/qrwlock.c | 17 ++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-)
I have no opinion on the change itself, but I made sure this does not conflict with the generic ticket spinlock changes that I merged in the asm-generic tree, since those also touch the comments in qrwlock.h [1]
I assume you are merging both patches through the tip tree, so
If you want me to pick them up in the asm-generic tree instead, just let me know.
Arnd
Waiman Long
reviewerRe: [PATCH 2/2] locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
Quoted 17 lines · Arnd Bergmann
Quoted 11 lines · Waiman Long
Even though qrwlock is supposed to be a fair lock, it does allow readers from interrupt context to spin on the lock until it can acquire it making it not as fair. This exception was added due to the requirement to allow recursive read lock in interrupt context. This can also be achieved by just ignoring the writer waiting bit without spinning on the lock.
By making this change, we make qrwlock a bit more fair and eliminating the problem of cacheline bouncing for rwlocks that are used heavily in interrupt context, like the networking stack. This should also reduce the chance of lock starvation for those interrupt context rwlocks.
Quoted 5 lines
Signed-off-by Waiman Long <[email protected]>--- include/asm-generic/qrwlock.h | 6 +++--- kernel/locking/qrwlock.c | 17 ++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-)
Quoted 6 lines
I have no opinion on the change itself, but I made sure this does not conflict with the generic ticket spinlock changes that I merged in the asm-generic tree, since those also touch the comments in
qrwlock.h[1]I assume you are merging both patches through the tip tree, so
Yes, it is based on the latest tip tree.
Cheers,
Longman
Peter Zijlstra
maintainerRe: [PATCH 2/2] locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
In reply to Waiman LongQuoted 33 lines · Waiman Long
Even though qrwlock is supposed to be a fair lock, it does allow readers from interrupt context to spin on the lock until it can acquire it making it not as fair. This exception was added due to the requirement to allow recursive read lock in interrupt context. This can also be achieved by just ignoring the writer waiting bit without spinning on the lock.
By making this change, we make qrwlock a bit more fair and eliminating the problem of cacheline bouncing for rwlocks that are used heavily in interrupt context, like the networking stack. This should also reduce the chance of lock starvation for those interrupt context rwlocks.
kernel/locking/qrwlock.c +4
-7@@ -18,21 +18,16 @@* queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath (struct qrwlock *lock) +void queued_read_lock_slowpath (struct qrwlock *lock,int cnts) {/* -* Readers come here when they cannot get the lock without waiting +* Readers come here when they cannot get the lock without waiting. +* Readers in interrupt context can steal the lock immediately +* if the writer is just waiting (not holding the lock yet). */ -if (unlikely (in_interrupt ())) { -/* -* Readers in interrupt context will get the lock immediately -* if the writer is just waiting (not holding the lock yet), -* so spin with ACQUIRE semantics until the lock is available
kernel/locking/qrwlock.c line 33
Quoted 10 lines
kernel/locking/qrwlock.c +2
-4-* without waiting in the queue. -*/ -atomic_cond_read_acquire (&lock->cnts, !(VAL & _QW_LOCKED)); +if (unlikely (!(cnts & _QW_LOCKED) &&in_interrupt ()))return ; - } +atomic_sub (_QR_BIAS, &lock->cnts);trace_contention_begin (lock,LCB_F_SPIN |LCB_F_READ );
I'm confused; prior to this change:
CPU0 CPU1 write_lock_irq(&l) read_lock(&l) <INRQ> read_lock(&l) ...
was not deadlock, but now it would AFAICT.
Waiman Long
reviewerRe: [PATCH 2/2] locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
Quoted 53 lines · Peter Zijlstra
Quoted 32 lines · Waiman Long
Even though qrwlock is supposed to be a fair lock, it does allow readers from interrupt context to spin on the lock until it can acquire it making it not as fair. This exception was added due to the requirement to allow recursive read lock in interrupt context. This can also be achieved by just ignoring the writer waiting bit without spinning on the lock.
By making this change, we make qrwlock a bit more fair and eliminating the problem of cacheline bouncing for rwlocks that are used heavily in interrupt context, like the networking stack. This should also reduce the chance of lock starvation for those interrupt context rwlocks.
kernel/locking/qrwlock.c +4
-7@@ -18,21 +18,16 @@* queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath (struct qrwlock *lock) +void queued_read_lock_slowpath (struct qrwlock *lock,int cnts) {/* -* Readers come here when they cannot get the lock without waiting +* Readers come here when they cannot get the lock without waiting. +* Readers in interrupt context can steal the lock immediately +* if the writer is just waiting (not holding the lock yet). */ -if (unlikely (in_interrupt ())) { -/* -* Readers in interrupt context will get the lock immediately -* if the writer is just waiting (not holding the lock yet), -* so spin with ACQUIRE semantics until the lock is available kernel/locking/qrwlock.c line 33
Quoted 10 lines
kernel/locking/qrwlock.c +2
-4-* without waiting in the queue. -*/ -atomic_cond_read_acquire (&lock->cnts, !(VAL & _QW_LOCKED)); +if (unlikely (!(cnts & _QW_LOCKED) &&in_interrupt ()))return ; - } +atomic_sub (_QR_BIAS, &lock->cnts);trace_contention_begin (lock,LCB_F_SPIN |LCB_F_READ );I'm confused; prior to this change:
CPU0 CPU1 write_lock_irq(&l) read_lock(&l) <INRQ> read_lock(&l) ...
Quoted 1 line
was not deadlock, but now it would AFAICT.
Oh you are right. I missed that scenario in my analysis. My bad.
Please scrap this patch. Patch 1 is just an update to the comment and so is still applicable.
Thanks,
Longman
Peter Zijlstra
maintainerRe: [PATCH 2/2] locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
Quoted 10 lines · Waiman Long
Quoted 10 lines
I'm confused; prior to this change:
CPU0 CPU1 write_lock_irq(&l) read_lock(&l) <INRQ> read_lock(&l) ...
Quoted 3 lines
Quoted 1 line
was not deadlock, but now it would AFAICT.
Oh you are right. I missed that scenario in my analysis. My bad.
No worries; I suppose we can also still do something like:
void queued_read_lock_slowpath(struct qrwlock *lock, int cnts)
{
/*
* the big comment
*/
if (unlikely(in_interrupt())) {
/*
* If not write-locked, insta-grant the reader
*/
if (!(cnts & _QW_LOCKED))
return;
/*
* otherwise, wait for the writer to go away.
*/
atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
return;
}
...
}Which saves one load in some cases... not sure it's worth it though.
Waiman Long
reviewerRe: [PATCH 2/2] locking/qrwlock: Reduce cacheline contention for rwlocks used in interrupt context
Quoted 38 lines · Peter Zijlstra
Quoted 4 lines · Waiman Long
Quoted 4 lines
I'm confused; prior to this change:
CPU0 CPU1Quoted 8 lines
Quoted 7 lines
write_lock_irq(&l) read_lock(&l) <INRQ> read_lock(&l) ...was not deadlock, but now it would AFAICT.
Oh you are right. I missed that scenario in my analysis. My bad.
No worries; I suppose we can also still do something like:
void
queued_read_lock_slowpath(struct qrwlock *lock, int cnts){ /* * the big comment */ if (unlikely(in_interrupt())) { /* * If not write-locked, insta-grant the reader */ if (!(cnts & _QW_LOCKED)) return; /* * otherwise, wait for the writer to go away. */ atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); return; } ... }
Quoted 1 line
Which saves one load in some cases... not sure it's worth it though.
Yes, it is a micro-optimization that can be done. The gain, if any, should be minor though.
Cheers,
Longman
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 434e09e7575b02e014931bc5672289fabd7a825c Gitweb: https://git.kernel.org/tip/434e09e7575b02e014931bc5672289fabd7a825c Author: Waiman Long <[email protected]> AuthorDate: Tue, 10 May 2022 15:21:33 -04:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Wed, 11 May 2022 16:27:04 +02:00
locking/qrwlock: Change "queue rwlock" to "queued rwlock"
Queued rwlock was originally named "queue rwlock" which wasn't quite grammatically correct. However there are still some "queue rwlock" references in the code. Change those to "queued rwlock" for consistency.
--- include/asm-generic/qrwlock.h | 28 ++++++++++++++-------------- include/asm-generic/qrwlock_types.h | 2 +- kernel/locking/qrwlock.c | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-)
include/asm-generic/qrwlock.h +14 -14
@@ -33,8 +33,8 @@ extern void queued_read_lock_slowpath(struct qrwlock *lock);extern void queued_write_lock_slowpath (struct qrwlock *lock);/** -* queued_read_trylock - try to acquire read lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_read_trylock - try to acquire read lock of a queued rwlock +* @lock : Pointer to queued rwlock structure * Return: 1 if lock acquired, 0 if failed */ static inline int queued_read_trylock (struct qrwlock *lock)
@@ -52,8 +52,8 @@ static inline int queued_read_trylock(struct qrwlock *lock) }/** -* queued_write_trylock - try to acquire write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_trylock - try to acquire write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure * Return: 1 if lock acquired, 0 if failed */ static inline int queued_write_trylock (struct qrwlock *lock)
@@ -68,8 +68,8 @@ static inline int queued_write_trylock(struct qrwlock *lock) _QW_LOCKED)); }/** -* queued_read_lock - acquire read lock of a queue rwlock -* @lock: Pointer to queue rwlock structure +* queued_read_lock - acquire read lock of a queued rwlock +* @lock: Pointer to queued rwlock structure */ static inline void queued_read_lock (struct qrwlock *lock) {
@@ -84,8 +84,8 @@ static inline void queued_read_lock(struct qrwlock *lock) }/** -* queued_write_lock - acquire write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_lock - acquire write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ static inline void queued_write_lock (struct qrwlock *lock) {
@@ -98,8 +98,8 @@ static inline void queued_write_lock(struct qrwlock *lock) }/** -* queued_read_unlock - release read lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_read_unlock - release read lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ static inline void queued_read_unlock (struct qrwlock *lock) {
@@ -110,8 +110,8 @@ static inline void queued_read_unlock(struct qrwlock *lock) }/** -* queued_write_unlock - release write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_unlock - release write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ static inline void queued_write_unlock (struct qrwlock *lock) {
@@ -120,7 +120,7 @@ static inline void queued_write_unlock(struct qrwlock *lock)/** * queued_rwlock_is_contended - check if the lock is contended -* @lock : Pointer to queue rwlock structure +* @lock : Pointer to queued rwlock structure * Return: 1 if lock contended, 0 otherwise */ static inline int queued_rwlock_is_contended (struct qrwlock *lock)
@@ -130,7 +130,7 @@ static inline int queued_rwlock_is_contended(struct qrwlock *lock)/* * Remapping rwlock architecture specific functions to the corresponding -* queue rwlock functions. +* queued rwlock functions. */ #define arch_read_lock (l) queued_read_lock(l)#define arch_write_lock (l) queued_write_lock(l)
include/asm-generic/qrwlock_types.h +1 -1
@@ -7,7 +7,7 @@#include <asm/spinlock_types.h> /* -* The queue read/write lock data structure +* The queued read/write lock data structure */ typedef struct qrwlock {
kernel/locking/qrwlock.c +4 -4
@@ -15,8 +15,8 @@#include <trace/events/lock.h> /** -* queued_read_lock_slowpath - acquire read lock of a queue rwlock -* @lock: Pointer to queue rwlock structure +* queued_read_lock_slowpath - acquire read lock of a queued rwlock +* @lock: Pointer to queued rwlock structure */ void queued_read_lock_slowpath (struct qrwlock *lock) {
@@ -60,8 +60,8 @@ void queued_read_lock_slowpath(struct qrwlock *lock)EXPORT_SYMBOL (queued_read_lock_slowpath);/** -* queued_write_lock_slowpath - acquire write lock of a queue rwlock -* @lock : Pointer to queue rwlock structure +* queued_write_lock_slowpath - acquire write lock of a queued rwlock +* @lock : Pointer to queued rwlock structure */ void queued_write_lock_slowpath (struct qrwlock *lock) {