locking: Add __lockfunc to slow path functions
Subsystems (2)
PATCH v2 1/1 locking: Add __lockfunc to slow path functions
So that we can skip the functions in the perf lock contention and other places like /proc/PID/wchan.
Signed-off-by Namhyung Kim <[email protected]>
---
* annotate pv qspinlock functions too
arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------ kernel/locking/qrwlock.c | 4 ++-- kernel/locking/qspinlock.c | 2 +- kernel/locking/qspinlock_paravirt.h | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-)
arch/x86/include/asm/qspinlock_paravirt.h +7 -6
@@ -12,7 +12,7 @@*/ #ifdef CONFIG_64BIT -PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock_slowpath); +__PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock_slowpath,".spinlock.text" );#define __pv_queued_spin_unlock __pv_queued_spin_unlock#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);/* * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock * which combines the registers saving trunk and the body of the following -* C code: +* C code. Note that it puts the code in the .spinlock.text section which +* is equivalent to adding __lockfunc in the C code: * -* void __pv_queued_spin_unlock(struct qspinlock *lock) +* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock) * { * u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0); *
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);* rsi = lockval (second argument) * rdx = internal variable (set to 0) */ -asm (".pushsection .text;" +asm (".pushsection .spinlock.text;" ".globl " PV_UNLOCK ";" ".type " PV_UNLOCK ", @function;" ".align 4,0x90;"
@@ -65,8 +66,8 @@ asm (".pushsection .text;"#else /* CONFIG_64BIT */ -extern void __pv_queued_spin_unlock (struct qspinlock *lock); -PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock); +extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock); +__PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock,".spinlock.text" );#endif /* CONFIG_64BIT */ #endif
kernel/locking/qrwlock.c +2 -2
@@ -18,7 +18,7 @@* queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath (struct qrwlock *lock) +void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock) {/* * Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);* queued_write_lock_slowpath - acquire write lock of a queued rwlock * @lock : Pointer to queued rwlock structure */ -void queued_write_lock_slowpath (struct qrwlock *lock) +void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock) {int cnts;
kernel/locking/qspinlock.c +1 -1
@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' : * queue : ^--' : */ -void queued_spin_lock_slowpath (struct qspinlock *lock,u32 val) +void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) {struct mcs_spinlock *prev, *next, *node;u32 old, tail;
kernel/locking/qspinlock_paravirt.h +2 -2
@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)* PV versions of the unlock fastpath and slowpath functions to be used * instead of queued_spin_unlock(). */ -__visible void +__visible __lockfunc void__pv_queued_spin_unlock_slowpath (struct qspinlock *lock,u8 locked) {struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)#include <asm/qspinlock_paravirt.h> #ifndef __pv_queued_spin_unlock -__visible void __pv_queued_spin_unlock(struct qspinlock *lock) +__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock) {u8 locked;
Signature · 1 line
2.37.1.559.g78731f0fdb-goog
Quoted 108 lines · Namhyung Kim
So that we can skip the functions in the perf lock contention and other places like
/proc/PID/wchan.Signed-off-by Namhyung Kim <[email protected]>---
* annotate pv qspinlock functions tooarch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------ kernel/locking/qrwlock.c | 4 ++-- kernel/locking/qspinlock.c | 2 +- kernel/locking/qspinlock_paravirt.h | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-)arch/x86/include/asm/qspinlock_paravirt.h +7
-6@@ -12,7 +12,7 @@*/ #ifdef CONFIG_64BIT -PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock_slowpath); +__PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock_slowpath,".spinlock.text" );#define __pv_queued_spin_unlock __pv_queued_spin_unlock#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);/* * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock * which combines the registers saving trunk and the body of the following -* C code: +* C code. Note that it puts the code in the .spinlock.text section which +* is equivalent to adding __lockfunc in the C code: * -* void __pv_queued_spin_unlock(struct qspinlock *lock) +* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock) * { * u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0); * @@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);* rsi = lockval (second argument) * rdx = internal variable (set to 0) */ -asm (".pushsection .text;" +asm (".pushsection .spinlock.text;" ".globl " PV_UNLOCK ";" ".type " PV_UNLOCK ", @function;" ".align 4,0x90;"@@ -65,8 +66,8 @@ asm (".pushsection .text;"#else /* CONFIG_64BIT */ -extern void __pv_queued_spin_unlock (struct qspinlock *lock); -PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock); +extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock); +__PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock,".spinlock.text" );#endif /* CONFIG_64BIT */ #endif kernel/locking/qrwlock.c +2
-2@@ -18,7 +18,7 @@* queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath (struct qrwlock *lock) +void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock) {/* * Readers come here when they cannot get the lock without waiting @@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);* queued_write_lock_slowpath - acquire write lock of a queued rwlock * @lock : Pointer to queued rwlock structure */ -void queued_write_lock_slowpath (struct qrwlock *lock) +void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock) {int cnts;kernel/locking/qspinlock.c +1
-1@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' : * queue : ^--' : */ -void queued_spin_lock_slowpath (struct qspinlock *lock,u32 val) +void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) {struct mcs_spinlock *prev, *next, *node;u32 old, tail;kernel/locking/qspinlock_paravirt.h +1
-1@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)* PV versions of the unlock fastpath and slowpath functions to be used * instead of queued_spin_unlock(). */ -__visible void +__visible __lockfunc void__pv_queued_spin_unlock_slowpath (struct qspinlock *lock,u8 locked) {struct pv_node *node;
kernel/locking/qspinlock_paravirt.h __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked) · line 550
Quoted 8 lines
kernel/locking/qspinlock_paravirt.h +1
-1@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)#include <asm/qspinlock_paravirt.h> #ifndef __pv_queued_spin_unlock -__visible void __pv_queued_spin_unlock(struct qspinlock *lock) +__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock) {u8 locked;
Acked-by Waiman Long <[email protected]>
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 501f7f69bca195da266de83eb2c26c30813fba97 Gitweb: https://git.kernel.org/tip/501f7f69bca195da266de83eb2c26c30813fba97 Author: Namhyung Kim <[email protected]> AuthorDate: Wed, 10 Aug 2022 15:03:46 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 19 Aug 2022 19:47:51 +02:00
locking: Add __lockfunc to slow path functions
So that we can skip the functions in the perf lock contention and other places like /proc/PID/wchan.
Signed-off-by Namhyung Kim <[email protected]>
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Waiman Long <[email protected]>
--- arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------ kernel/locking/qrwlock.c | 4 ++-- kernel/locking/qspinlock.c | 2 +- kernel/locking/qspinlock_paravirt.h | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-)
arch/x86/include/asm/qspinlock_paravirt.h +7 -6
@@ -12,7 +12,7 @@*/ #ifdef CONFIG_64BIT -PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock_slowpath); +__PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock_slowpath,".spinlock.text" );#define __pv_queued_spin_unlock __pv_queued_spin_unlock#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);/* * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock * which combines the registers saving trunk and the body of the following -* C code: +* C code. Note that it puts the code in the .spinlock.text section which +* is equivalent to adding __lockfunc in the C code: * -* void __pv_queued_spin_unlock(struct qspinlock *lock) +* void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock) * { * u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0); *
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);* rsi = lockval (second argument) * rdx = internal variable (set to 0) */ -asm (".pushsection .text;" +asm (".pushsection .spinlock.text;" ".globl " PV_UNLOCK ";" ".type " PV_UNLOCK ", @function;" ".align 4,0x90;"
@@ -65,8 +66,8 @@ asm (".pushsection .text;"#else /* CONFIG_64BIT */ -extern void __pv_queued_spin_unlock (struct qspinlock *lock); -PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock); +extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock); +__PV_CALLEE_SAVE_REGS_THUNK (__pv_queued_spin_unlock,".spinlock.text" );#endif /* CONFIG_64BIT */ #endif
kernel/locking/qrwlock.c +2 -2
@@ -18,7 +18,7 @@* queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath (struct qrwlock *lock) +void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock) {/* * Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);* queued_write_lock_slowpath - acquire write lock of a queued rwlock * @lock : Pointer to queued rwlock structure */ -void queued_write_lock_slowpath (struct qrwlock *lock) +void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock) {int cnts;
kernel/locking/qspinlock.c +1 -1
@@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' : * queue : ^--' : */ -void queued_spin_lock_slowpath (struct qspinlock *lock,u32 val) +void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) {struct mcs_spinlock *prev, *next, *node;u32 old, tail;
kernel/locking/qspinlock_paravirt.h +2 -2
@@ -489,7 +489,7 @@ gotlock:* PV versions of the unlock fastpath and slowpath functions to be used * instead of queued_spin_unlock(). */ -__visible void +__visible __lockfunc void__pv_queued_spin_unlock_slowpath (struct qspinlock *lock,u8 locked) {struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)#include <asm/qspinlock_paravirt.h> #ifndef __pv_queued_spin_unlock -__visible void __pv_queued_spin_unlock(struct qspinlock *lock) +__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock) {u8 locked;