Skip to contentSkip to conversation

locking: Add __lockfunc to slow path functions

PATCH Applied Namhyung Kim 3 participants 3 messages Aug 10, 2022 – Aug 22, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 locking: Add __lockfunc to slow path functions

So that we can skip the functions in the perf lock contention and other places like /proc/PID/wchan.

---
* annotate pv qspinlock functions too

 arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------
 kernel/locking/qrwlock.c                  |  4 ++--
 kernel/locking/qspinlock.c                |  2 +-
 kernel/locking/qspinlock_paravirt.h       |  4 ++--
 4 files changed, 12 insertions(+), 11 deletions(-)
arch/x86/include/asm/qspinlock_paravirt.h +7 -6
index 892fd8c3a6f7..60ece592b220 100644
@@ -12,7 +12,7 @@
  */
 #ifdef CONFIG_64BIT
 
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
 #define __pv_queued_spin_unlock	__pv_queued_spin_unlock
 #define PV_UNLOCK		"__raw_callee_save___pv_queued_spin_unlock"
 #define PV_UNLOCK_SLOWPATH	"__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
 /*
  * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
  * which combines the registers saving trunk and the body of the following
- * C code:
+ * C code.  Note that it puts the code in the .spinlock.text section which
+ * is equivalent to adding __lockfunc in the C code:
  *
- * void __pv_queued_spin_unlock(struct qspinlock *lock)
+ * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
  * {
  *	u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
  *
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
  *   rsi = lockval           (second argument)
  *   rdx = internal variable (set to 0)
  */
-asm    (".pushsection .text;"
+asm    (".pushsection .spinlock.text;"
 	".globl " PV_UNLOCK ";"
 	".type " PV_UNLOCK ", @function;"
 	".align 4,0x90;"
@@ -65,8 +66,8 @@ asm    (".pushsection .text;"
 
 #else /* CONFIG_64BIT */
 
-extern void __pv_queued_spin_unlock(struct qspinlock *lock);
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
+extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
 
 #endif /* CONFIG_64BIT */
 #endif
kernel/locking/qrwlock.c +2 -2
index 2e1600906c9f..d2ef312a8611 100644
@@ -18,7 +18,7 @@
  * queued_read_lock_slowpath - acquire read lock of a queued rwlock
  * @lock: Pointer to queued rwlock structure
  */
-void queued_read_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
 {
 	/*
 	 * Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
  * queued_write_lock_slowpath - acquire write lock of a queued rwlock
  * @lock : Pointer to queued rwlock structure
  */
-void queued_write_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
 {
 	int cnts;
 
kernel/locking/qspinlock.c +1 -1
index 65a9a10caa6f..2b23378775fe 100644
@@ -313,7 +313,7 @@ static __always_inline u32  __pv_wait_head_or_lock(struct qspinlock *lock,
  * contended             :    (*,x,y) +--> (*,0,0) ---> (*,0,1) -'  :
  *   queue               :         ^--'                             :
  */
-void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
 {
 	struct mcs_spinlock *prev, *next, *node;
 	u32 old, tail;
kernel/locking/qspinlock_paravirt.h +2 -2
index e84d21aa0722..6afc249ce697 100644
@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)
  * PV versions of the unlock fastpath and slowpath functions to be used
  * instead of queued_spin_unlock().
  */
-__visible void
+__visible __lockfunc void
 __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
 {
 	struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
 #include <asm/qspinlock_paravirt.h>
 
 #ifndef __pv_queued_spin_unlock
-__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
+__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
 {
 	u8 locked;
 
Signature · 1 line
2.37.1.559.g78731f0fdb-goog
reviewer Acked-by on v2 1/1
#1 lore raw
Quoted 108 lines · Namhyung Kim

So that we can skip the functions in the perf lock contention and other places like /proc/PID/wchan.

---
* annotate pv qspinlock functions too

  arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------
  kernel/locking/qrwlock.c                  |  4 ++--
  kernel/locking/qspinlock.c                |  2 +-
  kernel/locking/qspinlock_paravirt.h       |  4 ++--
  4 files changed, 12 insertions(+), 11 deletions(-)
arch/x86/include/asm/qspinlock_paravirt.h +7 -6
index 892fd8c3a6f7..60ece592b220 100644
@@ -12,7 +12,7 @@
   */
  #ifdef CONFIG_64BIT
 
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
  #define __pv_queued_spin_unlock	__pv_queued_spin_unlock
  #define PV_UNLOCK		"__raw_callee_save___pv_queued_spin_unlock"
  #define PV_UNLOCK_SLOWPATH	"__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
  /*
   * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
   * which combines the registers saving trunk and the body of the following
- * C code:
+ * C code.  Note that it puts the code in the .spinlock.text section which
+ * is equivalent to adding __lockfunc in the C code:
   *
- * void __pv_queued_spin_unlock(struct qspinlock *lock)
+ * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
   * {
   *	u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
   *
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
   *   rsi = lockval           (second argument)
   *   rdx = internal variable (set to 0)
   */
-asm    (".pushsection .text;"
+asm    (".pushsection .spinlock.text;"
  	".globl " PV_UNLOCK ";"
  	".type " PV_UNLOCK ", @function;"
  	".align 4,0x90;"
@@ -65,8 +66,8 @@ asm    (".pushsection .text;"
 
  #else /* CONFIG_64BIT */
 
-extern void __pv_queued_spin_unlock(struct qspinlock *lock);
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
+extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
 
  #endif /* CONFIG_64BIT */
  #endif
kernel/locking/qrwlock.c +2 -2
index 2e1600906c9f..d2ef312a8611 100644
@@ -18,7 +18,7 @@
   * queued_read_lock_slowpath - acquire read lock of a queued rwlock
   * @lock: Pointer to queued rwlock structure
   */
-void queued_read_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
  {
  	/*
  	 * Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
   * queued_write_lock_slowpath - acquire write lock of a queued rwlock
   * @lock : Pointer to queued rwlock structure
   */
-void queued_write_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
  {
  	int cnts;
 
kernel/locking/qspinlock.c +1 -1
index 65a9a10caa6f..2b23378775fe 100644
@@ -313,7 +313,7 @@ static __always_inline u32  __pv_wait_head_or_lock(struct qspinlock *lock,
   * contended             :    (*,x,y) +--> (*,0,0) ---> (*,0,1) -'  :
   *   queue               :         ^--'                             :
   */
-void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
  {
  	struct mcs_spinlock *prev, *next, *node;
  	u32 old, tail;
kernel/locking/qspinlock_paravirt.h +1 -1
index e84d21aa0722..6afc249ce697 100644
@@ -489,7 +489,7 @@ pv_wait_head_or_lock(struct qspinlock *lock, struct mcs_spinlock *node)
   * PV versions of the unlock fastpath and slowpath functions to be used
   * instead of queued_spin_unlock().
   */
-__visible void
+__visible __lockfunc void
  __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
  {
  	struct pv_node *node;

kernel/locking/qspinlock_paravirt.h __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked) · line 550

Quoted 8 lines
kernel/locking/qspinlock_paravirt.h +1 -1
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
  #include <asm/qspinlock_paravirt.h>
 
  #ifndef __pv_queued_spin_unlock
-__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
+__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
  {
  	u8 locked;
 
merge bot Applied
#2 lore raw
In reply to Namhyung Kim
The following commit has been merged into the locking/core branch of tip:

The following commit has been merged into the locking/core branch of tip:

Commit-ID:     501f7f69bca195da266de83eb2c26c30813fba97
Gitweb:        https://git.kernel.org/tip/501f7f69bca195da266de83eb2c26c30813fba97
Author:        Namhyung Kim <[email protected]>
AuthorDate:    Wed, 10 Aug 2022 15:03:46 -07:00
Committer:     Peter Zijlstra <[email protected]>
CommitterDate: Fri, 19 Aug 2022 19:47:51 +02:00

locking: Add __lockfunc to slow path functions

So that we can skip the functions in the perf lock contention and other places like /proc/PID/wchan.

---
 arch/x86/include/asm/qspinlock_paravirt.h | 13 +++++++------
 kernel/locking/qrwlock.c                  |  4 ++--
 kernel/locking/qspinlock.c                |  2 +-
 kernel/locking/qspinlock_paravirt.h       |  4 ++--
 4 files changed, 12 insertions(+), 11 deletions(-)
arch/x86/include/asm/qspinlock_paravirt.h +7 -6
index 892fd8c..60ece59 100644
@@ -12,7 +12,7 @@
  */
 #ifdef CONFIG_64BIT
 
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
 #define __pv_queued_spin_unlock	__pv_queued_spin_unlock
 #define PV_UNLOCK		"__raw_callee_save___pv_queued_spin_unlock"
 #define PV_UNLOCK_SLOWPATH	"__raw_callee_save___pv_queued_spin_unlock_slowpath"
@@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
 /*
  * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
  * which combines the registers saving trunk and the body of the following
- * C code:
+ * C code.  Note that it puts the code in the .spinlock.text section which
+ * is equivalent to adding __lockfunc in the C code:
  *
- * void __pv_queued_spin_unlock(struct qspinlock *lock)
+ * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
  * {
  *	u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
  *
@@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
  *   rsi = lockval           (second argument)
  *   rdx = internal variable (set to 0)
  */
-asm    (".pushsection .text;"
+asm    (".pushsection .spinlock.text;"
 	".globl " PV_UNLOCK ";"
 	".type " PV_UNLOCK ", @function;"
 	".align 4,0x90;"
@@ -65,8 +66,8 @@ asm    (".pushsection .text;"
 
 #else /* CONFIG_64BIT */
 
-extern void __pv_queued_spin_unlock(struct qspinlock *lock);
-PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock);
+extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
+__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
 
 #endif /* CONFIG_64BIT */
 #endif
kernel/locking/qrwlock.c +2 -2
index 2e16009..d2ef312 100644
@@ -18,7 +18,7 @@
  * queued_read_lock_slowpath - acquire read lock of a queued rwlock
  * @lock: Pointer to queued rwlock structure
  */
-void queued_read_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
 {
 	/*
 	 * Readers come here when they cannot get the lock without waiting
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
  * queued_write_lock_slowpath - acquire write lock of a queued rwlock
  * @lock : Pointer to queued rwlock structure
  */
-void queued_write_lock_slowpath(struct qrwlock *lock)
+void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
 {
 	int cnts;
 
kernel/locking/qspinlock.c +1 -1
index 65a9a10..2b23378 100644
@@ -313,7 +313,7 @@ static __always_inline u32  __pv_wait_head_or_lock(struct qspinlock *lock,
  * contended             :    (*,x,y) +--> (*,0,0) ---> (*,0,1) -'  :
  *   queue               :         ^--'                             :
  */
-void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
+void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
 {
 	struct mcs_spinlock *prev, *next, *node;
 	u32 old, tail;
kernel/locking/qspinlock_paravirt.h +2 -2
index e84d21a..6afc249 100644
@@ -489,7 +489,7 @@ gotlock:
  * PV versions of the unlock fastpath and slowpath functions to be used
  * instead of queued_spin_unlock().
  */
-__visible void
+__visible __lockfunc void
 __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
 {
 	struct pv_node *node;
@@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
 #include <asm/qspinlock_paravirt.h>
 
 #ifndef __pv_queued_spin_unlock
-__visible void __pv_queued_spin_unlock(struct qspinlock *lock)
+__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
 {
 	u8 locked;