x86/paravirt: Don't make vcpu_is_preempted() a callee-save function
Subsystems (8)
Waiman Long
reviewerPATCH 1/2 x86/paravirt: Don't make vcpu_is_preempted() a callee-save function
It was found when running fio sequential write test with a XFS ramdisk on a 2-socket x86-64 system, the %CPU times as reported by perf were as follows:
71.27% 0.28% fio [k] down_write 70.99% 0.01% fio [k] call_rwsem_down_write_failed 69.43% 1.18% fio [k] rwsem_down_write_failed 65.51% 54.57% fio [k] osq_lock 9.72% 7.99% fio [k] __raw_callee_save___kvm_vcpu_is_preempted 4.16% 4.16% fio [k] __kvm_vcpu_is_preempted
So making vcpu_is_preempted() a callee-save function has a pretty high cost associated with it. As vcpu_is_preempted() is called within the spinlock, mutex and rwsem slowpaths, there isn't much to gain by making it callee-save. So it is now changed to a normal function call instead.
With this patch applied, the aggregrate bandwidth of the fio sequential write test increased slightly from 2563.3MB/s to 2588.1MB/s (about 1%).
--- arch/x86/include/asm/paravirt.h | 2 +- arch/x86/include/asm/paravirt_types.h | 2 +- arch/x86/kernel/kvm.c | 7 ++----- arch/x86/kernel/paravirt-spinlocks.c | 6 ++---- arch/x86/xen/spinlock.c | 4 +--- 5 files changed, 7 insertions(+), 14 deletions(-)
arch/x86/include/asm/paravirt.h +1 -1
@@ -676,7 +676,7 @@ static __always_inline void pv_kick(int cpu) static __always_inline bool pv_vcpu_is_preempted(int cpu) { -return PVOP_CALLEE1 (bool, pv_lock_ops.vcpu_is_preempted, cpu); +return PVOP_CALL1 (bool, pv_lock_ops.vcpu_is_preempted, cpu); }#endif /* SMP && PARAVIRT_SPINLOCKS */
arch/x86/include/asm/paravirt_types.h +1 -1
@@ -309,7 +309,7 @@ struct pv_lock_ops {void (*wait)(u8 *ptr,u8 val);void (*kick)(int cpu); -struct paravirt_callee_save vcpu_is_preempted; +bool (*vcpu_is_preempted)(int cpu); };/* This contains all the paravirt structures: we get a convenient
arch/x86/kernel/kvm.c +2 -5
@@ -595,7 +595,6 @@ __visible bool __kvm_vcpu_is_preempted(int cpu)return !!src->preempted; } -PV_CALLEE_SAVE_REGS_THUNK (__kvm_vcpu_is_preempted);/* * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
@@ -614,10 +613,8 @@ void __init kvm_spinlock_init(void) pv_lock_ops.wait = kvm_wait; pv_lock_ops.kick = kvm_kick_cpu; -if (kvm_para_has_feature (KVM_FEATURE_STEAL_TIME )) { - pv_lock_ops.vcpu_is_preempted = -PV_CALLEE_SAVE (__kvm_vcpu_is_preempted); - } +if (kvm_para_has_feature (KVM_FEATURE_STEAL_TIME )) + pv_lock_ops.vcpu_is_preempted = __kvm_vcpu_is_preempted; }#endif /* CONFIG_PARAVIRT_SPINLOCKS */
arch/x86/kernel/paravirt-spinlocks.c +2 -4
@@ -24,12 +24,10 @@ __visible bool __native_vcpu_is_preempted(int cpu) {return false; } -PV_CALLEE_SAVE_REGS_THUNK (__native_vcpu_is_preempted);bool pv_is_native_vcpu_is_preempted (void ) { -return pv_lock_ops.vcpu_is_preempted.func == - __raw_callee_save___native_vcpu_is_preempted; +return pv_lock_ops.vcpu_is_preempted == __native_vcpu_is_preempted; }struct pv_lock_ops pv_lock_ops = {
@@ -38,7 +36,7 @@ struct pv_lock_ops pv_lock_ops = { .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), .wait = paravirt_nop, .kick = paravirt_nop, - .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), + .vcpu_is_preempted = __native_vcpu_is_preempted,#endif /* SMP */ };EXPORT_SYMBOL (pv_lock_ops);
arch/x86/xen/spinlock.c +1 -3
@@ -114,8 +114,6 @@ void xen_uninit_lock_cpu(int cpu)per_cpu (irq_name, cpu) =NULL ; } -PV_CALLEE_SAVE_REGS_THUNK (xen_vcpu_stolen); -/* * Our init of PV spinlocks is split in two init functions due to us * using paravirt patching and jump labels patching and having to do
@@ -138,7 +136,7 @@ void __init xen_init_spinlocks(void) pv_lock_ops.queued_spin_unlock =PV_CALLEE_SAVE (__pv_queued_spin_unlock); pv_lock_ops.wait = xen_qlock_wait; pv_lock_ops.kick = xen_qlock_kick; - pv_lock_ops.vcpu_is_preempted =PV_CALLEE_SAVE (xen_vcpu_stolen); + pv_lock_ops.vcpu_is_preempted = xen_vcpu_stolen; }static __init int xen_parse_nopvspin (char *arg)
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 2/2 locking/mutex,rwsem: Reduce vcpu_is_preempted() calling frequency
As the vcpu_is_preempted() call is pretty costly compared with other checks within mutex_spin_on_owner() and rwsem_spin_on_owner(), they are done at a reduce frequency of once every 256 iterations.
--- kernel/locking/mutex.c | 5 ++++- kernel/locking/rwsem-xadd.c | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-)
kernel/locking/mutex.c +4 -1
@@ -423,6 +423,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter) { bool ret = true; +int loop =0 ;rcu_read_lock ();while (__mutex_owner (lock) == owner) {
@@ -436,9 +437,11 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,/* * Use vcpu_is_preempted to detect lock holder preemption issue. +* As vcpu_is_preempted is more costly to use, it is called at +* a reduced frequencey (once every 256 iterations). */ if (!owner->on_cpu ||need_resched () || -vcpu_is_preempted (task_cpu (owner))) { + (!(++loop &0xff ) &&vcpu_is_preempted (task_cpu (owner)))) { ret = false;break ; }
kernel/locking/rwsem-xadd.c +4 -2
@@ -351,6 +351,7 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem) {struct task_struct *owner =READ_ONCE (sem->owner); +int loop =0 ;if (!rwsem_owner_is_writer (owner)) goto out;
@@ -367,10 +368,11 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)/* * abort spinning when need_resched or owner is not running or -* owner's cpu is preempted. +* owner's cpu is preempted. The preemption check is done at +* a lower frequencey because of its high cost. */ if (!owner->on_cpu ||need_resched () || -vcpu_is_preempted (task_cpu (owner))) { + (!(++loop &0xff ) &&vcpu_is_preempted (task_cpu (owner)))) {rcu_read_unlock ();return false; }
Signature · 1 line
1.8.3.1
Peter Zijlstra
maintainerRe: [PATCH 2/2] locking/mutex,rwsem: Reduce vcpu_is_preempted() calling frequency
Quoted 3 lines · Waiman Long
As the
vcpu_is_preempted()call is pretty costly compared with other checks withinmutex_spin_on_owner()andrwsem_spin_on_owner(), they are done at a reduce frequency of once every 256 iterations.
That's just disgusting.
Waiman Long
reviewerRe: [PATCH 2/2] locking/mutex,rwsem: Reduce vcpu_is_preempted() calling frequency
Quoted 5 lines · Peter Zijlstra
Quoted 3 lines · Waiman Long
As the
vcpu_is_preempted()call is pretty costly compared with other checks withinmutex_spin_on_owner()andrwsem_spin_on_owner(), they are done at a reduce frequency of once every 256 iterations.That's just disgusting.
I do have some doubt myself on the effectiveness of this patch. Anyway, it is the first patch that I think is more beneficial.
Cheers,
Longman
Waiman Long
reviewerRe: [PATCH 2/2] locking/mutex, rwsem: Reduce vcpu_is_preempted() calling frequency
In reply to Peter ZijlstraQuoted 5 lines · Peter Zijlstra
Quoted 3 lines · Waiman Long
As the
vcpu_is_preempted()call is pretty costly compared with other checks withinmutex_spin_on_owner()andrwsem_spin_on_owner(), they are done at a reduce frequency of once every 256 iterations.That's just disgusting.
I do have some doubt myself on the effectiveness of this patch. Anyway, it is the first patch that I think is more beneficial.
Cheers,
Longman
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xen.org/xen-devel
Peter Zijlstra
maintainerRe: [PATCH 2/2] locking/mutex, rwsem: Reduce vcpu_is_preempted() calling frequency
In reply to Waiman LongQuoted 3 lines · Waiman Long
As the
vcpu_is_preempted()call is pretty costly compared with other checks withinmutex_spin_on_owner()andrwsem_spin_on_owner(), they are done at a reduce frequency of once every 256 iterations.
That's just disgusting.
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xen.org/xen-devel
Waiman Long
reviewerPATCH 2/2 locking/mutex, rwsem: Reduce vcpu_is_preempted() calling frequency
In reply to Waiman LongAs the vcpu_is_preempted() call is pretty costly compared with other checks within mutex_spin_on_owner() and rwsem_spin_on_owner(), they are done at a reduce frequency of once every 256 iterations.
--- kernel/locking/mutex.c | 5 ++++- kernel/locking/rwsem-xadd.c | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-)
kernel/locking/mutex.c +4 -1
@@ -423,6 +423,7 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner, struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter) { bool ret = true; +int loop =0 ;rcu_read_lock ();while (__mutex_owner (lock) == owner) {
@@ -436,9 +437,11 @@ bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,/* * Use vcpu_is_preempted to detect lock holder preemption issue. +* As vcpu_is_preempted is more costly to use, it is called at +* a reduced frequencey (once every 256 iterations). */ if (!owner->on_cpu ||need_resched () || -vcpu_is_preempted (task_cpu (owner))) { + (!(++loop &0xff ) &&vcpu_is_preempted (task_cpu (owner)))) { ret = false;break ; }
kernel/locking/rwsem-xadd.c +4 -2
@@ -351,6 +351,7 @@ static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem) {struct task_struct *owner =READ_ONCE (sem->owner); +int loop =0 ;if (!rwsem_owner_is_writer (owner)) goto out;
@@ -367,10 +368,11 @@ static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)/* * abort spinning when need_resched or owner is not running or -* owner's cpu is preempted. +* owner's cpu is preempted. The preemption check is done at +* a lower frequencey because of its high cost. */ if (!owner->on_cpu ||need_resched () || -vcpu_is_preempted (task_cpu (owner))) { + (!(++loop &0xff ) &&vcpu_is_preempted (task_cpu (owner)))) {rcu_read_unlock ();return false; }
Signature · 7 lines
1.8.3.1 _______________________________________________ Xen-devel mailing list [email protected] https://lists.xen.org/xen-devel
Peter Zijlstra
maintainerQuoted 11 lines · Waiman Long
It was found when running fio sequential write test with a XFS ramdisk on a 2-socket x86-64 system, the %CPU times as reported by perf were as follows:
71.27% 0.28% fio [k] down_write 70.99% 0.01% fio [k] call_rwsem_down_write_failed 69.43% 1.18% fio [k] rwsem_down_write_failed 65.51% 54.57% fio [k] osq_lock 9.72% 7.99% fio [k] __raw_callee_save___kvm_vcpu_is_preempted 4.16% 4.16% fio [k] __kvm_vcpu_is_preempted
Quoted 5 lines
So making
vcpu_is_preempted()a callee-save function has a pretty high cost associated with it. Asvcpu_is_preempted()is called within the spinlock, mutex and rwsem slowpaths, there isn't much to gain by making it callee-save. So it is now changed to a normal function call instead.
Numbers for bare metal too please.
Waiman Long
reviewerQuoted 12 lines · Peter Zijlstra
Quoted 11 lines · Waiman Long
It was found when running fio sequential write test with a XFS ramdisk on a 2-socket x86-64 system, the %CPU times as reported by perf were as follows:
71.27% 0.28% fio [k] down_write 70.99% 0.01% fio [k] call_rwsem_down_write_failed 69.43% 1.18% fio [k] rwsem_down_write_failed 65.51% 54.57% fio [k] osq_lock 9.72% 7.99% fio [k] __raw_callee_save___kvm_vcpu_is_preempted 4.16% 4.16% fio [k] __kvm_vcpu_is_preempted
Quoted 6 lines
Quoted 5 lines
So making
vcpu_is_preempted()a callee-save function has a pretty high cost associated with it. Asvcpu_is_preempted()is called within the spinlock, mutex and rwsem slowpaths, there isn't much to gain by making it callee-save. So it is now changed to a normal function call instead.Numbers for bare metal too please.
I will run the test on bare metal, but I doubt there will be noticeable difference.
Cheers,
Longman
Waiman Long
reviewerQuoted 12 lines · Peter Zijlstra
Quoted 11 lines · Waiman Long
It was found when running fio sequential write test with a XFS ramdisk on a 2-socket x86-64 system, the %CPU times as reported by perf were as follows:
71.27% 0.28% fio [k] down_write 70.99% 0.01% fio [k] call_rwsem_down_write_failed 69.43% 1.18% fio [k] rwsem_down_write_failed 65.51% 54.57% fio [k] osq_lock 9.72% 7.99% fio [k] __raw_callee_save___kvm_vcpu_is_preempted 4.16% 4.16% fio [k] __kvm_vcpu_is_preempted
Quoted 6 lines
Quoted 5 lines
So making
vcpu_is_preempted()a callee-save function has a pretty high cost associated with it. Asvcpu_is_preempted()is called within the spinlock, mutex and rwsem slowpaths, there isn't much to gain by making it callee-save. So it is now changed to a normal function call instead.Numbers for bare metal too please.
I will run the test on bare metal, but I doubt there will be noticeable difference.
Cheers,
Longman
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xen.org/xen-devel
Peter Zijlstra
maintainerQuoted 11 lines · Waiman Long
It was found when running fio sequential write test with a XFS ramdisk on a 2-socket x86-64 system, the %CPU times as reported by perf were as follows:
71.27% 0.28% fio [k] down_write 70.99% 0.01% fio [k] call_rwsem_down_write_failed 69.43% 1.18% fio [k] rwsem_down_write_failed 65.51% 54.57% fio [k] osq_lock 9.72% 7.99% fio [k] __raw_callee_save___kvm_vcpu_is_preempted 4.16% 4.16% fio [k] __kvm_vcpu_is_preempted
Quoted 5 lines
So making
vcpu_is_preempted()a callee-save function has a pretty high cost associated with it. Asvcpu_is_preempted()is called within the spinlock, mutex and rwsem slowpaths, there isn't much to gain by making it callee-save. So it is now changed to a normal function call instead.
Numbers for bare metal too please.
_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xen.org/xen-devel