Skip to contentSkip to conversation

KVM: s390: Couple of small cmpxchg() optimizations

PATCH In review Heiko Carstens 3 participants 10 messages Nov 26, 2024

Subsystems (3)
lore
Share & tools
lore

Heiko Carstens

author maintainer
#0 lore raw

v2 cover KVM: s390: Couple of small cmpxchg() optimizations

v2:
- Replace broken WRITE_ONCE(..., 9) with intended WRITE_ONCE(..., 0).

v1:
Use try_cmpxchg() instead of cmpxchg() so compilers with flag output operand support (gcc 14 and newer) can generate slightly better code.

Also get rid of two cmpxchg() usages on one/two byte memory areas which generates inefficient code.

bloat-o-meter statistics of the kvm module:

add/remove: 0/0 grow/shrink: 0/11 up/down: 0/-318 (-318)

Function                                     old     new   delta
kvm_s390_handle_wait                         886     880      -6
kvm_s390_gisa_destroy                        226     220      -6
kvm_s390_gisa_clear                           96      90      -6
ipte_unlock                                  380     372      -8
kvm_s390_gisc_unregister                     270     260     -10
kvm_s390_gisc_register                       290     280     -10
gisa_vcpu_kicker                             200     190     -10
account_mem                                  250     232     -18
ipte_lock                                    416     368     -48
kvm_s390_update_topology_change_report       174     122     -52
kvm_s390_clear_local_irqs                    420     276    -144

Total: Before=316521, After=316203, chg -0.10%

Heiko Carstens (3):
KVM: s390: Use try_cmpxchg() instead of cmpxchg() loops
KVM: s390: Remove one byte cmpxchg() usage
KVM: s390: Increase size of union sca_utility to four bytes

 arch/s390/include/asm/kvm_host.h | 10 +++++-----
 arch/s390/kvm/gaccess.c          | 16 ++++++++--------
 arch/s390/kvm/interrupt.c        | 25 ++++++++-----------------
 arch/s390/kvm/kvm-s390.c         |  4 ++--
 arch/s390/kvm/pci.c              |  5 ++---
 5 files changed, 25 insertions(+), 35 deletions(-)
Signature · 1 line
2.45.2
author maintainer
#1 lore raw

PATCH v2 1/3 KVM: s390: Use try_cmpxchg() instead of cmpxchg() loops

Convert all cmpxchg() loops to try_cmpxchg() loops. With gcc 14 and the usage of flag output operands in try_cmpxchg() this allows the compiler to generate slightly better code.

---
 arch/s390/kvm/gaccess.c   | 16 ++++++++--------
 arch/s390/kvm/interrupt.c | 12 ++++++------
 arch/s390/kvm/kvm-s390.c  |  4 ++--
 arch/s390/kvm/pci.c       |  5 ++---
 4 files changed, 18 insertions(+), 19 deletions(-)
arch/s390/kvm/gaccess.c +8 -8
index a688351f4ab5..9816b0060fbe 100644
@@ -129,8 +129,8 @@ static void ipte_lock_simple(struct kvm *kvm)
 retry:
 	read_lock(&kvm->arch.sca_lock);
 	ic = kvm_s390_get_ipte_control(kvm);
+	old = READ_ONCE(*ic);
 	do {
-		old = READ_ONCE(*ic);
 		if (old.k) {
 			read_unlock(&kvm->arch.sca_lock);
 			cond_resched();
@@ -138,7 +138,7 @@ static void ipte_lock_simple(struct kvm *kvm)
 		}
 		new = old;
 		new.k = 1;
-	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
 	read_unlock(&kvm->arch.sca_lock);
 out:
 	mutex_unlock(&kvm->arch.ipte_mutex);
@@ -154,11 +154,11 @@ static void ipte_unlock_simple(struct kvm *kvm)
 		goto out;
 	read_lock(&kvm->arch.sca_lock);
 	ic = kvm_s390_get_ipte_control(kvm);
+	old = READ_ONCE(*ic);
 	do {
-		old = READ_ONCE(*ic);
 		new = old;
 		new.k = 0;
-	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
 	read_unlock(&kvm->arch.sca_lock);
 	wake_up(&kvm->arch.ipte_wq);
 out:
@@ -172,8 +172,8 @@ static void ipte_lock_siif(struct kvm *kvm)
 retry:
 	read_lock(&kvm->arch.sca_lock);
 	ic = kvm_s390_get_ipte_control(kvm);
+	old = READ_ONCE(*ic);
 	do {
-		old = READ_ONCE(*ic);
 		if (old.kg) {
 			read_unlock(&kvm->arch.sca_lock);
 			cond_resched();
@@ -182,7 +182,7 @@ static void ipte_lock_siif(struct kvm *kvm)
 		new = old;
 		new.k = 1;
 		new.kh++;
-	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
 	read_unlock(&kvm->arch.sca_lock);
 }
 
@@ -192,13 +192,13 @@ static void ipte_unlock_siif(struct kvm *kvm)
 
 	read_lock(&kvm->arch.sca_lock);
 	ic = kvm_s390_get_ipte_control(kvm);
+	old = READ_ONCE(*ic);
 	do {
-		old = READ_ONCE(*ic);
 		new = old;
 		new.kh--;
 		if (!new.kh)
 			new.k = 0;
-	} while (cmpxchg(&ic->val, old.val, new.val) != old.val);
+	} while (!try_cmpxchg(&ic->val, &old.val, new.val));
 	read_unlock(&kvm->arch.sca_lock);
 	if (!new.kh)
 		wake_up(&kvm->arch.ipte_wq);
arch/s390/kvm/interrupt.c +6 -6
index 4f0e7f61edf7..eff69018cbeb 100644
@@ -247,12 +247,12 @@ static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam)
 {
 	u64 word, _word;
 
+	word = READ_ONCE(gisa->u64.word[0]);
 	do {
-		word = READ_ONCE(gisa->u64.word[0]);
 		if ((u64)gisa != word >> 32)
 			return -EBUSY;
 		_word = (word & ~0xffUL) | iam;
-	} while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
+	} while (!try_cmpxchg(&gisa->u64.word[0], &word, _word));
 
 	return 0;
 }
@@ -270,10 +270,10 @@ static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa)
 {
 	u64 word, _word;
 
+	word = READ_ONCE(gisa->u64.word[0]);
 	do {
-		word = READ_ONCE(gisa->u64.word[0]);
 		_word = word & ~(0xffUL << 24);
-	} while (cmpxchg(&gisa->u64.word[0], word, _word) != word);
+	} while (!try_cmpxchg(&gisa->u64.word[0], &word, _word));
 }
 
 /**
@@ -291,14 +291,14 @@ static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi)
 	u8 pending_mask, alert_mask;
 	u64 word, _word;
 
+	word = READ_ONCE(gi->origin->u64.word[0]);
 	do {
-		word = READ_ONCE(gi->origin->u64.word[0]);
 		alert_mask = READ_ONCE(gi->alert.mask);
 		pending_mask = (u8)(word >> 24) & alert_mask;
 		if (pending_mask)
 			return pending_mask;
 		_word = (word & ~0xffUL) | alert_mask;
-	} while (cmpxchg(&gi->origin->u64.word[0], word, _word) != word);
+	} while (!try_cmpxchg(&gi->origin->u64.word[0], &word, _word));
 
 	return 0;
 }
arch/s390/kvm/kvm-s390.c +2 -2
index 442d4a227c0e..d8080c27d45b 100644
@@ -1937,11 +1937,11 @@ static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
 
 	read_lock(&kvm->arch.sca_lock);
 	sca = kvm->arch.sca;
+	old = READ_ONCE(sca->utility);
 	do {
-		old = READ_ONCE(sca->utility);
 		new = old;
 		new.mtcr = val;
-	} while (cmpxchg(&sca->utility.val, old.val, new.val) != old.val);
+	} while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
 	read_unlock(&kvm->arch.sca_lock);
 }
 
arch/s390/kvm/pci.c +2 -3
index a61518b549f0..9b9e7fdd5380 100644
@@ -208,13 +208,12 @@ static inline int account_mem(unsigned long nr_pages)
 
 	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 
+	cur_pages = atomic_long_read(&user->locked_vm);
 	do {
-		cur_pages = atomic_long_read(&user->locked_vm);
 		new_pages = cur_pages + nr_pages;
 		if (new_pages > page_limit)
 			return -ENOMEM;
-	} while (atomic_long_cmpxchg(&user->locked_vm, cur_pages,
-					new_pages) != cur_pages);
+	} while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages));
 
 	atomic64_add(nr_pages, &current->mm->pinned_vm);
 
Signature · 1 line
2.45.2
author maintainer
#2 lore raw

PATCH v2 2/3 KVM: s390: Remove one byte cmpxchg() usage

In reply to Heiko Carstens

Within sca_clear_ext_call() cmpxchg() is used to clear one or two bytes (depending on sca format). The cmpxchg() calls are not supposed to fail; if so that would be a bug. Given that cmpxchg() usage on one and two byte areas generates very inefficient code, replace them with block concurrent WRITE_ONCE() calls, and remove the WARN_ON().

---
 arch/s390/kvm/interrupt.c | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)
arch/s390/kvm/interrupt.c +2 -11
index eff69018cbeb..ea8dce299954 100644
@@ -118,8 +118,6 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
 
 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 {
-	int rc, expect;
-
 	if (!kvm_s390_use_sca_entries())
 		return;
 	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
@@ -128,23 +126,16 @@ static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 		struct esca_block *sca = vcpu->kvm->arch.sca;
 		union esca_sigp_ctrl *sigp_ctrl =
 			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
-		union esca_sigp_ctrl old;
 
-		old = READ_ONCE(*sigp_ctrl);
-		expect = old.value;
-		rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
+		WRITE_ONCE(sigp_ctrl->value, 0);
 	} else {
 		struct bsca_block *sca = vcpu->kvm->arch.sca;
 		union bsca_sigp_ctrl *sigp_ctrl =
 			&(sca->cpu[vcpu->vcpu_id].sigp_ctrl);
-		union bsca_sigp_ctrl old;
 
-		old = READ_ONCE(*sigp_ctrl);
-		expect = old.value;
-		rc = cmpxchg(&sigp_ctrl->value, old.value, 0);
+		WRITE_ONCE(sigp_ctrl->value, 0);
 	}
 	read_unlock(&vcpu->kvm->arch.sca_lock);
-	WARN_ON(rc != expect); /* cannot clear? */
 }
 
 int psw_extint_disabled(struct kvm_vcpu *vcpu)
Signature · 1 line
2.45.2
author maintainer
#3 lore raw

PATCH v2 3/3 KVM: s390: Increase size of union sca_utility to four bytes

In reply to Heiko Carstens

kvm_s390_update_topology_change_report() modifies a single bit within sca_utility using cmpxchg(). Given that the size of the sca_utility union is two bytes this generates very inefficient code. Change the size to four bytes, so better code can be generated.

Even though the size of sca_utility doesn't reflect architecture anymore this seems to be the easiest and most pragmatic approach to avoid inefficient code.

---
 arch/s390/include/asm/kvm_host.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
arch/s390/include/asm/kvm_host.h +5 -5
index 1cd8eaebd3c0..1cb1de232b9e 100644
@@ -95,10 +95,10 @@ union ipte_control {
 };
 
 union sca_utility {
-	__u16 val;
+	__u32 val;
 	struct {
-		__u16 mtcr : 1;
-		__u16 reserved : 15;
+		__u32 mtcr : 1;
+		__u32	   : 31;
 	};
 };
 
@@ -107,7 +107,7 @@ struct bsca_block {
 	__u64	reserved[5];
 	__u64	mcn;
 	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
 	struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS];
 };
 
@@ -115,7 +115,7 @@ struct esca_block {
 	union ipte_control ipte_control;
 	__u64   reserved1[6];
 	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
 	__u64   mcn[4];
 	__u64   reserved3[20];
 	struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS];
Signature · 1 line
2.45.2
maintainer
#4 lore raw

Re: [PATCH v2 3/3] KVM: s390: Increase size of union sca_utility to four bytes

Quoted 20 lines · On Tue, 26 Nov 2024 11:25:15 +0100 Heiko Carstens <[email protected]> wrote:

kvm_s390_update_topology_change_report() modifies a single bit within sca_utility using cmpxchg(). Given that the size of the sca_utility union is two bytes this generates very inefficient code. Change the size to four bytes, so better code can be generated.

Even though the size of sca_utility doesn't reflect architecture anymore this seems to be the easiest and most pragmatic approach to avoid inefficient code.

---
 arch/s390/include/asm/kvm_host.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
arch/s390/include/asm/kvm_host.h +0 -0
index 1cd8eaebd3c0..1cb1de232b9e 100644

arch/s390/include/asm/kvm_host.h union ipte_control { · line 98

Quoted 5 lines
arch/s390/include/asm/kvm_host.h +1 -1
@@ -95,10 +95,10 @@ union ipte_control {
 };
 
 union sca_utility {
-	__u16 val;
+	__u32 val;

I know I said the patch was fine but I realised now that I would like a short comment here explaining that 32 bits allows for more efficient code

you can add it when picking, no need to send a v3

Quoted 26 lines
arch/s390/include/asm/kvm_host.h +4 -4
 	struct {
-		__u16 mtcr : 1;
-		__u16 reserved : 15;
+		__u32 mtcr : 1;
+		__u32	   : 31;
 	};
 };
 
@@ -107,7 +107,7 @@ struct bsca_block {
 	__u64	reserved[5];
 	__u64	mcn;
 	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
 	struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS];
 };
 
@@ -115,7 +115,7 @@ struct esca_block {
 	union ipte_control ipte_control;
 	__u64   reserved1[6];
 	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
 	__u64   mcn[4];
 	__u64   reserved3[20];
 	struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS];
maintainer
#5 lore raw

Re: [PATCH v2 3/3] KVM: s390: Increase size of union sca_utility to four bytes

In reply to Heiko Carstens

On 11/26/24 11:25 AM, Heiko Carstens wrote:
[...]

Quoted 1 line
  union sca_utility {

Would you mind adding a comment?

""Utility is defined as 2 bytes but having it 4 bytes wide generates more efficient code. Since the following bytes are reserved this makes no functional difference.""

Quoted 28 lines
arch/s390/include/asm/kvm_host.h +5 -5
-	__u16 val;
+	__u32 val;
  	struct {
-		__u16 mtcr : 1;
-		__u16 reserved : 15;
+		__u32 mtcr : 1;
+		__u32	   : 31;
  	};
  };
@@ -107,7 +107,7 @@ struct bsca_block {
  	__u64	reserved[5];
  	__u64	mcn;
  	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
  	struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS];
  };
 
@@ -115,7 +115,7 @@ struct esca_block {
  	union ipte_control ipte_control;
  	__u64   reserved1[6];
  	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
  	__u64   mcn[4];
  	__u64   reserved3[20];
  	struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS];
maintainer
#6 lore raw

Re: [PATCH v2 3/3] KVM: s390: Increase size of union sca_utility to four bytes

Quoted 10 lines · On Tue, 26 Nov 2024 13:09:56 +0100 Janosch Frank <[email protected]> wrote:

On 11/26/24 11:25 AM, Heiko Carstens wrote:
[...]

Quoted 1 line
  union sca_utility {  

Would you mind adding a comment?

""Utility is defined as 2 bytes but having it 4 bytes wide generates more efficient code. Since the following bytes are reserved this makes no functional difference.""

looks good, thanks!

Quoted 30 lines
Quoted 29 lines
arch/s390/include/asm/kvm_host.h +5 -5
-	__u16 val;
+	__u32 val;
  	struct {
-		__u16 mtcr : 1;
-		__u16 reserved : 15;
+		__u32 mtcr : 1;
+		__u32	   : 31;
  	};
  };
@@ -107,7 +107,7 @@ struct bsca_block {
  	__u64	reserved[5];
  	__u64	mcn;
  	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
  	struct bsca_entry cpu[KVM_S390_BSCA_CPU_SLOTS];
  };
 
@@ -115,7 +115,7 @@ struct esca_block {
  	union ipte_control ipte_control;
  	__u64   reserved1[6];
  	union sca_utility utility;
-	__u8	reserved2[6];
+	__u8	reserved2[4];
  	__u64   mcn[4];
  	__u64   reserved3[20];
  	struct esca_entry cpu[KVM_S390_ESCA_CPU_SLOTS];  
author maintainer
#7 lore raw

Re: [PATCH v2 3/3] KVM: s390: Increase size of union sca_utility to four bytes

Quoted 7 lines · Claudio Imbrenda
Quoted 4 lines · On Tue, 26 Nov 2024 13:09:56 +0100 Janosch Frank <[email protected]> wrote:

On 11/26/24 11:25 AM, Heiko Carstens wrote:
[...]

Quoted 1 line
  union sca_utility {  
Quoted 8 lines
Quoted 6 lines

Would you mind adding a comment?

""Utility is defined as 2 bytes but having it 4 bytes wide generates more efficient code. Since the following bytes are reserved this makes no functional difference.""

looks good, thanks!

Thanks a lot! I added the comment and applied the series.

maintainer
#8 lore raw
In reply to Heiko Carstens

On Tue, 26 Nov 2024 11:25:12 +0100
Heiko Carstens <[email protected]> wrote:

looks good, and it's a follow-up of your cmpxchg series, so I think it should go through the s390 kernel tree

(but please try to add the comment in patch 3)

Quoted 39 lines

v2:
- Replace broken WRITE_ONCE(..., 9) with intended WRITE_ONCE(..., 0).

v1:
Use try_cmpxchg() instead of cmpxchg() so compilers with flag output operand support (gcc 14 and newer) can generate slightly better code.

Also get rid of two cmpxchg() usages on one/two byte memory areas which generates inefficient code.

bloat-o-meter statistics of the kvm module:

add/remove: 0/0 grow/shrink: 0/11 up/down: 0/-318 (-318)

Function                                     old     new   delta
kvm_s390_handle_wait                         886     880      -6
kvm_s390_gisa_destroy                        226     220      -6
kvm_s390_gisa_clear                           96      90      -6
ipte_unlock                                  380     372      -8
kvm_s390_gisc_unregister                     270     260     -10
kvm_s390_gisc_register                       290     280     -10
gisa_vcpu_kicker                             200     190     -10
account_mem                                  250     232     -18
ipte_lock                                    416     368     -48
kvm_s390_update_topology_change_report       174     122     -52
kvm_s390_clear_local_irqs                    420     276    -144

Total: Before=316521, After=316203, chg -0.10%

Heiko Carstens (3):
KVM: s390: Use try_cmpxchg() instead of cmpxchg() loops
KVM: s390: Remove one byte cmpxchg() usage
KVM: s390: Increase size of union sca_utility to four bytes

 arch/s390/include/asm/kvm_host.h | 10 +++++-----
 arch/s390/kvm/gaccess.c          | 16 ++++++++--------
 arch/s390/kvm/interrupt.c        | 25 ++++++++-----------------
 arch/s390/kvm/kvm-s390.c         |  4 ++--
 arch/s390/kvm/pci.c              |  5 ++---
 5 files changed, 25 insertions(+), 35 deletions(-)
maintainer
#9 lore raw
In reply to Heiko Carstens
Quoted 12 lines · Heiko Carstens

v2:
- Replace broken WRITE_ONCE(..., 9) with intended WRITE_ONCE(..., 0).

v1:
Use try_cmpxchg() instead of cmpxchg() so compilers with flag output operand support (gcc 14 and newer) can generate slightly better code.

Also get rid of two cmpxchg() usages on one/two byte memory areas which generates inefficient code.

bloat-o-meter statistics of the kvm module:

Looks good to me, Ack to the series.