Skip to contentSkip to conversation

KVM: Speed up MMIO registrations

PATCH Superseded Keir Fraser 4 participants 22 messages Jul 16, 2025 – Jul 22, 2025

Subsystems (5)
x86 KVM arm64 KVM x86 +1 more subsystems

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover KVM: Speed up MMIO registrations

This is version 2 of the patches I previously posted here:

https://lore.kernel.org/all/[email protected]/

Changes since v1:

* Memory barriers introduced (or implied and documented) on the kvm->buses[] SRCU read paths

* Disallow kvm_get_bus() from SRCU readers

* Rebased to v6.16-rc6

Thanks to Sean for the review feedback and patch suggestions!

Keir Fraser (4):
KVM: arm64: vgic-init: Remove vgic_ready() macro
KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering
KVM: Implement barriers before accessing kvm->buses[] on SRCU read
paths
KVM: Avoid synchronize_srcu() in kvm_io_bus_register_dev()

 arch/arm64/kvm/vgic/vgic-init.c | 14 +++--------
 arch/x86/kvm/vmx/vmx.c          |  7 ++++++
 include/kvm/arm_vgic.h          |  1 -
 include/linux/kvm_host.h        | 11 ++++++---
 virt/kvm/kvm_main.c             | 43 +++++++++++++++++++++++++++------
 5 files changed, 53 insertions(+), 23 deletions(-)
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#1 lore raw

PATCH v2 1/4 KVM: arm64: vgic-init: Remove vgic_ready() macro

It is now used only within kvm_vgic_map_resources(). vgic_dist::ready is already written directly by this function, so it is clearer to bypass the macro for reads as well.

---
 arch/arm64/kvm/vgic/vgic-init.c | 5 ++---
 include/kvm/arm_vgic.h          | 1 -
 2 files changed, 2 insertions(+), 4 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +2 -3
index eb1205654ac8..502b65049703 100644
@@ -559,7 +559,6 @@ int vgic_lazy_init(struct kvm *kvm)
  * Also map the virtual CPU interface into the VM.
  * v2 calls vgic_init() if not already done.
  * v3 and derivatives return an error if the VGIC is not initialized.
- * vgic_ready() returns true if this function has succeeded.
  */
 int kvm_vgic_map_resources(struct kvm *kvm)
 {
@@ -568,12 +567,12 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(vgic_ready(kvm)))
+	if (likely(dist->ready))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
 	mutex_lock(&kvm->arch.config_lock);
-	if (vgic_ready(kvm))
+	if (dist->ready)
 		goto out;
 
 	if (!irqchip_in_kernel(kvm))
include/kvm/arm_vgic.h +0 -1
index 4a34f7f0a864..233eaa6d1267 100644
@@ -399,7 +399,6 @@ u64 vgic_v3_get_misr(struct kvm_vcpu *vcpu);
 
 #define irqchip_in_kernel(k)	(!!((k)->arch.vgic.in_kernel))
 #define vgic_initialized(k)	((k)->arch.vgic.initialized)
-#define vgic_ready(k)		((k)->arch.vgic.ready)
 #define vgic_valid_spi(k, i)	(((i) >= VGIC_NR_PRIVATE_IRQS) && \
 			((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
 
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#2 lore raw

PATCH v2 2/4 KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

In reply to Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +2 -9
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);
 	goto out_slots;
 out:
 	mutex_unlock(&kvm->arch.config_lock);
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#3 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 26 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 10 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
 
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

Tree SRCU on SMP:
call_srcu()
__call_srcu()
srcu_gp_start_if_needed()

    __srcu_read_unlock_nmisafe()
	 #ifdef	CONFIG_NEED_SRCU_NMI_SAFE
	   	  smp_mb__before_atomic() // __smp_mb() on ARM64, do nothing on x86.
	 #else
          __srcu_read_unlock()
		   smp_mb()
	 #endif

TINY SRCY on UP:
Should have no memory ordering issue on UP.

Quoted 6 lines
arch/arm64/kvm/vgic/vgic-init.c +0 -1
 	goto out_slots;
 out:
 	mutex_unlock(&kvm->arch.config_lock);
--

2.50.0.727.gbf7dc18ff4-goog

#4 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 45 lines · Yao Yuan
Quoted 26 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 10 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
 
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

Tree SRCU on SMP:
call_srcu()
__call_srcu()
srcu_gp_start_if_needed()

Quoted 7 lines
    __srcu_read_unlock_nmisafe()
	 #ifdef	CONFIG_NEED_SRCU_NMI_SAFE
	   	  smp_mb__before_atomic() // __smp_mb() on ARM64, do nothing on x86.
	 #else
          __srcu_read_unlock()
		   smp_mb()
	 #endif

I don't think it's nice to depend on an implementation detail of kvm_io_bus_register_dev() and, transitively, on implementation details of call_srcu().

kvm_vgic_map_resources() isn't called that often and can afford its own synchronization.

-- Keir

Quoted 9 lines

TINY SRCY on UP:
Should have no memory ordering issue on UP.

Quoted 6 lines
arch/arm64/kvm/vgic/vgic-init.c +0 -1
 	goto out_slots;
 out:
 	mutex_unlock(&kvm->arch.config_lock);
--

2.50.0.727.gbf7dc18ff4-goog

#5 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 54 lines · Keir Fraser
Quoted 45 lines · Yao Yuan
Quoted 26 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 10 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
 
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

Tree SRCU on SMP:
call_srcu()
__call_srcu()
srcu_gp_start_if_needed()

Quoted 7 lines
    __srcu_read_unlock_nmisafe()
	 #ifdef	CONFIG_NEED_SRCU_NMI_SAFE
	   	  smp_mb__before_atomic() // __smp_mb() on ARM64, do nothing on x86.
	 #else
          __srcu_read_unlock()
		   smp_mb()
	 #endif
Quoted 6 lines

I don't think it's nice to depend on an implementation detail of kvm_io_bus_register_dev() and, transitively, on implementation details of call_srcu().

kvm_vgic_map_resources() isn't called that often and can afford its own synchronization.

+1

#6 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

In reply to Keir Fraser
Quoted 54 lines · Keir Fraser
Quoted 45 lines · Yao Yuan
Quoted 26 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 10 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
 
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

Tree SRCU on SMP:
call_srcu()
__call_srcu()
srcu_gp_start_if_needed()

Quoted 7 lines
    __srcu_read_unlock_nmisafe()
	 #ifdef	CONFIG_NEED_SRCU_NMI_SAFE
	   	  smp_mb__before_atomic() // __smp_mb() on ARM64, do nothing on x86.
	 #else
          __srcu_read_unlock()
		   smp_mb()
	 #endif
Quoted 3 lines

I don't think it's nice to depend on an implementation detail of kvm_io_bus_register_dev() and, transitively, on implementation details of call_srcu().

Also I should note that this is moot because the smp_mb() would *not* safely replace the load-acquire.

Quoted 14 lines

kvm_vgic_map_resources() isn't called that often and can afford its own synchronization.

-- Keir

Quoted 9 lines

TINY SRCY on UP:
Should have no memory ordering issue on UP.

Quoted 6 lines
arch/arm64/kvm/vgic/vgic-init.c +0 -1
 	goto out_slots;
 out:
 	mutex_unlock(&kvm->arch.config_lock);
--

2.50.0.727.gbf7dc18ff4-goog

#7 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 55 lines · Keir Fraser
Quoted 54 lines · Keir Fraser
Quoted 52 lines · Yao Yuan
Quoted 26 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 10 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
 
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

Tree SRCU on SMP:
call_srcu()
__call_srcu()
srcu_gp_start_if_needed()

    __srcu_read_unlock_nmisafe()
	 #ifdef	CONFIG_NEED_SRCU_NMI_SAFE
	   	  smp_mb__before_atomic() // __smp_mb() on ARM64, do nothing on x86.
	 #else
          __srcu_read_unlock()
		   smp_mb()
	 #endif
Quoted 3 lines
Quoted 3 lines

I don't think it's nice to depend on an implementation detail of kvm_io_bus_register_dev() and, transitively, on implementation details of call_srcu().

This is good point, I agree with you.

Quoted 3 lines

Also I should note that this is moot because the smp_mb() would *not* safely replace the load-acquire.

Hmm.. do you mean it can't order the write to dist->ready here while read it in aonther thread at the same time ?

Quoted 16 lines
Quoted 15 lines

kvm_vgic_map_resources() isn't called that often and can afford its own synchronization.

-- Keir

Quoted 9 lines

TINY SRCY on UP:
Should have no memory ordering issue on UP.

Quoted 6 lines
arch/arm64/kvm/vgic/vgic-init.c +0 -1
 	goto out_slots;
 out:
 	mutex_unlock(&kvm->arch.config_lock);
--

2.50.0.727.gbf7dc18ff4-goog

#8 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

In reply to Yao Yuan
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

#9 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 47 lines · Sean Christopherson
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

Quoted 4 lines

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

Yes, I understand this and agree with your point.

Just for discusstion: I thought it should also work even w/o introduce the load acqure + store release after switch to call_srcu(): The smp_mb() in call_srcu() order the all store to kvm->arch.vgic before store kvm->arch.vgic.ready in current implementation.

Quoted 1 line
#10 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 53 lines · Yao Yuan
Quoted 47 lines · Sean Christopherson
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

Quoted 4 lines

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

Quoted 7 lines

Yes, I understand this and agree with your point.

Just for discusstion: I thought it should also work even w/o introduce the load acqure + store release after switch to call_srcu(): The smp_mb() in call_srcu() order the all store to kvm->arch.vgic before store kvm->arch.vgic.ready in current implementation.

The load-acquire would still be required, to ensure that accesses to kvm->arch.vgic do not get reordered earlier than the lock-free check of kvm->arch.vgic.ready. Otherwise that CPU could see that the vgic is initialised, but then use speculated reads of uninitialised vgic state.

Quoted 1 line
Quoted 1 line
#11 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 62 lines · Keir Fraser
Quoted 53 lines · Yao Yuan
Quoted 47 lines · Sean Christopherson
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

Quoted 4 lines

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

Quoted 7 lines

Yes, I understand this and agree with your point.

Just for discusstion: I thought it should also work even w/o introduce the load acqure + store release after switch to call_srcu(): The smp_mb() in call_srcu() order the all store to kvm->arch.vgic before store kvm->arch.vgic.ready in current implementation.

Quoted 5 lines

The load-acquire would still be required, to ensure that accesses to kvm->arch.vgic do not get reordered earlier than the lock-free check of kvm->arch.vgic.ready. Otherwise that CPU could see that the vgic is initialised, but then use speculated reads of uninitialised vgic state.

Thanks for your explanation.

I see. But there's "mutex_lock(&kvm->slot_lock);" before later acccessing to the kvm->arch.vgic, so I think the order can be guaranteed. Of cause as you said a explicitly acquire-load + store-release is better than before implicitly implementation.

Quoted 1 line
Quoted 1 line
Quoted 1 line
#12 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 69 lines · Yao Yuan
Quoted 62 lines · Keir Fraser
Quoted 53 lines · Yao Yuan
Quoted 47 lines · Sean Christopherson
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

Quoted 4 lines

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

Quoted 7 lines

Yes, I understand this and agree with your point.

Just for discusstion: I thought it should also work even w/o introduce the load acqure + store release after switch to call_srcu(): The smp_mb() in call_srcu() order the all store to kvm->arch.vgic before store kvm->arch.vgic.ready in current implementation.

Quoted 5 lines

The load-acquire would still be required, to ensure that accesses to kvm->arch.vgic do not get reordered earlier than the lock-free check of kvm->arch.vgic.ready. Otherwise that CPU could see that the vgic is initialised, but then use speculated reads of uninitialised vgic state.

Quoted 6 lines

Thanks for your explanation.

I see. But there's "mutex_lock(&kvm->slot_lock);" before later acccessing to the kvm->arch.vgic, so I think the order can be guaranteed. Of cause as you said a explicitly acquire-load + store-release is better than before implicitly implementation.

If vgic_dist::ready is observed true by the lock-free read (the one which is turned into load-acquire by this patch) then the function immediately returns with no mutex_lock() executed. It is reads of vgic_dist *after* return from kvm_vgic_map_resources() that you have to worry about, and which require load-acquire semantics.

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
#13 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 77 lines · Keir Fraser
Quoted 69 lines · Yao Yuan
Quoted 62 lines · Keir Fraser
Quoted 53 lines · Yao Yuan
Quoted 47 lines · Sean Christopherson
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

Quoted 4 lines

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

Quoted 7 lines

Yes, I understand this and agree with your point.

Just for discusstion: I thought it should also work even w/o introduce the load acqure + store release after switch to call_srcu(): The smp_mb() in call_srcu() order the all store to kvm->arch.vgic before store kvm->arch.vgic.ready in current implementation.

Quoted 5 lines

The load-acquire would still be required, to ensure that accesses to kvm->arch.vgic do not get reordered earlier than the lock-free check of kvm->arch.vgic.ready. Otherwise that CPU could see that the vgic is initialised, but then use speculated reads of uninitialised vgic state.

Quoted 6 lines

Thanks for your explanation.

I see. But there's "mutex_lock(&kvm->slot_lock);" before later acccessing to the kvm->arch.vgic, so I think the order can be guaranteed. Of cause as you said a explicitly acquire-load + store-release is better than before implicitly implementation.

Quoted 5 lines

If vgic_dist::ready is observed true by the lock-free read (the one which is turned into load-acquire by this patch) then the function immediately returns with no mutex_lock() executed. It is reads of vgic_dist *after* return from kvm_vgic_map_resources() that you have to worry about, and which require load-acquire semantics.

I think this is the main purpose of such lock-free reading here, to avoid lock contention on VM w/ large vCPUs for vcpus' first time run together.

store-release makes sure the changes to vgic_dist::ready become visible after the changes to vgic_dist become visible, but it doesn't guarantee the vgic_dist::ready becomes visible to reader on aother CPU **IMMEDIATELY**, thus load-acquire in reader side is request for this. Is above understanding correct ?

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
#14 lore raw

Re: [PATCH v2 2/4] KVM: arm64: vgic: Explicitly implement vgic_dist::ready ordering

Quoted 88 lines · Yao Yuan
Quoted 77 lines · Keir Fraser
Quoted 69 lines · Yao Yuan
Quoted 62 lines · Keir Fraser
Quoted 53 lines · Yao Yuan
Quoted 47 lines · Sean Christopherson
Quoted 28 lines · Yao Yuan
Quoted 27 lines · Keir Fraser

In preparation to remove synchronize_srcu() from MMIO registration, remove the distributor's dependency on this implicit barrier by direct acquire-release synchronization on the flag write and its lock-free check.

---
 arch/arm64/kvm/vgic/vgic-init.c | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +1 -1
index 502b65049703..bc83672e461b 100644
@@ -567,7 +567,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 	gpa_t dist_base;
 	int ret = 0;
 
-	if (likely(dist->ready))
+	if (likely(smp_load_acquire(&dist->ready)))
 		return 0;
 
 	mutex_lock(&kvm->slots_lock);
@@ -598,14 +598,7 @@ int kvm_vgic_map_resources(struct kvm *kvm)
 		goto out_slots;
 	}
 
Quoted 12 lines

arch/arm64/kvm/vgic/vgic-init.c int kvm_vgic_map_resources(struct kvm *kvm) · line 601

Quoted 9 lines
arch/arm64/kvm/vgic/vgic-init.c +1 -8
-	/*
-	 * kvm_io_bus_register_dev() guarantees all readers see the new MMIO
-	 * registration before returning through synchronize_srcu(), which also
-	 * implies a full memory barrier. As such, marking the distributor as
-	 * 'ready' here is guaranteed to be ordered after all vCPUs having seen
-	 * a completely configured distributor.
-	 */
-	dist->ready = true;
+	smp_store_release(&dist->ready, true);

No need the store-release and load-acquire for replacing synchronize_srcu_expedited() w/ call_srcu() IIUC:

This isn't about using call_srcu(), because it's not actually about kvm->buses. This code is concerned with ensuring that all stores to kvm->arch.vgic are ordered before the store to set kvm->arch.vgic.ready, so that vCPUs never see "ready==true" with a half-baked distributor.

Quoted 4 lines

In the current code, kvm_vgic_map_resources() relies on the synchronize_srcu() in kvm_io_bus_register_dev() to provide the ordering guarantees. Switching to smp_store_release() + smp_load_acquire() removes the dependency on the synchronize_srcu() so that the synchronize_srcu() call can be safely removed.

Quoted 7 lines

Yes, I understand this and agree with your point.

Just for discusstion: I thought it should also work even w/o introduce the load acqure + store release after switch to call_srcu(): The smp_mb() in call_srcu() order the all store to kvm->arch.vgic before store kvm->arch.vgic.ready in current implementation.

Quoted 5 lines

The load-acquire would still be required, to ensure that accesses to kvm->arch.vgic do not get reordered earlier than the lock-free check of kvm->arch.vgic.ready. Otherwise that CPU could see that the vgic is initialised, but then use speculated reads of uninitialised vgic state.

Quoted 6 lines

Thanks for your explanation.

I see. But there's "mutex_lock(&kvm->slot_lock);" before later acccessing to the kvm->arch.vgic, so I think the order can be guaranteed. Of cause as you said a explicitly acquire-load + store-release is better than before implicitly implementation.

Quoted 5 lines

If vgic_dist::ready is observed true by the lock-free read (the one which is turned into load-acquire by this patch) then the function immediately returns with no mutex_lock() executed. It is reads of vgic_dist *after* return from kvm_vgic_map_resources() that you have to worry about, and which require load-acquire semantics.

I think this is the main purpose of such lock-free reading here, to avoid lock contention on VM w/ large vCPUs for vcpus' first time run together.

Quoted 6 lines

store-release makes sure the changes to vgic_dist::ready become visible after the changes to vgic_dist become visible, but it doesn't guarantee the vgic_dist::ready becomes visible to reader on aother CPU **IMMEDIATELY**, thus load-acquire in reader side is request for this. Is above understanding correct ?

No. I get your point, the load-acuqire here for:

There's code path that the cpu read the vgic_dist reorder before the vgic_dist::ready in case of the cpu get vgic_dist::ready is true w/o the load_acquire here.

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
#15 lore raw

PATCH v2 3/4 KVM: Implement barriers before accessing kvm->buses[] on SRCU read paths

In reply to Keir Fraser

This ensures that, if a VCPU has "observed" that an IO registration has occurred, the instruction currently being trapped or emulated will also observe the IO registration.

At the same time, enforce that kvm_get_bus() is used only on the update side, ensuring that a long-term reference cannot be obtained by an SRCU reader.

---
 arch/x86/kvm/vmx/vmx.c   |  7 +++++++
 include/linux/kvm_host.h | 10 +++++++---
 virt/kvm/kvm_main.c      | 33 +++++++++++++++++++++++++++------
 3 files changed, 41 insertions(+), 9 deletions(-)
arch/x86/kvm/vmx/vmx.c +7 -0
index 191a9ed0da22..425e3d8074ab 100644
@@ -5861,6 +5861,13 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
 		if (kvm_test_request(KVM_REQ_EVENT, vcpu))
 			return 1;
 
+		/*
+		 * Ensure that any updates to kvm->buses[] observed by the
+		 * previous instruction (emulated or otherwise) are also
+		 * visible to the instruction we are about to emulate.
+		 */
+		smp_rmb();
+
 		if (!kvm_emulate_instruction(vcpu, 0))
 			return 0;
 
include/linux/kvm_host.h +7 -3
index 3bde4fb5c6aa..9132148fb467 100644
@@ -965,11 +965,15 @@ static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
 }
 
+/*
+ * Get a bus reference under the update-side lock. No long-term SRCU reader
+ * references are permitted, to avoid stale reads vs concurrent IO
+ * registrations.
+ */
 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 {
-	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
-				      lockdep_is_held(&kvm->slots_lock) ||
-				      !refcount_read(&kvm->users_count));
+	return rcu_dereference_protected(kvm->buses[idx],
+					 lockdep_is_held(&kvm->slots_lock));
 }
 
 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
virt/kvm/kvm_main.c +27 -6
index 222f0e894a0c..9ec3b96b9666 100644
@@ -1103,6 +1103,15 @@ void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
 {
 }
 
+/* Called only on cleanup and destruction paths when there are no users. */
+static inline struct kvm_io_bus *kvm_get_bus_for_destruction(struct kvm *kvm,
+							     enum kvm_bus idx)
+{
+	return rcu_dereference_protected(kvm->buses[idx],
+					 !refcount_read(&kvm->users_count));
+}
+
+
 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 {
 	struct kvm *kvm = kvm_arch_alloc_vm();
@@ -1228,7 +1237,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 out_err_no_arch_destroy_vm:
 	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
 	for (i = 0; i < KVM_NR_BUSES; i++)
-		kfree(kvm_get_bus(kvm, i));
+		kfree(kvm_get_bus_for_destruction(kvm, i));
 	kvm_free_irq_routing(kvm);
 out_err_no_irq_routing:
 	cleanup_srcu_struct(&kvm->irq_srcu);
@@ -1276,7 +1285,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 
 	kvm_free_irq_routing(kvm);
 	for (i = 0; i < KVM_NR_BUSES; i++) {
-		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
+		struct kvm_io_bus *bus = kvm_get_bus_for_destruction(kvm, i);
 
 		if (bus)
 			kvm_io_bus_destroy(bus);
@@ -5838,6 +5847,18 @@ static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
 	return -EOPNOTSUPP;
 }
 
+static struct kvm_io_bus *kvm_get_bus_srcu(struct kvm *kvm, enum kvm_bus idx)
+{
+	/*
+	 * Ensure that any updates to kvm_buses[] observed by the previous VCPU
+	 * machine instruction are also visible to the VCPU machine instruction
+	 * that triggered this call.
+	 */
+	smp_mb__after_srcu_read_lock();
+
+	return srcu_dereference(kvm->buses[idx], &kvm->srcu);
+}
+
 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
@@ -5850,7 +5871,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_write(vcpu, bus, &range, val);
@@ -5869,7 +5890,7 @@ int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 
@@ -5919,7 +5940,7 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_read(vcpu, bus, &range, val);
@@ -6028,7 +6049,7 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 
 	srcu_idx = srcu_read_lock(&kvm->srcu);
 
-	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
+	bus = kvm_get_bus_srcu(kvm, bus_idx);
 	if (!bus)
 		goto out_unlock;
 
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#16 lore raw

Re: [PATCH v2 3/4] KVM: Implement barriers before accessing kvm->buses[] on SRCU read paths

Quoted 43 lines · Keir Fraser

This ensures that, if a VCPU has "observed" that an IO registration has occurred, the instruction currently being trapped or emulated will also observe the IO registration.

At the same time, enforce that kvm_get_bus() is used only on the update side, ensuring that a long-term reference cannot be obtained by an SRCU reader.

---
 arch/x86/kvm/vmx/vmx.c   |  7 +++++++
 include/linux/kvm_host.h | 10 +++++++---
 virt/kvm/kvm_main.c      | 33 +++++++++++++++++++++++++++------
 3 files changed, 41 insertions(+), 9 deletions(-)
arch/x86/kvm/vmx/vmx.c +7 -0
index 191a9ed0da22..425e3d8074ab 100644
@@ -5861,6 +5861,13 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
 		if (kvm_test_request(KVM_REQ_EVENT, vcpu))
 			return 1;
 
+		/*
+		 * Ensure that any updates to kvm->buses[] observed by the
+		 * previous instruction (emulated or otherwise) are also
+		 * visible to the instruction we are about to emulate.
+		 */
+		smp_rmb();
+
 		if (!kvm_emulate_instruction(vcpu, 0))
 			return 0;
 
include/linux/kvm_host.h +2 -0
index 3bde4fb5c6aa..9132148fb467 100644
@@ -965,11 +965,15 @@ static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
 }
 
+/*
+ * Get a bus reference under the update-side lock. No long-term SRCU reader

include/linux/kvm_host.h static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) · line 976

Quoted 10 lines
include/linux/kvm_host.h +5 -3
+ * references are permitted, to avoid stale reads vs concurrent IO
+ * registrations.
+ */
 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 {
-	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
-				      lockdep_is_held(&kvm->slots_lock) ||
-				      !refcount_read(&kvm->users_count));
+	return rcu_dereference_protected(kvm->buses[idx],
+					 lockdep_is_held(&kvm->slots_lock));

I want to consult the true reason for using protected version here, save unnecessary READ_ONCE() ?

Quoted 99 lines
include/linux/kvm_host.h +0 -0
 }
 
 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
virt/kvm/kvm_main.c +27 -7
index 222f0e894a0c..9ec3b96b9666 100644
@@ -1103,6 +1103,15 @@ void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
 {
 }
 
+/* Called only on cleanup and destruction paths when there are no users. */
+static inline struct kvm_io_bus *kvm_get_bus_for_destruction(struct kvm *kvm,
+							     enum kvm_bus idx)
+{
+	return rcu_dereference_protected(kvm->buses[idx],
+					 !refcount_read(&kvm->users_count));
+}
+
+
 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 {
 	struct kvm *kvm = kvm_arch_alloc_vm();
@@ -1228,7 +1237,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 out_err_no_arch_destroy_vm:
 	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
 	for (i = 0; i < KVM_NR_BUSES; i++)
-		kfree(kvm_get_bus(kvm, i));
+		kfree(kvm_get_bus_for_destruction(kvm, i));
 	kvm_free_irq_routing(kvm);
 out_err_no_irq_routing:
 	cleanup_srcu_struct(&kvm->irq_srcu);
@@ -1276,7 +1285,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 
 	kvm_free_irq_routing(kvm);
 	for (i = 0; i < KVM_NR_BUSES; i++) {
-		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
+		struct kvm_io_bus *bus = kvm_get_bus_for_destruction(kvm, i);
 
 		if (bus)
 			kvm_io_bus_destroy(bus);
@@ -5838,6 +5847,18 @@ static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
 	return -EOPNOTSUPP;
 }
 
+static struct kvm_io_bus *kvm_get_bus_srcu(struct kvm *kvm, enum kvm_bus idx)
+{
+	/*
+	 * Ensure that any updates to kvm_buses[] observed by the previous VCPU
+	 * machine instruction are also visible to the VCPU machine instruction
+	 * that triggered this call.
+	 */
+	smp_mb__after_srcu_read_lock();
+
+	return srcu_dereference(kvm->buses[idx], &kvm->srcu);
+}
+
 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
@@ -5850,7 +5871,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_write(vcpu, bus, &range, val);
@@ -5869,7 +5890,7 @@ int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 
@@ -5919,7 +5940,7 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_read(vcpu, bus, &range, val);
@@ -6028,7 +6049,7 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 
 	srcu_idx = srcu_read_lock(&kvm->srcu);
 
-	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
+	bus = kvm_get_bus_srcu(kvm, bus_idx);
 	if (!bus)
 		goto out_unlock;
 
--

2.50.0.727.gbf7dc18ff4-goog

#17 lore raw

Re: [PATCH v2 3/4] KVM: Implement barriers before accessing kvm->buses[] on SRCU read paths

Quoted 9 lines · Yao Yuan
Quoted 8 lines · Keir Fraser
include/linux/kvm_host.h +0 -0
index 3bde4fb5c6aa..9132148fb467 100644
@@ -965,11 +965,15 @@ static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
 }
 
Quoted 15 lines

include/linux/kvm_host.h static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) · line 976

Quoted 12 lines
include/linux/kvm_host.h +7 -3
+/*
+ * Get a bus reference under the update-side lock. No long-term SRCU reader
+ * references are permitted, to avoid stale reads vs concurrent IO
+ * registrations.
+ */
 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 {
-	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
-				      lockdep_is_held(&kvm->slots_lock) ||
-				      !refcount_read(&kvm->users_count));
+	return rcu_dereference_protected(kvm->buses[idx],
+					 lockdep_is_held(&kvm->slots_lock));

I want to consult the true reason for using protected version here, save unnecessary READ_ONCE() ?

Avoiding the READ_ONCE() is a happy bonus. The main goal is to help document and enforce that kvm_get_bus() can only be used if slots_lock is held. Keeping this as srcu_dereference_check() would result in PROVE_RCU getting a false negative if the caller held kvm->srcu but not slots_lock.

From a documentation perspective, rcu_dereference_protected() (hopefully) helps highlight that there's something "special" about this helper, e.g. gives the reader a hint that they probably shouldn't be using kvm_get_bus().

#18 lore raw

Re: [PATCH v2 3/4] KVM: Implement barriers before accessing kvm->buses[] on SRCU read paths

Quoted 26 lines · Sean Christopherson
Quoted 9 lines · Yao Yuan
Quoted 8 lines · Keir Fraser
include/linux/kvm_host.h +0 -0
index 3bde4fb5c6aa..9132148fb467 100644
@@ -965,11 +965,15 @@ static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
 }
 
Quoted 15 lines

include/linux/kvm_host.h static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) · line 976

Quoted 12 lines
include/linux/kvm_host.h +7 -3
+/*
+ * Get a bus reference under the update-side lock. No long-term SRCU reader
+ * references are permitted, to avoid stale reads vs concurrent IO
+ * registrations.
+ */
 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 {
-	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
-				      lockdep_is_held(&kvm->slots_lock) ||
-				      !refcount_read(&kvm->users_count));
+	return rcu_dereference_protected(kvm->buses[idx],
+					 lockdep_is_held(&kvm->slots_lock));

I want to consult the true reason for using protected version here, save unnecessary READ_ONCE() ?

Quoted 4 lines

Avoiding the READ_ONCE() is a happy bonus. The main goal is to help document and enforce that kvm_get_bus() can only be used if slots_lock is held. Keeping this as srcu_dereference_check() would result in PROVE_RCU getting a false negative if the caller held kvm->srcu but not slots_lock.

Ah, I noticed the srcu_read_lock_held(ssp) in srcu_dereference_check() this time !

Quoted 4 lines

From a documentation perspective, rcu_dereference_protected() (hopefully) helps highlight that there's something "special" about this helper, e.g. gives the reader a hint that they probably shouldn't be using kvm_get_bus().

Yes, I got it. Thanks for your nice explanation!

Quoted 1 line
#19 lore raw

Re: [PATCH v2 3/4] KVM: Implement barriers before accessing kvm->buses[] on SRCU read paths

In reply to Yao Yuan
Quoted 42 lines · Yao Yuan
Quoted 41 lines · Keir Fraser

This ensures that, if a VCPU has "observed" that an IO registration has occurred, the instruction currently being trapped or emulated will also observe the IO registration.

At the same time, enforce that kvm_get_bus() is used only on the update side, ensuring that a long-term reference cannot be obtained by an SRCU reader.

---
 arch/x86/kvm/vmx/vmx.c   |  7 +++++++
 include/linux/kvm_host.h | 10 +++++++---
 virt/kvm/kvm_main.c      | 33 +++++++++++++++++++++++++++------
 3 files changed, 41 insertions(+), 9 deletions(-)
arch/x86/kvm/vmx/vmx.c +7 -0
index 191a9ed0da22..425e3d8074ab 100644
@@ -5861,6 +5861,13 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
 		if (kvm_test_request(KVM_REQ_EVENT, vcpu))
 			return 1;
 
+		/*
+		 * Ensure that any updates to kvm->buses[] observed by the
+		 * previous instruction (emulated or otherwise) are also
+		 * visible to the instruction we are about to emulate.
+		 */
+		smp_rmb();
+
 		if (!kvm_emulate_instruction(vcpu, 0))
 			return 0;
 
include/linux/kvm_host.h +0 -0
index 3bde4fb5c6aa..9132148fb467 100644
@@ -965,11 +965,15 @@ static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
 }
 
Quoted 15 lines

include/linux/kvm_host.h static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) · line 976

Quoted 12 lines
include/linux/kvm_host.h +7 -3
+/*
+ * Get a bus reference under the update-side lock. No long-term SRCU reader
+ * references are permitted, to avoid stale reads vs concurrent IO
+ * registrations.
+ */
 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 {
-	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
-				      lockdep_is_held(&kvm->slots_lock) ||
-				      !refcount_read(&kvm->users_count));
+	return rcu_dereference_protected(kvm->buses[idx],
+					 lockdep_is_held(&kvm->slots_lock));

I want to consult the true reason for using protected version here, save unnecessary READ_ONCE() ?

We don't want this function to be callable from SRCU read section, but *only* during teardown. Hence protected version provides a better, stricter safety check (that there are no users).

-- Keir

Quoted 99 lines
Quoted 99 lines
include/linux/kvm_host.h +0 -0
 }
 
 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
virt/kvm/kvm_main.c +27 -7
index 222f0e894a0c..9ec3b96b9666 100644
@@ -1103,6 +1103,15 @@ void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
 {
 }
 
+/* Called only on cleanup and destruction paths when there are no users. */
+static inline struct kvm_io_bus *kvm_get_bus_for_destruction(struct kvm *kvm,
+							     enum kvm_bus idx)
+{
+	return rcu_dereference_protected(kvm->buses[idx],
+					 !refcount_read(&kvm->users_count));
+}
+
+
 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 {
 	struct kvm *kvm = kvm_arch_alloc_vm();
@@ -1228,7 +1237,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 out_err_no_arch_destroy_vm:
 	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
 	for (i = 0; i < KVM_NR_BUSES; i++)
-		kfree(kvm_get_bus(kvm, i));
+		kfree(kvm_get_bus_for_destruction(kvm, i));
 	kvm_free_irq_routing(kvm);
 out_err_no_irq_routing:
 	cleanup_srcu_struct(&kvm->irq_srcu);
@@ -1276,7 +1285,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 
 	kvm_free_irq_routing(kvm);
 	for (i = 0; i < KVM_NR_BUSES; i++) {
-		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
+		struct kvm_io_bus *bus = kvm_get_bus_for_destruction(kvm, i);
 
 		if (bus)
 			kvm_io_bus_destroy(bus);
@@ -5838,6 +5847,18 @@ static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
 	return -EOPNOTSUPP;
 }
 
+static struct kvm_io_bus *kvm_get_bus_srcu(struct kvm *kvm, enum kvm_bus idx)
+{
+	/*
+	 * Ensure that any updates to kvm_buses[] observed by the previous VCPU
+	 * machine instruction are also visible to the VCPU machine instruction
+	 * that triggered this call.
+	 */
+	smp_mb__after_srcu_read_lock();
+
+	return srcu_dereference(kvm->buses[idx], &kvm->srcu);
+}
+
 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
@@ -5850,7 +5871,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_write(vcpu, bus, &range, val);
@@ -5869,7 +5890,7 @@ int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 
@@ -5919,7 +5940,7 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_read(vcpu, bus, &range, val);
@@ -6028,7 +6049,7 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 
 	srcu_idx = srcu_read_lock(&kvm->srcu);
 
-	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
+	bus = kvm_get_bus_srcu(kvm, bus_idx);
 	if (!bus)
 		goto out_unlock;
 
--

2.50.0.727.gbf7dc18ff4-goog

#20 lore raw

Re: [PATCH v2 3/4] KVM: Implement barriers before accessing kvm->buses[] on SRCU read paths

Quoted 59 lines · Keir Fraser
Quoted 42 lines · Yao Yuan
Quoted 41 lines · Keir Fraser

This ensures that, if a VCPU has "observed" that an IO registration has occurred, the instruction currently being trapped or emulated will also observe the IO registration.

At the same time, enforce that kvm_get_bus() is used only on the update side, ensuring that a long-term reference cannot be obtained by an SRCU reader.

---
 arch/x86/kvm/vmx/vmx.c   |  7 +++++++
 include/linux/kvm_host.h | 10 +++++++---
 virt/kvm/kvm_main.c      | 33 +++++++++++++++++++++++++++------
 3 files changed, 41 insertions(+), 9 deletions(-)
arch/x86/kvm/vmx/vmx.c +7 -0
index 191a9ed0da22..425e3d8074ab 100644
@@ -5861,6 +5861,13 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
 		if (kvm_test_request(KVM_REQ_EVENT, vcpu))
 			return 1;
 
+		/*
+		 * Ensure that any updates to kvm->buses[] observed by the
+		 * previous instruction (emulated or otherwise) are also
+		 * visible to the instruction we are about to emulate.
+		 */
+		smp_rmb();
+
 		if (!kvm_emulate_instruction(vcpu, 0))
 			return 0;
 
include/linux/kvm_host.h +0 -0
index 3bde4fb5c6aa..9132148fb467 100644
@@ -965,11 +965,15 @@ static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm)
 	return !!(kvm->manual_dirty_log_protect & KVM_DIRTY_LOG_INITIALLY_SET);
 }
 
Quoted 15 lines

include/linux/kvm_host.h static inline bool kvm_dirty_log_manual_protect_and_init_set(struct kvm *kvm) · line 976

Quoted 12 lines
include/linux/kvm_host.h +7 -3
+/*
+ * Get a bus reference under the update-side lock. No long-term SRCU reader
+ * references are permitted, to avoid stale reads vs concurrent IO
+ * registrations.
+ */
 static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx)
 {
-	return srcu_dereference_check(kvm->buses[idx], &kvm->srcu,
-				      lockdep_is_held(&kvm->slots_lock) ||
-				      !refcount_read(&kvm->users_count));
+	return rcu_dereference_protected(kvm->buses[idx],
+					 lockdep_is_held(&kvm->slots_lock));

I want to consult the true reason for using protected version here, save unnecessary READ_ONCE() ?

Quoted 3 lines

We don't want this function to be callable from SRCU read section, but *only* during teardown. Hence protected version provides a better, stricter safety check (that there are no users).

I see, thanks for your explanation!

Quoted 103 lines

-- Keir

Quoted 99 lines
Quoted 99 lines
include/linux/kvm_host.h +0 -0
 }
 
 static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i)
virt/kvm/kvm_main.c +27 -7
index 222f0e894a0c..9ec3b96b9666 100644
@@ -1103,6 +1103,15 @@ void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
 {
 }
 
+/* Called only on cleanup and destruction paths when there are no users. */
+static inline struct kvm_io_bus *kvm_get_bus_for_destruction(struct kvm *kvm,
+							     enum kvm_bus idx)
+{
+	return rcu_dereference_protected(kvm->buses[idx],
+					 !refcount_read(&kvm->users_count));
+}
+
+
 static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 {
 	struct kvm *kvm = kvm_arch_alloc_vm();
@@ -1228,7 +1237,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 out_err_no_arch_destroy_vm:
 	WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
 	for (i = 0; i < KVM_NR_BUSES; i++)
-		kfree(kvm_get_bus(kvm, i));
+		kfree(kvm_get_bus_for_destruction(kvm, i));
 	kvm_free_irq_routing(kvm);
 out_err_no_irq_routing:
 	cleanup_srcu_struct(&kvm->irq_srcu);
@@ -1276,7 +1285,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 
 	kvm_free_irq_routing(kvm);
 	for (i = 0; i < KVM_NR_BUSES; i++) {
-		struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
+		struct kvm_io_bus *bus = kvm_get_bus_for_destruction(kvm, i);
 
 		if (bus)
 			kvm_io_bus_destroy(bus);
@@ -5838,6 +5847,18 @@ static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
 	return -EOPNOTSUPP;
 }
 
+static struct kvm_io_bus *kvm_get_bus_srcu(struct kvm *kvm, enum kvm_bus idx)
+{
+	/*
+	 * Ensure that any updates to kvm_buses[] observed by the previous VCPU
+	 * machine instruction are also visible to the VCPU machine instruction
+	 * that triggered this call.
+	 */
+	smp_mb__after_srcu_read_lock();
+
+	return srcu_dereference(kvm->buses[idx], &kvm->srcu);
+}
+
 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
@@ -5850,7 +5871,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_write(vcpu, bus, &range, val);
@@ -5869,7 +5890,7 @@ int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 
@@ -5919,7 +5940,7 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 		.len = len,
 	};
 
-	bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
+	bus = kvm_get_bus_srcu(vcpu->kvm, bus_idx);
 	if (!bus)
 		return -ENOMEM;
 	r = __kvm_io_bus_read(vcpu, bus, &range, val);
@@ -6028,7 +6049,7 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 
 	srcu_idx = srcu_read_lock(&kvm->srcu);
 
-	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
+	bus = kvm_get_bus_srcu(kvm, bus_idx);
 	if (!bus)
 		goto out_unlock;
 
--

2.50.0.727.gbf7dc18ff4-goog

#21 lore raw

PATCH v2 4/4 KVM: Avoid synchronize_srcu() in kvm_io_bus_register_dev()

In reply to Keir Fraser

Device MMIO registration may happen quite frequently during VM boot, and the SRCU synchronization each time has a measurable effect on VM startup time. In our experiments it can account for around 25% of a VM's startup time.

Replace the synchronization with a deferred free of the old kvm_io_bus structure.

---
 include/linux/kvm_host.h |  1 +
 virt/kvm/kvm_main.c      | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)
include/linux/kvm_host.h +1 -0
index 9132148fb467..802ca46f7537 100644
@@ -205,6 +205,7 @@ struct kvm_io_range {
 struct kvm_io_bus {
 	int dev_count;
 	int ioeventfd_count;
+	struct rcu_head rcu;
 	struct kvm_io_range range[];
 };
 
virt/kvm/kvm_main.c +8 -2
index 9ec3b96b9666..f690a4997a0f 100644
@@ -5948,6 +5948,13 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
 }
 EXPORT_SYMBOL_GPL(kvm_io_bus_read);
 
+static void __free_bus(struct rcu_head *rcu)
+{
+	struct kvm_io_bus *bus = container_of(rcu, struct kvm_io_bus, rcu);
+
+	kfree(bus);
+}
+
 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 			    int len, struct kvm_io_device *dev)
 {
@@ -5986,8 +5993,7 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 	memcpy(new_bus->range + i + 1, bus->range + i,
 		(bus->dev_count - i) * sizeof(struct kvm_io_range));
 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
-	synchronize_srcu_expedited(&kvm->srcu);
-	kfree(bus);
+	call_srcu(&kvm->srcu, &bus->rcu, __free_bus);
 
 	return 0;
 }
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog