Skip to contentSkip to conversation

KVM: s390: Introducing kvm_arch_set_irq_inatomic Fast Inject

PATCH Superseded Douglas Freimuth 2 participants 7 messages May 11 – May 13

Subsystems (3)

v6 of 10 Superseded Latest is v10 →

lore
Share & tools
lore
#0 lore raw

v6 cover KVM: s390: Introducing kvm_arch_set_irq_inatomic Fast Inject

S390 needs this series of three patches in order to enable a non-blocking path for irqfd injection on s390 via kvm_arch_set_irq_inatomic(). Before these changes, kvm_arch_set_irq_inatomic() would just return -EWOULDBLOCK and place all interrupts on the global work queue, which must subsequently be processed by a different thread. This series of patches implements an s390 version of inatomic and is relevant to virtio-blk and virtio-net and was tested against virtio-pci and virtio-ccw.

The inatomic fast path cannot lose control since it is running with interrupts disabled. This meant making the following changes that exist on the slow path today. First, the adapter_indicators page needs to be mapped since it is accessed with interrupts disabled, so we added map/unmap functions. Second, access to shared resources between the fast and slow paths needed to be changed from mutex and semaphores to raw_spin_lock's. Finally, the memory allocation on the slow path utilizes GFP_KERNEL_ACCOUNT but we had to implement the fast path with GFP_ATOMIC allocation. Each of these enhancements were required to prevent blocking on the fast inject path. In order handle cross platform common code raw spin locks were added.

Statistical counters have been added to enable analysis of irq injection on the fast path and slow path including io_390_inatomic, io_flic_inject_airq, io_set_adapter_int and io_390_inatomic_adapter_masked. And counters have been added to analyze map/unmap of the adapter_indicator
pages in non-Secure Execution environments and to track fencing of Fast Inject in Secure Execution environments. In order to take advantage of this kernel series with virtio-pci, a QEMU that includes the 's390x/pci: set kvm_msi_via_irqfd_allowed' fix is needed. Additionally, the guest xml needs a thread pool and threads explicitly assigned per disk device using the common way of defining threads for disks.

Patch 1 enables map/unmap of adapter indicator pages but for Secure Execution environments it avoids the long term mapping.

v5->v6: Change one instance of put_page to unpin_user_page v5->v6: Make local interrupt lock li->lock raw_spinlock v5->v6: Defer two instances of kfree until after release raw_spinlock

Douglas Freimuth (4):
KVM: s390: Add map/unmap ioctl and clean mappings post-guest
KVM: s390: Enable adapter_indicators_set to use mapped pages
KVM: S390: Change the fi->lock and li->lock to raw_spinlock
KVM: s390: Introducing kvm_arch_set_irq_inatomic fast inject

 arch/s390/include/asm/kvm_host.h |  15 +-
 arch/s390/kvm/intercept.c        |   8 +-
 arch/s390/kvm/interrupt.c        | 545 ++++++++++++++++++++++++-------
 arch/s390/kvm/kvm-s390.c         |  33 +-
 arch/s390/kvm/kvm-s390.h         |   5 +-
 arch/s390/kvm/sigp.c             |   4 +-
 6 files changed, 475 insertions(+), 135 deletions(-)
Signature · 1 line
2.52.0
#1 lore raw

PATCH v6 1/4 KVM: s390: Add map/unmap ioctl and clean mappings post-guest

S390 needs map/unmap ioctls, which map the adapter set indicator pages, so the pages can be accessed when interrupts are disabled. The mappings are cleaned up when the guest is removed.

Map/Unmap ioctls are fenced in order to avoid the longterm pinning in Secure Execution environments. In Secure Execution environments the path of execution available before this patch is followed.

Statistical counters to count map/unmap functions for adapter indicator pages are added. The counters can be used to analyze map/unmap functions in non-Secure Execution environments and similarly can be used to analyze Secure Execution environments where the counters will not be incremented as the adapter indicator pages are not mapped.

---
 arch/s390/include/asm/kvm_host.h |   5 +
 arch/s390/kvm/interrupt.c        | 192 ++++++++++++++++++++++++++-----
 arch/s390/kvm/kvm-s390.c         |   8 ++
 arch/s390/kvm/kvm-s390.h         |   2 +
 4 files changed, 177 insertions(+), 30 deletions(-)
arch/s390/include/asm/kvm_host.h +5 -0
index 8a4f4a39f7a2..fbb2406b31d2 100644
@@ -448,6 +448,8 @@ struct kvm_vcpu_arch {
 struct kvm_vm_stat {
 	struct kvm_vm_stat_generic generic;
 	u64 inject_io;
+	u64 io_390_adapter_map;
+	u64 io_390_adapter_unmap;
 	u64 inject_float_mchk;
 	u64 inject_pfault_done;
 	u64 inject_service_signal;
@@ -479,6 +481,9 @@ struct s390_io_adapter {
 	bool masked;
 	bool swap;
 	bool suppressible;
+	raw_spinlock_t maps_lock;
+	struct list_head maps;
+	unsigned int nr_maps;
 };
 
 #define MAX_S390_IO_ADAPTERS ((MAX_ISC + 1) * 8)
arch/s390/kvm/interrupt.c +162 -30

238 lines not on this page · raw message

arch/s390/kvm/kvm-s390.c +8 -0
index e09960c2e6ed..74f453f039a3 100644
@@ -68,6 +68,8 @@
 const struct kvm_stats_desc kvm_vm_stats_desc[] = {
 	KVM_GENERIC_VM_STATS(),
 	STATS_DESC_COUNTER(VM, inject_io),
+	STATS_DESC_COUNTER(VM, io_390_adapter_map),
+	STATS_DESC_COUNTER(VM, io_390_adapter_unmap),
 	STATS_DESC_COUNTER(VM, inject_float_mchk),
 	STATS_DESC_COUNTER(VM, inject_pfault_done),
 	STATS_DESC_COUNTER(VM, inject_service_signal),
@@ -2497,6 +2499,11 @@ static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
 	return r;
 }
 
+static void kvm_s390_unmap_all_adapters_pv(struct kvm *kvm)
+{
+	kvm_s390_unmap_all_adapters(kvm);
+}
+
 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
 {
 	const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
@@ -2513,6 +2520,7 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
 		if (kvm_s390_pv_is_protected(kvm))
 			break;
 
+		kvm_s390_unmap_all_adapters_pv(kvm);
 		mmap_write_lock(kvm->mm);
 		/*
 		 * Disable creation of new THPs. Existing THPs can stay, they
arch/s390/kvm/kvm-s390.h +2 -0
index dc0573b7aa4b..7ba885cb6bd1 100644
@@ -560,6 +560,8 @@ void kvm_s390_gisa_disable(struct kvm *kvm);
 void kvm_s390_gisa_enable(struct kvm *kvm);
 int __init kvm_s390_gib_init(u8 nisc);
 void kvm_s390_gib_destroy(void);
+void kvm_s390_unmap_all_adapters(struct kvm *kvm);
+
 
 /* implemented in guestdbg.c */
 void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu);
Signature · 1 line
2.52.0
#2 lore raw

Re: [PATCH v6 1/4] KVM: s390: Add map/unmap ioctl and clean mappings post-guest

Quoted 8 lines · Douglas Freimuth

S390 needs map/unmap ioctls, which map the adapter set indicator pages, so the pages can be accessed when interrupts are disabled. The mappings are cleaned up when the guest is removed.

Map/Unmap ioctls are fenced in order to avoid the longterm pinning in Secure Execution environments. In Secure Execution environments the path of execution available before this patch is followed.

Quoted 5 lines

Statistical counters to count map/unmap functions for adapter indicator pages are added. The counters can be used to analyze map/unmap functions in non-Secure Execution environments and similarly can be used to analyze Secure Execution environments where the counters will not be incremented as the adapter indicator pages are not mapped.

I think it's worth mentioning that this patch also converts to pin_user_pages_remote for both the ioctl as well as the pin-on-demand logic in adapter_indicators_set()

[...]

arch/s390/kvm/interrupt.c int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) · line 2568

Quoted 6 lines
arch/s390/kvm/interrupt.c +6 -0
+
+void kvm_s390_unmap_all_adapters(struct kvm *kvm)
+{
+	struct s390_map_info *map, *tmp;
+	unsigned long flags;
+	int i, idx;

Sashiko makes a valid point here:
the ioctl is doing unmaps with the kvm->lock held.
the kvm_s390_unmap_all_adapters_pv() path is also.
But the registration of adapters is not, nor is the direct call to kvm_s390_unmap_all_adapter() from kvm_s390_destroy_adapters(), the latter of which is not too big of a deal since vcpus are already gone.

The goal here is to protect the contents of kvm->arch.adapters[] from changing while you're running the list here.

Can you look at acquiring the kvm->lock over register_io_adapter() as part of this patch? AFAICT it should be fine to do so. There are other flic operations like kvm_s390_clear_float_irqs() which do so.

Quoted 24 lines
arch/s390/kvm/interrupt.c +24 -0
+
+	for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) {
+		struct s390_io_adapter *adapter = kvm->arch.adapters[i];
+		LIST_HEAD(local_list);
+
+		if (!adapter)
+			continue;
+
+		raw_spin_lock_irqsave(&adapter->maps_lock, flags);
+		list_splice_init(&adapter->maps, &local_list);
+		adapter->nr_maps = 0;
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+
+		list_for_each_entry_safe(map, tmp, &local_list, list) {
+			list_del(&map->list);
+			idx = srcu_read_lock(&kvm->srcu);
+			mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT);
+			set_page_dirty_lock(map->page);
+			srcu_read_unlock(&kvm->srcu, idx);
+			unpin_user_page(map->page);
+			kfree(map);
+		}
+	}
+}

[...]

arch/s390/kvm/kvm-s390.c static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd, · line 2505

Quoted 4 lines
arch/s390/kvm/kvm-s390.c +4 -0
+static void kvm_s390_unmap_all_adapters_pv(struct kvm *kvm)
+{
+	kvm_s390_unmap_all_adapters(kvm);
+}

Just get rid of kvm_s390_unmap_all_adapters_pv() and call kvm_s390_unmap_all_adapters() directly in kvm_s390_handle_pv()?

Otherwise this patch looks good to me.

Thanks,
Matt

#3 lore raw

PATCH v6 2/4 KVM: s390: Enable adapter_indicators_set to use mapped pages

In reply to Douglas Freimuth

The S390 adapter_indicators_set function needs to be able to use mapped pages so that worked can be processed,on a fast path when interrupts are disabled. If adapter indicator pages are not mapped then local mapping is done on a slow path as it is prior to this patch. For example, Secure Execution environments will take the local mapping path as it does prior to this patch.

---
 arch/s390/kvm/interrupt.c | 94 ++++++++++++++++++++++++++++-----------
 1 file changed, 69 insertions(+), 25 deletions(-)
arch/s390/kvm/interrupt.c +69 -25
index 96ead830c35f..12d8d38c260d 100644
@@ -2883,41 +2883,85 @@ static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap)
 	return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit;
 }
 
+static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter,
+					  u64 addr)
+{
+	struct s390_map_info *map;
+
+	if (!adapter)
+		return NULL;
+
+	list_for_each_entry(map, &adapter->maps, list) {
+		if (map->addr == addr)
+			return map;
+	}
+	return NULL;
+}
+
 static int adapter_indicators_set(struct kvm *kvm,
 				  struct s390_io_adapter *adapter,
 				  struct kvm_s390_adapter_int *adapter_int)
 {
 	unsigned long bit;
 	int summary_set, idx;
-	struct page *ind_page, *summary_page;
+	struct s390_map_info *ind_info, *summary_info;
 	void *map;
+	struct page *ind_page, *summary_page;
+	unsigned long flags;
 
-	ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0);
-	if (!ind_page)
-		return -1;
-	summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0);
-	if (!summary_page) {
-		unpin_user_page(ind_page);
-		return -1;
+	raw_spin_lock_irqsave(&adapter->maps_lock, flags);
+	ind_info = get_map_info(adapter, adapter_int->ind_addr);
+	if (!ind_info) {
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+		ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0);
+		if (!ind_page)
+			return -1;
+		idx = srcu_read_lock(&kvm->srcu);
+		map = page_address(ind_page);
+		bit = get_ind_bit(adapter_int->ind_addr,
+				  adapter_int->ind_offset, adapter->swap);
+		set_bit(bit, map);
+		mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT);
+		set_page_dirty_lock(ind_page);
+		srcu_read_unlock(&kvm->srcu, idx);
+	} else {
+		map = page_address(ind_info->page);
+		bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap);
+		set_bit(bit, map);
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+	}
+	raw_spin_lock_irqsave(&adapter->maps_lock, flags);
+	summary_info = get_map_info(adapter, adapter_int->summary_addr);
+	if (!summary_info) {
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+		summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0);
+		if (!summary_page) {
+			if (!ind_info) {
+				WARN_ON_ONCE(!ind_page);
+				unpin_user_page(ind_page);
+			}
+			return -1;
+		}
+		idx = srcu_read_lock(&kvm->srcu);
+		map = page_address(summary_page);
+		bit = get_ind_bit(adapter_int->summary_addr,
+				  adapter_int->summary_offset, adapter->swap);
+		summary_set = test_and_set_bit(bit, map);
+		mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT);
+		set_page_dirty_lock(summary_page);
+		srcu_read_unlock(&kvm->srcu, idx);
+	} else {
+		map = page_address(summary_info->page);
+		bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset,
+				  adapter->swap);
+		summary_set = test_and_set_bit(bit, map);
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
 	}
 
-	idx = srcu_read_lock(&kvm->srcu);
-	map = page_address(ind_page);
-	bit = get_ind_bit(adapter_int->ind_addr,
-			  adapter_int->ind_offset, adapter->swap);
-	set_bit(bit, map);
-	mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT);
-	set_page_dirty_lock(ind_page);
-	map = page_address(summary_page);
-	bit = get_ind_bit(adapter_int->summary_addr,
-			  adapter_int->summary_offset, adapter->swap);
-	summary_set = test_and_set_bit(bit, map);
-	mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT);
-	set_page_dirty_lock(summary_page);
-	srcu_read_unlock(&kvm->srcu, idx);
-
-	unpin_user_page(ind_page);
-	unpin_user_page(summary_page);
+	if (!ind_info)
+		unpin_user_page(ind_page);
+	if (!summary_info)
+		unpin_user_page(summary_page);
 	return summary_set ? 0 : 1;
 }
 
Signature · 1 line
2.52.0
#4 lore raw

Re: [PATCH v6 2/4] KVM: s390: Enable adapter_indicators_set to use mapped pages

Quoted 1 line · Douglas Freimuth

The S390 adapter_indicators_set function needs to be able to use mapped

Nit: s/S390/s390/

Quoted 5 lines

pages so that worked can be processed,on a fast path when interrupts are disabled. If adapter indicator pages are not mapped then local mapping is done on a slow path as it is prior to this patch. For example, Secure Execution environments will take the local mapping path as it does prior to this patch.

Slight tweak to the commit message: adapter_indicators_set doesn't really _need_ to use the long-term mapped pages. It's an optimization to do so, while your will later add the _fast() variant where it is a requirement.

Maybe re-word like:
'The S390 adapter_indicators_set function can now be optimized to use long-term mapped pages when available so that work can be processed...'

[...]

Quoted 24 lines

static int adapter_indicators_set(struct kvm *kvm,

arch/s390/kvm/interrupt.c +8 -8
 				  struct s390_io_adapter *adapter,
 				  struct kvm_s390_adapter_int *adapter_int)
 {
 	unsigned long bit;
 	int summary_set, idx;
-	struct page *ind_page, *summary_page;
+	struct s390_map_info *ind_info, *summary_info;
 	void *map;
+	struct page *ind_page, *summary_page;
+	unsigned long flags;
 
-	ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0);
-	if (!ind_page)
-		return -1;
-	summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0);
-	if (!summary_page) {
-		unpin_user_page(ind_page);
-		return -1;
+	raw_spin_lock_irqsave(&adapter->maps_lock, flags);
+	ind_info = get_map_info(adapter, adapter_int->ind_addr);
+	if (!ind_info) {
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+		ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0);

arch/s390/kvm/interrupt.c static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) · line 2926

Quoted 10 lines
arch/s390/kvm/interrupt.c +10 -0
+		if (!ind_page)
+			return -1;
+		idx = srcu_read_lock(&kvm->srcu);
+		map = page_address(ind_page);
+		bit = get_ind_bit(adapter_int->ind_addr,
+				  adapter_int->ind_offset, adapter->swap);
+		set_bit(bit, map);
+		mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT);
+		set_page_dirty_lock(ind_page);
+		srcu_read_unlock(&kvm->srcu, idx);

I think you can simplify some of this unpin logic by just calling unpin_user_page(ind_page) here? You don't use it past this point and you aren't holding a raw spinlock ...

Quoted 6 lines
arch/s390/kvm/interrupt.c +6 -0
+	} else {
+		map = page_address(ind_info->page);
+		bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap);
+		set_bit(bit, map);
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+	}

arch/s390/kvm/interrupt.c static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) · line 2942

Quoted 10 lines
arch/s390/kvm/interrupt.c +10 -0
+	raw_spin_lock_irqsave(&adapter->maps_lock, flags);
+	summary_info = get_map_info(adapter, adapter_int->summary_addr);
+	if (!summary_info) {
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
+		summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0);
+		if (!summary_page) {
+			if (!ind_info) {
+				WARN_ON_ONCE(!ind_page);
+				unpin_user_page(ind_page);
+			}

... Then you wouldn't need this !ind_info logic, it's safe to return immediately

I also don't think that WARN_ON_ONCE does what was intended. Looking back any my v3 comments, the idea was to catch a case where we map indicator pages and set the associated bits but cannot map summary page. That means if you want a WARN here, then something like...

if (!summary_page) {
WARN_ON_ONCE(ind_page || ind_info);
return -1;
}

Also ind_page needs to be initialized to NULL for that to work.

arch/s390/kvm/interrupt.c static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) · line 2952

Quoted 10 lines
arch/s390/kvm/interrupt.c +10 -0
+			return -1;
+		}
+		idx = srcu_read_lock(&kvm->srcu);
+		map = page_address(summary_page);
+		bit = get_ind_bit(adapter_int->summary_addr,
+				  adapter_int->summary_offset, adapter->swap);
+		summary_set = test_and_set_bit(bit, map);
+		mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT);
+		set_page_dirty_lock(summary_page);
+		srcu_read_unlock(&kvm->srcu, idx);

Same here, unpin_user_page(summary_page) ...

Quoted 19 lines
arch/s390/kvm/interrupt.c +6 -11
+	} else {
+		map = page_address(summary_info->page);
+		bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset,
+				  adapter->swap);
+		summary_set = test_and_set_bit(bit, map);
+		raw_spin_unlock_irqrestore(&adapter->maps_lock, flags);
 	}
 
-	idx = srcu_read_lock(&kvm->srcu);
-	map = page_address(ind_page);
-	bit = get_ind_bit(adapter_int->ind_addr,
-			  adapter_int->ind_offset, adapter->swap);
-	set_bit(bit, map);
-	mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT);
-	set_page_dirty_lock(ind_page);
-	map = page_address(summary_page);
-	bit = get_ind_bit(adapter_int->summary_addr,
-			  adapter_int->summary_offset, adapter->swap);
-	summary_set = test_and_set_bit(bit, map);

arch/s390/kvm/interrupt.c static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) · line 2964

Quoted 10 lines
arch/s390/kvm/interrupt.c +4 -6
-	mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT);
-	set_page_dirty_lock(summary_page);
-	srcu_read_unlock(&kvm->srcu, idx);
-
-	unpin_user_page(ind_page);
-	unpin_user_page(summary_page);
+	if (!ind_info)
+		unpin_user_page(ind_page);
+	if (!summary_info)
+		unpin_user_page(summary_page);

... And then you don't need either of these checks/unpins?

Quoted 3 lines

return summary_set ? 0 : 1;
}

#5 lore raw

PATCH v6 3/4 KVM: S390: Change fi->lock, li->lock to raw

In reply to Douglas Freimuth

s390 needs to maintain support for common code. This requires the floating interrupt lock and local interrupt lock to be changed to raw_spinlocks. Most or all other platforms use raw_spinlocks in their inatomic implementations since they will be called with interrupts disabled and the locks in that context cannot sleep.

---
 arch/s390/include/asm/kvm_host.h |   4 +-
 arch/s390/kvm/intercept.c        |   8 +--
 arch/s390/kvm/interrupt.c        | 112 +++++++++++++++----------------
 arch/s390/kvm/kvm-s390.c         |   4 +-
 arch/s390/kvm/sigp.c             |   4 +-
 5 files changed, 66 insertions(+), 66 deletions(-)
arch/s390/include/asm/kvm_host.h +2 -2
index fbb2406b31d2..295f0770bcb1 100644
@@ -321,7 +321,7 @@ struct kvm_s390_irq_payload {
 };
 
 struct kvm_s390_local_interrupt {
-	spinlock_t lock;
+	raw_spinlock_t lock;
 	DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
 	struct kvm_s390_irq_payload irq;
 	unsigned long pending_irqs;
@@ -353,7 +353,7 @@ struct kvm_s390_local_interrupt {
 struct kvm_s390_float_interrupt {
 	unsigned long pending_irqs;
 	unsigned long masked_irqs;
-	spinlock_t lock;
+	raw_spinlock_t lock;
 	struct list_head lists[FIRQ_LIST_COUNT];
 	int counters[FIRQ_MAX_COUNT];
 	struct kvm_s390_mchk_info mchk;
arch/s390/kvm/intercept.c +4 -4
index 39aff324203e..fff2baa31651 100644
@@ -64,10 +64,10 @@ static int handle_stop(struct kvm_vcpu *vcpu)
 		return 0;
 
 	/* avoid races with the injection/SIGP STOP code */
-	spin_lock(&li->lock);
+	raw_spin_lock(&li->lock);
 	flags = li->irq.stop.flags;
 	stop_pending = kvm_s390_is_stop_irq_pending(vcpu);
-	spin_unlock(&li->lock);
+	raw_spin_unlock(&li->lock);
 
 	trace_kvm_s390_stop_request(stop_pending, flags);
 	if (!stop_pending)
@@ -518,7 +518,7 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu)
 {
 	struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
 
-	spin_lock(&fi->lock);
+	raw_spin_lock(&fi->lock);
 	/*
 	 * 2 cases:
 	 * a: an sccb answering interrupt was already pending or in flight.
@@ -534,7 +534,7 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu)
 	fi->srv_signal.ext_params |= 0x43000;
 	set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
 	clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs);
-	spin_unlock(&fi->lock);
+	raw_spin_unlock(&fi->lock);
 	return 0;
 }
 
arch/s390/kvm/interrupt.c +56 -56

375 lines not on this page · raw message

arch/s390/kvm/kvm-s390.c +2 -2
index 74f453f039a3..5695038119c8 100644
@@ -3263,7 +3263,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	}
 
 	mutex_init(&kvm->arch.float_int.ais_lock);
-	spin_lock_init(&kvm->arch.float_int.lock);
+	raw_spin_lock_init(&kvm->arch.float_int.lock);
 	for (i = 0; i < FIRQ_LIST_COUNT; i++)
 		INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
 	init_waitqueue_head(&kvm->arch.ipte_wq);
@@ -3734,7 +3734,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 	vcpu->arch.sie_block->msl = sclp.hamax;
 
 	vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
-	spin_lock_init(&vcpu->arch.local_int.lock);
+	raw_spin_lock_init(&vcpu->arch.local_int.lock);
 	vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
 	seqcount_init(&vcpu->arch.cputm_seqcount);
 
arch/s390/kvm/sigp.c +2 -2
index 55c34cb35428..911bb8e6ed69 100644
@@ -245,10 +245,10 @@ static int __prepare_sigp_re_start(struct kvm_vcpu *vcpu,
 	int rc = -EOPNOTSUPP;
 
 	/* make sure we don't race with STOP irq injection */
-	spin_lock(&li->lock);
+	raw_spin_lock(&li->lock);
 	if (kvm_s390_is_stop_irq_pending(dst_vcpu))
 		rc = SIGP_CC_BUSY;
-	spin_unlock(&li->lock);
+	raw_spin_unlock(&li->lock);
 
 	return rc;
 }
Signature · 1 line
2.52.0
#6 lore raw

PATCH v6 4/4 KVM: s390: Introducing kvm_arch_set_irq_inatomic fast inject

In reply to Douglas Freimuth

S390 needs a fast path for irq injection, and along those lines we introduce kvm_arch_set_irq_inatomic. Instead of placing all interrupts on the global work queue as it does today, this patch provides a fast path for irq injection.

The inatomic fast path cannot lose control since it is running with interrupts disabled. This meant making the following changes that exist on the slow path today. First, the adapter_indicators page needs to be mapped since it is accessed with interrupts disabled, so we added map/unmap functions. Second, access to shared resources between the fast and slow paths needed to be changed from mutex and semaphores to spin_lock's. Finally, the memory allocation on the slow path utilizes GFP_KERNEL_ACCOUNT but we had to implement the fast path with GFP_ATOMIC allocation. Each of these enhancements were required to prevent blocking on the fast inject path.

Fencing of Fast Inject in Secure Execution environments is enabled in the patch series by not mapping adapter indicator pages. In Secure Execution environments the path of execution available before this patch is followed.

Statistical counters have been added to enable analysis of irq injection on the fast path and slow path including io_390_inatomic, io_flic_inject_airq, io_set_adapter_int and io_390_inatomic_adapter_masked.

---
 arch/s390/include/asm/kvm_host.h |   6 +-
 arch/s390/kvm/interrupt.c        | 167 +++++++++++++++++++++++++++----
 arch/s390/kvm/kvm-s390.c         |  21 +++-
 arch/s390/kvm/kvm-s390.h         |   3 +-
 4 files changed, 173 insertions(+), 24 deletions(-)
arch/s390/include/asm/kvm_host.h +5 -1
index 295f0770bcb1..221d41f3902e 100644
@@ -359,7 +359,7 @@ struct kvm_s390_float_interrupt {
 	struct kvm_s390_mchk_info mchk;
 	struct kvm_s390_ext_info srv_signal;
 	int last_sleep_cpu;
-	struct mutex ais_lock;
+	raw_spinlock_t ais_lock;
 	u8 simm;
 	u8 nimm;
 };
@@ -450,6 +450,10 @@ struct kvm_vm_stat {
 	u64 inject_io;
 	u64 io_390_adapter_map;
 	u64 io_390_adapter_unmap;
+	u64 io_390_inatomic;
+	u64 io_flic_inject_airq;
+	u64 io_set_adapter_int;
+	u64 io_390_inatomic_adapter_masked;
 	u64 inject_float_mchk;
 	u64 inject_pfault_done;
 	u64 inject_service_signal;
arch/s390/kvm/interrupt.c +149 -18

281 lines not on this page · raw message

arch/s390/kvm/kvm-s390.c +17 -4
index 5695038119c8..6a4ecd3f3539 100644
@@ -70,6 +70,10 @@ const struct kvm_stats_desc kvm_vm_stats_desc[] = {
 	STATS_DESC_COUNTER(VM, inject_io),
 	STATS_DESC_COUNTER(VM, io_390_adapter_map),
 	STATS_DESC_COUNTER(VM, io_390_adapter_unmap),
+	STATS_DESC_COUNTER(VM, io_390_inatomic),
+	STATS_DESC_COUNTER(VM, io_flic_inject_airq),
+	STATS_DESC_COUNTER(VM, io_set_adapter_int),
+	STATS_DESC_COUNTER(VM, io_390_inatomic_adapter_masked),
 	STATS_DESC_COUNTER(VM, inject_float_mchk),
 	STATS_DESC_COUNTER(VM, inject_pfault_done),
 	STATS_DESC_COUNTER(VM, inject_service_signal),
@@ -2856,6 +2860,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 	void __user *argp = (void __user *)arg;
 	struct kvm_device_attr attr;
 	int r;
+	struct kvm_s390_interrupt_info *inti;
 
 	switch (ioctl) {
 	case KVM_S390_INTERRUPT: {
@@ -2864,7 +2869,10 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
 		r = -EFAULT;
 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
 			break;
-		r = kvm_s390_inject_vm(kvm, &s390int);
+		inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT);
+		if (!inti)
+			return -ENOMEM;
+		r = kvm_s390_inject_vm(kvm, &s390int, inti);
 		break;
 	}
 	case KVM_CREATE_IRQCHIP: {
@@ -3262,7 +3270,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 		mutex_unlock(&kvm->lock);
 	}
 
-	mutex_init(&kvm->arch.float_int.ais_lock);
+	raw_spin_lock_init(&kvm->arch.float_int.ais_lock);
 	raw_spin_lock_init(&kvm->arch.float_int.lock);
 	for (i = 0; i < FIRQ_LIST_COUNT; i++)
 		INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
@@ -4384,19 +4392,24 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo
 }
 
 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
-				      unsigned long token)
+				     unsigned long token)
 {
 	struct kvm_s390_interrupt inti;
 	struct kvm_s390_irq irq;
+	struct kvm_s390_interrupt_info *inti_mem = NULL;
 
 	if (start_token) {
 		irq.u.ext.ext_params2 = token;
 		irq.type = KVM_S390_INT_PFAULT_INIT;
 		WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
 	} else {
+		inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT);
+		if (WARN_ON_ONCE(!inti_mem))
+			return;
+
 		inti.type = KVM_S390_INT_PFAULT_DONE;
 		inti.parm64 = token;
-		WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
+		WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem));
 	}
 }
 
arch/s390/kvm/kvm-s390.h +2 -1
index 7ba885cb6bd1..6d2842fb71a3 100644
@@ -376,7 +376,8 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu);
 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu);
 void kvm_s390_clear_float_irqs(struct kvm *kvm);
 int __must_check kvm_s390_inject_vm(struct kvm *kvm,
-				    struct kvm_s390_interrupt *s390int);
+				    struct kvm_s390_interrupt *s390int,
+				    struct kvm_s390_interrupt_info *inti);
 int __must_check kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu,
 				      struct kvm_s390_irq *irq);
 static inline int kvm_s390_inject_prog_irq(struct kvm_vcpu *vcpu,
Signature · 1 line
2.52.0