KVM: x86: Split the APIC from the rest of IRQCHIP.

PATCH Superseded Steve Rutherford 1 participant 4 messages Jun 16, 2015

Subsystems (9)

Documentation x86 KVM PowerPC +5 more subsystems

lore

Share & tools

lore

Steve Rutherford

author

2015-06-16 02:28 UTC #0 lore raw

PATCH v4 1/4 KVM: x86: Split the APIC from the rest of IRQCHIP.

First patch in a series which enables the relocation of the PIC/IOAPIC to userspace.

Adds capability KVM_CAP_SPLIT_IRQCHIP;

KVM_CAP_SPLIT_IRQCHIP enables the construction of LAPICs without the rest of the irqchip.

Compile tested for x86.

Signed-off-by Steve Rutherford <[email protected]>

---
 Documentation/virtual/kvm/api.txt | 16 ++++++++++++++++
 arch/powerpc/kvm/irq.h            |  1 -
 arch/s390/kvm/irq.h               |  1 -
 arch/x86/include/asm/kvm_host.h   |  2 ++
 arch/x86/kvm/i8254.c              |  5 ++++-
 arch/x86/kvm/ioapic.h             |  9 +++++++++
 arch/x86/kvm/irq.h                |  6 ++++++
 arch/x86/kvm/irq_comm.c           |  9 ++++++++-
 arch/x86/kvm/lapic.c              |  9 ++++++---
 arch/x86/kvm/vmx.c                |  4 ++--
 arch/x86/kvm/x86.c                | 23 +++++++++++++++++++++--
 include/kvm/arm_vgic.h            |  1 +
 include/linux/kvm_host.h          |  1 +
 include/uapi/linux/kvm.h          |  1 +
 14 files changed, 77 insertions(+), 11 deletions(-)

Documentation/virtual/kvm/api.txt +16 -0

index 6955444..3143f9e 100644
@@ -2979,6 +2979,7 @@ len must be a multiple of sizeof(struct kvm_s390_irq). It must be > 0
 and it must not exceed (max_vcpus + 32) * sizeof(struct kvm_s390_irq),
 which is the maximum number of possibly pending cpu-local interrupts.
 
+
 5. The kvm_run structure
 ------------------------

@@ -3575,6 +3576,21 @@ struct {
 
 KVM handlers should exit to userspace with rc = -EREMOTE.
 
+7.5 KVM_SPLIT_IRQCHIP
+
+Architectures: x86
+Parameters: None
+Returns: 0 on success, -1 on error
+
+Create a local apic for each processor in the kernel. With this capability
+enabled, the userspace VMM is expected to emulate the IOAPIC and PIC.
+
+This supersedes KVM_CREATE_IRQCHIP, creating only local APICs, but no in kernel
+IOAPIC or PIC. This also enables in kernel routing of interrupt requests.
+
+Fails if VCPU has already been created, or if the irqchip is already in the
+kernel (i.e. KVM_CREATE_IRQCHIP has already been called).
+
 
 8. Other capabilities.
 ----------------------

arch/powerpc/kvm/irq.h +0 -1

index 5a9a10b..772fa8c 100644
@@ -16,5 +16,4 @@ static inline int irqchip_in_kernel(struct kvm *kvm)
 	smp_rmb();
 	return ret;
 }
-
 #endif

arch/s390/kvm/irq.h +0 -1

index d98e415..9a21a86 100644
@@ -18,5 +18,4 @@ static inline int irqchip_in_kernel(struct kvm *kvm)
 {
 	return 1;
 }
-
 #endif

arch/x86/include/asm/kvm_host.h +2 -0

index 7276107..af3225a 100644
@@ -639,6 +639,8 @@ struct kvm_arch {
 	bool boot_vcpu_runs_old_kvmclock;
 
 	u64 disabled_quirks;
+
+	bool irqchip_split;
 };
 
 struct kvm_vm_stat {

arch/x86/kvm/i8254.c +4 -1

index 4dce6f8..de06e7e 100644
@@ -35,6 +35,7 @@
 #include <linux/kvm_host.h>
 #include <linux/slab.h>
 
+#include "ioapic.h"
 #include "irq.h"
 #include "i8254.h"
 #include "x86.h"

@@ -333,7 +334,9 @@ static void create_pit_timer(struct kvm *kvm, u32 val, int is_period)
 	struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state;
 	s64 interval;
 
-	if (!irqchip_in_kernel(kvm) || ps->flags & KVM_PIT_FLAGS_HPET_LEGACY)
+	if (!irqchip_in_kernel(kvm) ||
+	    !ioapic_in_kernel(kvm) ||
+	    ps->flags & KVM_PIT_FLAGS_HPET_LEGACY)
 		return;
 
 	interval = muldiv64(val, NSEC_PER_SEC, KVM_PIT_FREQ);

arch/x86/kvm/ioapic.h +9 -0

index ca0b0b4..d8cc54b 100644
@@ -98,6 +98,15 @@ static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm)
 	return kvm->arch.vioapic;
 }
 
+static inline int ioapic_in_kernel(struct kvm *kvm)
+{
+	int ret;
+
+	ret = (ioapic_irqchip(kvm) != NULL);
+	smp_rmb();
+	return ret;
+}
+
 static inline bool kvm_ioapic_handles_vector(struct kvm *kvm, int vector)
 {
 	struct kvm_ioapic *ioapic = kvm->arch.vioapic;

arch/x86/kvm/irq.h +6 -0

index ad68c73..2f13dd5 100644
@@ -83,11 +83,17 @@ static inline struct kvm_pic *pic_irqchip(struct kvm *kvm)
 	return kvm->arch.vpic;
 }
 
+static inline int irqchip_split(struct kvm *kvm)
+{
+	return kvm->arch.irqchip_split;
+}
+
 static inline int irqchip_in_kernel(struct kvm *kvm)
 {
 	int ret;
 
 	ret = (pic_irqchip(kvm) != NULL);
+	ret |= irqchip_split(kvm);
 	smp_rmb();
 	return ret;
 }

arch/x86/kvm/irq_comm.c +8 -1

index 9efff9e..67f6b62 100644
@@ -208,7 +208,7 @@ void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id)
 		goto unlock;
 	}
 	clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap);
-	if (!irqchip_in_kernel(kvm))
+	if (!ioapic_in_kernel(kvm))
 		goto unlock;
 
 	kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id);

@@ -328,3 +328,10 @@ int kvm_setup_default_irq_routing(struct kvm *kvm)
 	return kvm_set_irq_routing(kvm, default_routing,
 				   ARRAY_SIZE(default_routing), 0);
 }
+
+static const struct kvm_irq_routing_entry empty_routing[] = {};
+
+int kvm_setup_empty_irq_routing(struct kvm *kvm)
+{
+	return kvm_set_irq_routing(kvm, empty_routing, 0, 0);
+}

arch/x86/kvm/lapic.c +6 -3

index c789e00..717b4e2 100644
@@ -209,7 +209,8 @@ out:
 	if (old)
 		kfree_rcu(old, rcu);
 
-	kvm_vcpu_request_scan_ioapic(kvm);
+	if (!irqchip_split(kvm))
+		kvm_vcpu_request_scan_ioapic(kvm);
 }
 
 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)

@@ -1827,7 +1828,8 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
 		kvm_x86_ops->hwapic_isr_update(vcpu->kvm,
 				apic_find_highest_isr(apic));
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
-	kvm_rtc_eoi_tracking_restore_one(vcpu);
+	if (!irqchip_split(vcpu->kvm))
+		kvm_rtc_eoi_tracking_restore_one(vcpu);
 }
 
 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)

@@ -1910,7 +1912,8 @@ static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
 	    /* Cache not set: could be safe but we don't bother. */
 	    apic->highest_isr_cache == -1 ||
 	    /* Need EOI to update ioapic. */
-	    kvm_ioapic_handles_vector(vcpu->kvm, apic->highest_isr_cache)) {
+	    kvm_ioapic_handles_vector(vcpu->kvm, apic->highest_isr_cache) ||
+	    irqchip_split(vcpu->kvm)) {
 		/*
 		 * PV EOI was disabled by apic_sync_pv_eoi_from_guest
 		 * so we need not do anything here.

arch/x86/kvm/vmx.c +2 -2

index 9cf5030..b4a239c 100644
@@ -948,7 +948,7 @@ static inline bool cpu_has_vmx_tpr_shadow(void)
 
 static inline bool vm_need_tpr_shadow(struct kvm *kvm)
 {
-	return (cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm));
+	return (cpu_has_vmx_tpr_shadow()) && irqchip_in_kernel(kvm);
 }
 
 static inline bool cpu_has_secondary_exec_ctrls(void)

@@ -9419,7 +9419,7 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
 	/* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
 	 * emulated by vmx_set_efer(), below.
 	 */
-	vm_entry_controls_init(vmx, 
+	vm_entry_controls_init(vmx,
 		(vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
 			~VM_ENTRY_IA32E_MODE) |
 		(vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));

arch/x86/kvm/x86.c +21 -2

index 79dde16..1cc6a65 100644
@@ -2866,6 +2866,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_TSC_DEADLINE_TIMER:
 	case KVM_CAP_ENABLE_CAP_VM:
 	case KVM_CAP_DISABLE_QUIRKS:
+	case KVM_CAP_SPLIT_IRQCHIP:
 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
 	case KVM_CAP_ASSIGN_DEV_IRQ:
 	case KVM_CAP_PCI_2_3:

@@ -3926,6 +3927,23 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
 		kvm->arch.disabled_quirks = cap->args[0];
 		r = 0;
 		break;
+	case KVM_CAP_SPLIT_IRQCHIP: {
+		mutex_lock(&kvm->lock);
+		r = -EEXIST;
+		if (irqchip_in_kernel(kvm))
+			goto split_irqchip_unlock;
+		r = -EINVAL;
+		if (atomic_read(&kvm->online_vcpus))
+			goto split_irqchip_unlock;
+		r = kvm_setup_empty_irq_routing(kvm);
+		if (r)
+			goto split_irqchip_unlock;
+		kvm->arch.irqchip_split = true;
+		r = 0;
+split_irqchip_unlock:
+		mutex_unlock(&kvm->lock);
+		break;
+	}
 	default:
 		r = -EINVAL;
 		break;

@@ -4044,7 +4062,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		}
 
 		r = -ENXIO;
-		if (!irqchip_in_kernel(kvm))
+		if (!irqchip_in_kernel(kvm) || !ioapic_in_kernel(kvm))
 			goto get_irqchip_out;
 		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
 		if (r)

@@ -4068,7 +4086,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		}
 
 		r = -ENXIO;
-		if (!irqchip_in_kernel(kvm))
+		if (!irqchip_in_kernel(kvm) || !ioapic_in_kernel(kvm))
 			goto set_irqchip_out;
 		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
 		if (r)

@@ -4194,6 +4212,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 		break;
 	}
+
 	default:
 		r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
 	}

include/kvm/arm_vgic.h +1 -0

index 133ea00..ffe1f4e 100644
@@ -329,6 +329,7 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
 int kvm_vgic_vcpu_active_irq(struct kvm_vcpu *vcpu);
 
 #define irqchip_in_kernel(k)	(!!((k)->arch.vgic.in_kernel))
+#define lapic_in_kernel(k)      (irqchip_in_kernel(k))
 #define vgic_initialized(k)	(!!((k)->arch.vgic.nr_cpus))
 #define vgic_ready(k)		((k)->arch.vgic.ready)

include/linux/kvm_host.h +1 -0

index a8bcbc9..7e2b41a 100644
@@ -935,6 +935,7 @@ static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq)
 #endif
 
 int kvm_setup_default_irq_routing(struct kvm *kvm);
+int kvm_setup_empty_irq_routing(struct kvm *kvm);
 int kvm_set_irq_routing(struct kvm *kvm,
 			const struct kvm_irq_routing_entry *entries,
 			unsigned nr,

include/uapi/linux/kvm.h +1 -0

index 75bd9f7..1e6f6c3 100644
@@ -815,6 +815,7 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_S390_IRQ_STATE 114
 #define KVM_CAP_PPC_HWRNG 115
 #define KVM_CAP_DISABLE_QUIRKS 116
+#define KVM_CAP_SPLIT_IRQCHIP 117
 
 #ifdef KVM_CAP_IRQ_ROUTING

Signature · 1 line

2.2.0.rc0.207.ga3a616c

Steve Rutherford

author

2015-06-16 02:28 UTC #1 lore raw

PATCH v4 2/4 KVM: x86: Add KVM exit for IOAPIC EOIs

Adds KVM_EXIT_IOAPIC_EOI which allows the kernel to EOI level-triggered IOAPIC interrupts.

Uses a per VCPU exit bitmap to decide whether or not the IOAPIC needs to be informed (which is identical to the EOI_EXIT_BITMAP field used by modern x86 processors, but can also be used to elide kvm IOAPIC EOI exits on older processors).

[Note: A prototype using ResampleFDs found that decoupling the EOI from the VCPU's thread made it possible for the VCPU to not see a recent EOI after reentering the guest. This does not match real hardware.]

Compile tested for Intel x86.

Signed-off-by Steve Rutherford <[email protected]>

---
 Documentation/virtual/kvm/api.txt | 13 +++++++++++++
 arch/x86/include/asm/kvm_host.h   |  3 +++
 arch/x86/kvm/lapic.c              |  9 +++++++++
 arch/x86/kvm/x86.c                | 11 +++++++++++
 include/linux/kvm_host.h          |  1 +
 include/uapi/linux/kvm.h          |  5 +++++
 6 files changed, 42 insertions(+)

Documentation/virtual/kvm/api.txt +13 -0

index 3143f9e..ec0cb19 100644
@@ -3258,6 +3258,18 @@ Valid values for 'type' are:
    As with SHUTDOWN, userspace can choose to ignore the request, or
    to schedule the reset to occur in the future and may call KVM_RUN again.
 
+	/* KVM_EXIT_IOAPIC_EOI */
+        struct {
+	       __u8 vector;
+        } eoi;
+
+Indicates that the VCPU's in-kernel local APIC received an EOI for a
+level-triggered IOAPIC interrupt.  This exit only triggers when the
+IOAPIC is implemented in userspace (i.e. KVM_CAP_SPLIT_IRQCHIP is enabled);
+the userspace IOAPIC should process the EOI and retrigger the interrupt if
+it is still asserted.  Vector is the LAPIC interrupt vector for which the
+EOI was received.
+
 		/* Fix the size of the union. */
 		char padding[256];
 	};

@@ -3271,6 +3283,7 @@ Valid values for 'type' are:
 	 */
 	__u64 kvm_valid_regs;
 	__u64 kvm_dirty_regs;
+
 	union {
 		struct kvm_sync_regs regs;
 		char padding[1024];

arch/x86/include/asm/kvm_host.h +3 -0

index af3225a..2778d36 100644
@@ -540,6 +540,9 @@ struct kvm_vcpu_arch {
 	struct {
 		bool pv_unhalted;
 	} pv;
+
+	u64 eoi_exit_bitmaps[4];
+	int pending_ioapic_eoi;
 };
 
 struct kvm_lpage_info {

arch/x86/kvm/lapic.c +9 -0

index 717b4e2..e2ae4b4 100644
@@ -869,6 +869,15 @@ int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
 
 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
 {
+	if (irqchip_split(apic->vcpu->kvm)) {
+		if (test_bit(vector,
+			     (void *) apic->vcpu->arch.eoi_exit_bitmaps)) {
+			apic->vcpu->arch.pending_ioapic_eoi = vector;
+			kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
+		}
+		return;
+	}
+
 	if (kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
 		int trigger_mode;
 		if (apic_test_vector(vector, apic->regs + APIC_TMR))

arch/x86/kvm/x86.c +11 -0

index 1cc6a65..3b05c01 100644
@@ -6392,6 +6392,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 			kvm_handle_pmu_event(vcpu);
 		if (kvm_check_request(KVM_REQ_PMI, vcpu))
 			kvm_deliver_pmi(vcpu);
+		if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
+			BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
+			if (test_bit(vcpu->arch.pending_ioapic_eoi,
+				     (void *) vcpu->arch.eoi_exit_bitmaps)) {
+				vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
+				vcpu->run->eoi.vector =
+						vcpu->arch.pending_ioapic_eoi;
+				r = 0;
+				goto out;
+			}
+		}
 		if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
 			vcpu_scan_ioapic(vcpu);
 		if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))

include/linux/kvm_host.h +1 -0

index 7e2b41a..c6df36f 100644
@@ -134,6 +134,7 @@ static inline bool is_error_page(struct page *page)
 #define KVM_REQ_ENABLE_IBS        23
 #define KVM_REQ_DISABLE_IBS       24
 #define KVM_REQ_APIC_PAGE_RELOAD  25
+#define KVM_REQ_IOAPIC_EOI_EXIT   26
 
 #define KVM_USERSPACE_IRQ_SOURCE_ID		0
 #define KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID	1

include/uapi/linux/kvm.h +5 -0

index 1e6f6c3..826a08d 100644
@@ -183,6 +183,7 @@ struct kvm_s390_skeys {
 #define KVM_EXIT_EPR              23
 #define KVM_EXIT_SYSTEM_EVENT     24
 #define KVM_EXIT_S390_STSI        25
+#define KVM_EXIT_IOAPIC_EOI       26
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */

@@ -329,6 +330,10 @@ struct kvm_run {
 			__u8 sel1;
 			__u16 sel2;
 		} s390_stsi;
+		/* KVM_EXIT_IOAPIC_EOI */
+		struct {
+			__u8 vector;
+		} eoi;
 		/* Fix the size of the union. */
 		char padding[256];
 	};

Signature · 1 line

2.2.0.rc0.207.ga3a616c

Steve Rutherford

author

2015-06-16 02:28 UTC #2 lore raw

PATCH v4 3/4 KVM: x86: Add EOI exit bitmap inference

In reply to Steve Rutherford

In order to support a userspace IOAPIC interacting with an in kernel APIC, the EOI exit bitmaps need to be configurable.

If the IOAPIC is in userspace (i.e. the irqchip has been split), the EOI exit bitmaps will be set whenever the GSI Routes are configured. In particular, for the low MSI routes are reservable for userspace IOAPICs. For these MSI routes, the EOI Exit bit corresponding to the destination vector of the route will be set for the destination VCPU.

The intention is for the userspace IOAPICs to use the reservable MSI routes to inject interrupts into the guest.

This is a slight abuse of the notion of an MSI Route, given that MSIs classically bypass the IOAPIC. It might be worthwhile to add an additional route type to improve clarity.

Compile tested for Intel x86.

Signed-off-by Steve Rutherford <[email protected]>

---
 Documentation/virtual/kvm/api.txt |  8 ++++----
 arch/x86/include/asm/kvm_host.h   |  1 +
 arch/x86/kvm/ioapic.h             |  2 ++
 arch/x86/kvm/irq_comm.c           | 42 +++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/lapic.c              |  3 +--
 arch/x86/kvm/x86.c                | 29 +++++++++++++++++----------
 include/linux/kvm_host.h          | 21 +++++++++++++++++++-
 virt/kvm/irqchip.c                | 13 ++----------
 8 files changed, 91 insertions(+), 28 deletions(-)

Documentation/virtual/kvm/api.txt +4 -4

index ec0cb19..e5e3e94 100644
@@ -3258,10 +3258,10 @@ Valid values for 'type' are:
    As with SHUTDOWN, userspace can choose to ignore the request, or
    to schedule the reset to occur in the future and may call KVM_RUN again.
 
-	/* KVM_EXIT_IOAPIC_EOI */
-        struct {
-	       __u8 vector;
-        } eoi;
+		/* KVM_EXIT_IOAPIC_EOI */
+		struct {
+			__u8 vector;
+		} eoi;
 
 Indicates that the VCPU's in-kernel local APIC received an EOI for a
 level-triggered IOAPIC interrupt.  This exit only triggers when the

arch/x86/include/asm/kvm_host.h +1 -0

index 2778d36..4f439ff 100644
@@ -644,6 +644,7 @@ struct kvm_arch {
 	u64 disabled_quirks;
 
 	bool irqchip_split;
+	u8 nr_reserved_ioapic_pins;
 };
 
 struct kvm_vm_stat {

arch/x86/kvm/ioapic.h +2 -0

index d8cc54b..f6ce112 100644
@@ -9,6 +9,7 @@ struct kvm;
 struct kvm_vcpu;
 
 #define IOAPIC_NUM_PINS  KVM_IOAPIC_NUM_PINS
+#define MAX_NR_RESERVED_IOAPIC_PINS 48
 #define IOAPIC_VERSION_ID 0x11	/* IOAPIC version */
 #define IOAPIC_EDGE_TRIG  0
 #define IOAPIC_LEVEL_TRIG 1

@@ -132,4 +133,5 @@ int kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
 void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap,
 			u32 *tmr);
 
+void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
 #endif

arch/x86/kvm/irq_comm.c +42 -0

45 lines not on this page · raw message

arch/x86/kvm/lapic.c +1 -2

index e2ae4b4..45be02b 100644
@@ -209,8 +209,7 @@ out:
 	if (old)
 		kfree_rcu(old, rcu);
 
-	if (!irqchip_split(kvm))
-		kvm_vcpu_request_scan_ioapic(kvm);
+	kvm_make_scan_ioapic_request(kvm);
 }
 
 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)

arch/x86/kvm/x86.c +19 ~~-10~~

index 3b05c01..a1a1c4a 100644
@@ -3933,12 +3933,17 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
 		if (irqchip_in_kernel(kvm))
 			goto split_irqchip_unlock;
 		r = -EINVAL;
-		if (atomic_read(&kvm->online_vcpus))
-			goto split_irqchip_unlock;
-		r = kvm_setup_empty_irq_routing(kvm);
-		if (r)
+		if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
 			goto split_irqchip_unlock;
-		kvm->arch.irqchip_split = true;
+		if (!irqchip_split(kvm)) {
+			if (atomic_read(&kvm->online_vcpus))
+				goto split_irqchip_unlock;
+			r = kvm_setup_empty_irq_routing(kvm);
+			if (r)
+				goto split_irqchip_unlock;
+			kvm->arch.irqchip_split = true;
+		}
+		kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
 		r = 0;
 split_irqchip_unlock:
 		mutex_unlock(&kvm->lock);

@@ -6280,18 +6285,22 @@ static void process_nmi(struct kvm_vcpu *vcpu)
 
 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
 {
-	u64 eoi_exit_bitmap[4];
+	struct kvm *kvm = vcpu->kvm;
 	u32 tmr[8];
 
 	if (!kvm_apic_hw_enabled(vcpu->arch.apic))
 		return;
 
-	memset(eoi_exit_bitmap, 0, 32);
+	memset(vcpu->arch.eoi_exit_bitmaps, 0, 32);
 	memset(tmr, 0, 32);
+	if (irqchip_split(kvm))
+		kvm_scan_ioapic_routes(vcpu, vcpu->arch.eoi_exit_bitmaps);
+	else
+		kvm_ioapic_scan_entry(vcpu, vcpu->arch.eoi_exit_bitmaps, tmr);
+	kvm_x86_ops->load_eoi_exitmap(vcpu, vcpu->arch.eoi_exit_bitmaps);
 
-	kvm_ioapic_scan_entry(vcpu, eoi_exit_bitmap, tmr);
-	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
-	kvm_apic_update_tmr(vcpu, tmr);
+	if (!irqchip_split(kvm))
+		kvm_apic_update_tmr(vcpu, tmr);
 }
 
 static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu)

include/linux/kvm_host.h +20 -1

index c6df36f..83d8d7a 100644
@@ -323,6 +323,17 @@ struct kvm_kernel_irq_routing_entry {
 	struct hlist_node link;
 };
 
+struct kvm_irq_routing_table {
+	int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS];
+	struct kvm_kernel_irq_routing_entry *rt_entries;
+	u32 nr_rt_entries;
+	/*
+	 * Array indexed by gsi. Each entry contains list of irq chips
+	 * the gsi is connected to.
+	 */
+	struct hlist_head map[0];
+};
+
 #ifndef KVM_PRIVATE_MEM_SLOTS
 #define KVM_PRIVATE_MEM_SLOTS 0
 #endif

@@ -438,10 +449,19 @@ void vcpu_put(struct kvm_vcpu *vcpu);
 
 #ifdef __KVM_HAVE_IOAPIC
 void kvm_vcpu_request_scan_ioapic(struct kvm *kvm);
+void kvm_arch_irq_routing_update(struct kvm *kvm);
+u8 kvm_arch_nr_userspace_ioapic_pins(struct kvm *kvm);
 #else
 static inline void kvm_vcpu_request_scan_ioapic(struct kvm *kvm)
 {
 }
+static inline void kvm_arch_irq_routing_update(struct kvm *kvm)
+{
+}
+static inline u8 kvm_arch_nr_userspace_ioapic_pins(struct kvm *kvm)
+{
+	return 0;
+}
 #endif
 
 #ifdef CONFIG_HAVE_KVM_IRQFD

@@ -1093,4 +1113,3 @@ static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val)
 }
 #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */
 #endif
-

virt/kvm/irqchip.c +2 ~~-11~~

index 1d56a90..8867e1d 100644
@@ -31,17 +31,6 @@
 #include <trace/events/kvm.h>
 #include "irq.h"
 
-struct kvm_irq_routing_table {
-	int chip[KVM_NR_IRQCHIPS][KVM_IRQCHIP_NUM_PINS];
-	struct kvm_kernel_irq_routing_entry *rt_entries;
-	u32 nr_rt_entries;
-	/*
-	 * Array indexed by gsi. Each entry contains list of irq chips
-	 * the gsi is connected to.
-	 */
-	struct hlist_head map[0];
-};
-
 int kvm_irq_map_gsi(struct kvm *kvm,
 		    struct kvm_kernel_irq_routing_entry *entries, int gsi)
 {

@@ -203,6 +192,8 @@ int kvm_set_irq_routing(struct kvm *kvm,
 	kvm_irq_routing_update(kvm);
 	mutex_unlock(&kvm->irq_lock);
 
+	kvm_arch_irq_routing_update(kvm);
+
 	synchronize_srcu_expedited(&kvm->irq_srcu);
 
 	new = old;

Signature · 1 line

2.2.0.rc0.207.ga3a616c

Steve Rutherford

author

2015-06-16 02:28 UTC #3 lore raw

PATCH v4 4/4 KVM: x86: Add support for local interrupt requests from userspace

In reply to Steve Rutherford

In order to enable userspace PIC support, the userspace PIC needs to be able to inject local interrupt requests.

This adds the ioctl KVM_REQUEST_PIC_INJECTION and kvm exit KVM_EXIT_GET_EXTINT.

The vm ioctl KVM_REQUEST_PIC_INJECTION makes a KVM_REQ_EVENT request on the BSP, which causes the BSP to exit to userspace to fetch the vector of the underlying external interrupt, which the BSP then injects into the guest. This matches the PIC spec, and is necessary to boot Windows.

Compiles for x86.

Update: Boots Windows and passes the KVM Unit Tests.

Signed-off-by Steve Rutherford <[email protected]>

---
 Documentation/virtual/kvm/api.txt | 21 +++++++++++++++
 arch/x86/include/asm/kvm_host.h   |  2 ++
 arch/x86/kvm/irq.c                | 22 +++++++++++++--
 arch/x86/kvm/lapic.c              |  7 +++++
 arch/x86/kvm/lapic.h              |  2 ++
 arch/x86/kvm/x86.c                | 57 ++++++++++++++++++++++++++++++++++++---
 include/uapi/linux/kvm.h          | 11 ++++++++
 7 files changed, 117 insertions(+), 5 deletions(-)

Documentation/virtual/kvm/api.txt +21 -0

index e5e3e94..b2e48a0 100644
@@ -2979,6 +2979,19 @@ len must be a multiple of sizeof(struct kvm_s390_irq). It must be > 0
 and it must not exceed (max_vcpus + 32) * sizeof(struct kvm_s390_irq),
 which is the maximum number of possibly pending cpu-local interrupts.
 
+4.96 KVM_REQUEST_PIC_INJECTION
+
+Capability: KVM_CAP_SPLIT_IRQCHIP
+Type: VM ioctl
+Parameters: struct kvm_pic_injection (in)
+Returns: 0 on success, -1 on error.
+
+Informs the kernel that userspace has a pending external interrupt for
+the specified cpu.
+
+struct kvm_pic_injection {
+       __u32 cpu;
+};
 
 5. The kvm_run structure
 ------------------------

@@ -3270,6 +3283,14 @@ the userspace IOAPIC should process the EOI and retrigger the interrupt if
 it is still asserted.  Vector is the LAPIC interrupt vector for which the
 EOI was received.
 
+		/* KVM_EXIT_GET_EXTINT */
+		struct {
+			__u8 vector;
+		} extint;
+Used when a VCPU needs to exit to userspace to fetch an external interrupt
+vector from a userspace PIC (which is necessary when KVM_CAP_SPLIT_IRQCHIP
+is enabled). The vector should be stored in the exit struct upon reentry.
+
 		/* Fix the size of the union. */
 		char padding[256];
 	};

arch/x86/include/asm/kvm_host.h +2 -0

index 4f439ff..0e8b0fc 100644
@@ -543,6 +543,8 @@ struct kvm_vcpu_arch {
 
 	u64 eoi_exit_bitmaps[4];
 	int pending_ioapic_eoi;
+	bool userspace_extint_available;
+	int pending_external_vector;
 };
 
 struct kvm_lpage_info {

arch/x86/kvm/irq.c +20 -2

index a1ec6a50..e9ecade 100644
@@ -38,12 +38,25 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 EXPORT_SYMBOL(kvm_cpu_has_pending_timer);
 
 /*
+ * check if there is a pending userspace external interrupt
+ */
+static int pending_userspace_extint(struct kvm_vcpu *v)
+{
+	return v->arch.userspace_extint_available ||
+	       v->arch.pending_external_vector != -1;
+}
+
+/*
  * check if there is pending interrupt from
  * non-APIC source without intack.
  */
 static int kvm_cpu_has_extint(struct kvm_vcpu *v)
 {
-	if (kvm_apic_accept_pic_intr(v))
+	u8 accept = kvm_apic_accept_pic_intr(v);
+
+	if (accept && irqchip_split(v->kvm))
+		return pending_userspace_extint(v);
+	else if (accept)
 		return pic_irqchip(v->kvm)->output;	/* PIC */
 	else
 		return 0;

@@ -91,7 +104,12 @@ EXPORT_SYMBOL_GPL(kvm_cpu_has_interrupt);
  */
 static int kvm_cpu_get_extint(struct kvm_vcpu *v)
 {
-	if (kvm_cpu_has_extint(v))
+	if (irqchip_split(v->kvm) && kvm_cpu_has_extint(v)) {
+		int vector = v->arch.pending_external_vector;
+
+		v->arch.pending_external_vector = -1;
+		return vector;
+	} else if (kvm_cpu_has_extint(v))
 		return kvm_pic_read_irq(v->kvm); /* PIC */
 	return -1;
 }

arch/x86/kvm/lapic.c +7 -0

index 45be02b..932ab94 100644
@@ -2094,3 +2094,10 @@ void kvm_lapic_init(void)
 	jump_label_rate_limit(&apic_hw_disabled, HZ);
 	jump_label_rate_limit(&apic_sw_disabled, HZ);
 }
+
+void kvm_request_pic_injection(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.userspace_extint_available = true;
+	kvm_make_request(KVM_REQ_EVENT, vcpu);
+	kvm_vcpu_kick(vcpu);
+}

arch/x86/kvm/lapic.h +2 -0

index 71b150c..7831e4d 100644
@@ -63,6 +63,8 @@ int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 		unsigned long *dest_map);
 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type);
 
+void kvm_request_pic_injection(struct kvm_vcpu *vcpu);
+
 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
 		struct kvm_lapic_irq *irq, int *r, unsigned long *dest_map);

arch/x86/kvm/x86.c +54 -3

104 lines not on this page · raw message

include/uapi/linux/kvm.h +11 -0

index 826a08d..9887cfd 100644
@@ -184,6 +184,7 @@ struct kvm_s390_skeys {
 #define KVM_EXIT_SYSTEM_EVENT     24
 #define KVM_EXIT_S390_STSI        25
 #define KVM_EXIT_IOAPIC_EOI       26
+#define KVM_EXIT_GET_EXTINT       27
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */

@@ -334,6 +335,10 @@ struct kvm_run {
 		struct {
 			__u8 vector;
 		} eoi;
+		/* KVM_EXIT_GET_EXTINT */
+		struct {
+			__u8 vector;
+		} extint;
 		/* Fix the size of the union. */
 		char padding[256];
 	};

@@ -1003,6 +1008,10 @@ struct kvm_device_attr {
 	__u64	addr;		/* userspace address of attr data */
 };
 
+struct kvm_pic_injection {
+	__u32 cpu;
+};
+
 #define  KVM_DEV_VFIO_GROUP			1
 #define   KVM_DEV_VFIO_GROUP_ADD			1
 #define   KVM_DEV_VFIO_GROUP_DEL			2

@@ -1206,6 +1215,8 @@ struct kvm_s390_ucas_mapping {
 /* Available with KVM_CAP_S390_IRQ_STATE */
 #define KVM_S390_SET_IRQ_STATE	  _IOW(KVMIO, 0xb5, struct kvm_s390_irq_state)
 #define KVM_S390_GET_IRQ_STATE	  _IOW(KVMIO, 0xb6, struct kvm_s390_irq_state)
+/* Available with KVM_CAP_SPLIT_IRQCHIP */
+#define KVM_REQUEST_PIC_INJECTION _IOW(KVMIO, 0xb7, struct kvm_pic_injection)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)

Signature · 1 line

2.2.0.rc0.207.ga3a616c