Skip to contentSkip to conversation

IRQ Routing

PATCH In review Avi Kivity 2 participants 4 messages Jan 13, 2009 – Jan 14, 2009

Subsystems (3)
lore
Share & tools
lore

Avi Kivity

author
#0 lore raw

cover IRQ Routing

Following is my alternative to irq routing. The differences compared to Sheng's version are:

- A single ioctl to replace the entire routing table, instead of add/remove ioctls for individual routing entries. Routing changes are rare, and we need to track the entire table in userspace anyway (for save/restore, and for user irqchip). As a side effect changes are atomic.
- Interrupt numbers are allocated by userspace, instead of the kernel
- I implemented irqchip routings rather then MSIs, it should be easy to add MSIs later on.

Please review and comment.

Avi Kivity (1):
KVM: Userspace controlled irq routing

 arch/ia64/kvm/kvm-ia64.c |    5 ++
 arch/x86/kvm/x86.c       |    6 ++
 include/linux/kvm.h      |   33 +++++++++
 include/linux/kvm_host.h |   31 +++++++++
 virt/kvm/irq_comm.c      |  168 ++++++++++++++++++++++++++++++++++++++++++++-
 virt/kvm/kvm_main.c      |   25 +++++++
 6 files changed, 264 insertions(+), 4 deletions(-)
author
#1 lore raw

PATCH 1/1 KVM: Userspace controlled irq routing

Currently KVM has a static routing from GSI numbers to interrupts (namely, 0-15 are mapped 1:1 to both PIC and IOAPIC, and 16:23 are mapped 1:1 to the IOAPIC). This is insufficient for several reasons:

- HPET requires non 1:1 mapping for the timer interrupt
- MSIs need a new method to assign interrupt numbers and dispatch them
- ACPI APIC mode needs to be able to reassign the PCI LINK interrupts to the ioapics

This patch implements an interrupt routing table (as a linked list, but this can be easily changed) and a userspace interface to replace the table. The routing table is initialized according to the current hardwired mapping.

---
 arch/ia64/kvm/kvm-ia64.c |    5 ++
 arch/x86/kvm/x86.c       |    6 ++
 include/linux/kvm.h      |   33 +++++++++
 include/linux/kvm_host.h |   31 +++++++++
 virt/kvm/irq_comm.c      |  168 ++++++++++++++++++++++++++++++++++++++++++++-
 virt/kvm/kvm_main.c      |   25 +++++++
 6 files changed, 264 insertions(+), 4 deletions(-)
arch/ia64/kvm/kvm-ia64.c +5 -0
index 1477f91..dbf527a 100644
@@ -919,6 +919,11 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		r = kvm_ioapic_init(kvm);
 		if (r)
 			goto out;
+		r = kvm_setup_default_irq_routing(kvm);
+		if (r) {
+			kfree(kvm->arch.vioapic);
+			goto out;
+		}
 		break;
 	case KVM_IRQ_LINE: {
 		struct kvm_irq_level irq_event;
arch/x86/kvm/x86.c +6 -0
index 4b17253..69628c9 100644
@@ -1833,6 +1833,12 @@ long kvm_arch_vm_ioctl(struct file *filp,
 			}
 		} else
 			goto out;
+		r = kvm_setup_default_irq_routing(kvm);
+		if (r) {
+			kfree(kvm->arch.vpic);
+			kfree(kvm->arch.vioapic);
+			goto out;
+		}
 		break;
 	case KVM_CREATE_PIT:
 		mutex_lock(&kvm->lock);
include/linux/kvm.h +33 -0
index 71c150f..b871417 100644
@@ -399,6 +399,38 @@ struct kvm_trace_rec {
 #if defined(CONFIG_X86)
 #define KVM_CAP_REINJECT_CONTROL 24
 #endif
+#if defined(CONFIG_X86)||defined(CONFIG_IA64)
+#define KVM_CAP_IRQ_ROUTING 25
+#endif
+
+#ifdef KVM_CAP_IRQ_ROUTING
+
+struct kvm_irq_routing_irqchip {
+	__u32 irqchip;
+	__u32 pin;
+};
+
+/* gsi routing entry types */
+#define KVM_IRQ_ROUTING_IRQCHIP 1
+
+struct kvm_irq_routing_entry {
+	__u32 gsi;
+	__u32 type;
+	__u32 flags;
+	__u32 pad;
+	union {
+		struct kvm_irq_routing_irqchip irqchip;
+		__u32 pad[8];
+	} u;
+};
+
+struct kvm_irq_routing {
+	__u32 nr;
+	__u32 flags;
+	struct kvm_irq_routing_entry entries[0];
+};
+
+#endif
 
 /*
  * ioctls for VM fds
@@ -430,6 +462,7 @@ struct kvm_trace_rec {
 			_IOW(KVMIO,  0x68, struct kvm_coalesced_mmio_zone)
 #define KVM_ASSIGN_PCI_DEVICE _IOR(KVMIO, 0x69, \
 				   struct kvm_assigned_pci_dev)
+#define KVM_SET_GSI_ROUTING       _IOW(KVMIO, 0x6a, struct kvm_irq_routing)
 #define KVM_ASSIGN_IRQ _IOR(KVMIO, 0x70, \
 			    struct kvm_assigned_irq)
 #define KVM_REINJECT_CONTROL      _IO(KVMIO, 0x71)
include/linux/kvm_host.h +31 -0
index 99963f3..ce285e0 100644
@@ -107,6 +107,19 @@ struct kvm_memory_slot {
 	int user_alloc;
 };
 
+struct kvm_kernel_irq_routing_entry {
+	u32 gsi;
+	void (*set)(struct kvm_kernel_irq_routing_entry *e,
+		    struct kvm *kvm, int level);
+	union {
+		struct {
+			unsigned irqchip;
+			unsigned pin;
+		} irqchip;
+	};
+	struct list_head link;
+};
+
 struct kvm {
 	struct mutex lock; /* protects the vcpus array and APIC accesses */
 	spinlock_t mmu_lock;
@@ -128,6 +141,7 @@ struct kvm {
 #endif
 
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
+	struct list_head irq_routing; /* of kvm_kernel_irq_routing_entry */
 	struct hlist_head mask_notifier_list;
 #endif
 
@@ -480,4 +494,21 @@ static inline int mmu_notifier_retry(struct kvm_vcpu *vcpu, unsigned long mmu_se
 }
 #endif
 
+#ifdef CONFIG_HAVE_KVM_IRQCHIP
+
+#define KVM_MAX_IRQ_ROUTES 1024
+
+int kvm_setup_default_irq_routing(struct kvm *kvm);
+int kvm_set_irq_routing(struct kvm *kvm,
+			const struct kvm_irq_routing_entry *entries,
+			unsigned nr,
+			unsigned flags);
+void kvm_free_irq_routing(struct kvm *kvm);
+
+#else
+
+static inline void kvm_free_irq_routing(struct kvm *kvm) {}
+
+#endif
+
 #endif
virt/kvm/irq_comm.c +164 -4

186 lines not on this page · raw message

virt/kvm/kvm_main.c +25 -0
index 103bc08..7ab7e67 100644
@@ -840,6 +840,7 @@ static struct kvm *kvm_create_vm(void)
 	if (IS_ERR(kvm))
 		goto out;
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
+	INIT_LIST_HEAD(&kvm->irq_routing);
 	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
 #endif
 
@@ -923,6 +924,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 	spin_lock(&kvm_lock);
 	list_del(&kvm->vm_list);
 	spin_unlock(&kvm_lock);
+	kvm_free_irq_routing(kvm);
 	kvm_io_bus_destroy(&kvm->pio_bus);
 	kvm_io_bus_destroy(&kvm->mmio_bus);
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
@@ -1943,6 +1945,29 @@ static long kvm_vm_ioctl(struct file *filp,
 		break;
 	}
 #endif
+#ifdef KVM_CAP_IRQ_ROUTING
+	case KVM_SET_GSI_ROUTING: {
+		struct kvm_irq_routing routing;
+		struct kvm_irq_routing_entry *entries;
+
+		r = -EFAULT;
+		if (copy_from_user(&routing, argp, sizeof(routing)))
+			goto out;
+		r = -EINVAL;
+		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
+			goto out;
+		if (routing.flags)
+			goto out;
+		r = -ENOMEM;
+		entries = vmalloc(routing.nr * sizeof(*entries));
+		if (!entries)
+			goto out;
+		r = kvm_set_irq_routing(kvm, entries, routing.nr,
+					routing.flags);
+		vfree(entries);
+		break;
+	}
+#endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
 	}
Signature · 1 line
1.6.0.6
#2 lore raw
In reply to Avi Kivity
Quoted 11 lines · On Wednesday 14 January 2009 04:02:29 Avi Kivity wrote:

Following is my alternative to irq routing. The differences compared to Sheng's version are:

- A single ioctl to replace the entire routing table, instead of add/remove ioctls for individual routing entries. Routing changes are rare, and we need to track the entire table in userspace anyway (for save/restore, and for user irqchip). As a side effect changes are atomic.
- Interrupt numbers are allocated by userspace, instead of the kernel
- I implemented irqchip routings rather then MSIs, it should be easy to add MSIs later on.

Quoted 1 line

Please review and comment.

Look nice to me now... Save/restore is a good reason to maintain a table in userspace.

And looking forward to userspace patch. Put a table in kvm_context?

Signature · 2 lines
regards
Yang, Sheng
Quoted 12 lines

Avi Kivity (1):
KVM: Userspace controlled irq routing

 arch/ia64/kvm/kvm-ia64.c |    5 ++
 arch/x86/kvm/x86.c       |    6 ++
 include/linux/kvm.h      |   33 +++++++++
 include/linux/kvm_host.h |   31 +++++++++
 virt/kvm/irq_comm.c      |  168
++++++++++++++++++++++++++++++++++++++++++++- virt/kvm/kvm_main.c      |  
25 +++++++
 6 files changed, 264 insertions(+), 4 deletions(-)
author
#3 lore raw
Quoted 13 lines · Sheng Yang
Quoted 11 lines · On Wednesday 14 January 2009 04:02:29 Avi Kivity wrote:

Following is my alternative to irq routing. The differences compared to Sheng's version are:

- A single ioctl to replace the entire routing table, instead of add/remove ioctls for individual routing entries. Routing changes are rare, and we need to track the entire table in userspace anyway (for save/restore, and for user irqchip). As a side effect changes are atomic.
- Interrupt numbers are allocated by userspace, instead of the kernel
- I implemented irqchip routings rather then MSIs, it should be easy to add MSIs later on.

Quoted 7 lines
Quoted 2 lines

Please review and comment.

Look nice to me now... Save/restore is a good reason to maintain a table in userspace.

And looking forward to userspace patch. Put a table in kvm_context?

Sent out. I indeed placed a table in kvm_context, with add/del route APIs.

Signature · 1 line
error compiling committee.c: too many arguments to function