GSI->MSG route layer for MSI/MSI-X
v3 cover GSI->MSG route layer for MSI/MSI-X
Update from v2:
Add gsi_msg_pending_bitmap, in order to support MSI-X mutiply interrupts.
And this one depends on the former "Device assignment code clean up and MSI disable support" patchset.
--
regards
Yang, Sheng
PATCH v3 1/10 KVM: Add a route layer to convert MSI message to GSI
Avi's purpose, to use single kvm_set_irq() to deal with all interrupt, including MSI. So here is it.
struct gsi_msg is a mapping from a special gsi(with KVM_GSI_MSG_MASK) to MSI/MSI-X message address/data.
Now we support up to 256 gsi_msg mapping, and gsi_msg is allocated by kernel and provide two ioctls to userspace, which is more flexiable.
--- include/linux/kvm.h | 14 +++++++++ include/linux/kvm_host.h | 16 ++++++++++ virt/kvm/irq_comm.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++ virt/kvm/kvm_main.c | 66 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 0 deletions(-)
include/linux/kvm.h +14 -0
@@ -396,6 +396,9 @@ struct kvm_trace_rec {#if defined(CONFIG_X86 )#define KVM_CAP_SET_GUEST_DEBUG 23#endif +#if defined(CONFIG_X86 ) +#define KVM_CAP_GSI_MSG 24 +#endif /* * ioctls for VM fds
@@ -429,6 +432,8 @@ struct kvm_trace_rec { struct kvm_assigned_pci_dev)#define KVM_ASSIGN_IRQ _IOR(KVMIO, 0x70, \ struct kvm_assigned_irq) +#define KVM_REQUEST_GSI_MSG _IOWR(KVMIO, 0x71, struct kvm_assigned_gsi_msg) +#define KVM_FREE_GSI_MSG _IOR(KVMIO, 0x72, struct kvm_assigned_gsi_msg)/* * ioctls for vcpu fds
@@ -549,4 +554,13 @@ struct kvm_assigned_irq {#define KVM_DEV_IRQ_ASSIGN_MSI_ACTION (1 << 0)#define KVM_DEV_IRQ_ASSIGN_ENABLE_MSI (1 << 1) +struct kvm_assigned_gsi_msg { +__u32 gsi; +struct { +__u32 addr_lo; +__u32 addr_hi; +__u32 data; + } msg; +}; +#endif
include/linux/kvm_host.h +16 -0
@@ -132,6 +132,10 @@ struct kvm {unsigned long mmu_notifier_seq;long mmu_notifier_count;#endif +struct hlist_head gsi_msg_list; +struct mutex gsi_msg_lock; +#define KVM_NR_GSI_MSG 256 +DECLARE_BITMAP (gsi_msg_bitmap,KVM_NR_GSI_MSG ); };/* The guest did something we don't support. */
@@ -319,6 +323,14 @@ struct kvm_assigned_dev_kernel {struct pci_dev *dev;struct kvm *kvm; }; + +#define KVM_GSI_MSG_MASK 0x1000000ull +struct kvm_gsi_msg { +u32 gsi; +struct msi_msg msg; +struct hlist_node link; +}; +void kvm_set_irq (struct kvm *kvm,int irq_source_id,int irq,int level);void kvm_notify_acked_irq (struct kvm *kvm,unsigned gsi); void kvm_register_irq_ack_notifier(struct kvm *kvm,
@@ -326,6 +338,10 @@ void kvm_register_irq_ack_notifier(struct kvm *kvm,void kvm_unregister_irq_ack_notifier (struct kvm_irq_ack_notifier *kian);int kvm_request_irq_source_id (struct kvm *kvm);void kvm_free_irq_source_id (struct kvm *kvm,int irq_source_id); +int kvm_update_gsi_msg (struct kvm *kvm,struct kvm_gsi_msg *gsi_msg); +struct kvm_gsi_msg *kvm_find_gsi_msg (struct kvm *kvm,u32 gsi); +void kvm_free_gsi_msg (struct kvm *kvm,struct kvm_gsi_msg *gsi_msg); +void kvm_free_gsi_msg_list (struct kvm *kvm);#ifdef CONFIG_DMAR int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
virt/kvm/irq_comm.c +70 -0
73 lines not on this page · raw message
virt/kvm/kvm_main.c +66 -0
90 lines not on this page · raw message
Signature · 1 line
1.5.4.5
Re: [PATCH 01/10] KVM: Add a route layer to convert MSI message to GSI
Quoted 12 lines · Sheng Yang
Avi's purpose, to use single
kvm_set_irq()to deal with all interrupt, including MSI. So here is it.struct
gsi_msgis a mapping from a specialgsi(with KVM_GSI_MSG_MASK)to MSI/MSI-X message address/data.Now we support up to 256
gsi_msgmapping, andgsi_msgis allocated by kernel and provide two ioctls to userspace, which is more flexiable.
include/linux/kvm.h struct kvm_trace_rec { · line 436
Quoted 2 lines
include/linux/kvm.h +2
-0+#define KVM_REQUEST_GSI_MSG _IOWR(KVMIO, 0x71, struct kvm_assigned_gsi_msg) +#define KVM_FREE_GSI_MSG _IOR(KVMIO, 0x72, struct kvm_assigned_gsi_msg)
We will also need a pair for PIC and a pair for IOAPIC routing.
How about a single ioctl to set the entire routing table? It would take an array of structures:
struct {
__u32 gsi;
__u32 type;
__u32 flags;
__u32 reserved;
union {
struct {
__u32 irq;
} pic;
struct {
__u32 ioapic; // can have >1 ioapic
__u32 inti;
} ioapic;
struct {
...
} msi;
__u32 reserved[8]; };
};
This way we can solve the HPET irq0/inti2 mess, and also have a simpler way of setting MSI. All the mess in one ioctl.
Quoted 7 lines
virt/kvm/irq_comm.c +6
-0+ +/* Call with kvm->gsi_msg_lock hold */ +struct kvm_gsi_msg *kvm_find_gsi_msg (struct kvm *kvm,u32 gsi) +{ +struct kvm_gsi_msg *gsi_msg; +struct hlist_node *n;
virt/kvm/irq_comm.c void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) · line 149
Quoted 10 lines
virt/kvm/irq_comm.c +10
-0+ +if (!(gsi &KVM_GSI_MSG_MASK )) +return NULL ; +hlist_for_each_entry (gsi_msg, n, &kvm->gsi_msg_list, link) +if (gsi_msg->gsi == gsi) + goto out; + gsi_msg =NULL ; +out: +return gsi_msg; +}
Linear search is a bit sad but fine for now. Later we can add an array indexed by gsi.
Signature · 1 line
error compiling committee.c: too many arguments to function
PATCH v3 2/10 KVM: Using gsi_msg mapping for MSI device assignment
In reply to Sheng YangConvert MSI userspace interface to support gsi_msg mapping(and nobody should be the user of the old interface...).
--- include/linux/kvm_host.h | 1 - virt/kvm/kvm_main.c | 35 ++++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 14 deletions(-)
include/linux/kvm_host.h +0 -1
@@ -313,7 +313,6 @@ struct kvm_assigned_dev_kernel {int host_irq;bool host_irq_disabled;int guest_irq; -struct msi_msg guest_msi;#define KVM_ASSIGNED_DEV_GUEST_INTX (1 << 0)#define KVM_ASSIGNED_DEV_GUEST_MSI (1 << 1)#define KVM_ASSIGNED_DEV_HOST_INTX (1 << 8)
virt/kvm/kvm_main.c +22 -13
@@ -92,20 +92,30 @@ static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev)int vcpu_id;struct kvm_vcpu *vcpu;struct kvm_ioapic *ioapic =ioapic_irqchip (dev->kvm); -int dest_id = (dev->guest_msi.address_lo &MSI_ADDR_DEST_ID_MASK ) - >>MSI_ADDR_DEST_ID_SHIFT ; -int vector = (dev->guest_msi.data &MSI_DATA_VECTOR_MASK ) - >>MSI_DATA_VECTOR_SHIFT ; -int dest_mode =test_bit (MSI_ADDR_DEST_MODE_SHIFT , - (unsigned long *)&dev->guest_msi.address_lo); -int trig_mode =test_bit (MSI_DATA_TRIGGER_SHIFT , - (unsigned long *)&dev->guest_msi.data); -int delivery_mode =test_bit (MSI_DATA_DELIVERY_MODE_SHIFT , - (unsigned long *)&dev->guest_msi.data); +struct kvm_gsi_msg *gsi_msg; +int dest_id, vector, dest_mode, trig_mode, delivery_mode;u32 deliver_bitmask;BUG_ON (!ioapic); +mutex_lock (&dev->kvm->gsi_msg_lock); + gsi_msg =kvm_find_gsi_msg (dev->kvm, dev->guest_irq); +if (!gsi_msg) { +printk (KERN_WARNING "kvm: fail to find correlated gsi_msg\n" ); +return ; + } +mutex_unlock (&dev->kvm->gsi_msg_lock); + + dest_id = (gsi_msg->msg.address_lo &MSI_ADDR_DEST_ID_MASK ) + >>MSI_ADDR_DEST_ID_SHIFT ; + vector = (gsi_msg->msg.data &MSI_DATA_VECTOR_MASK ) + >>MSI_DATA_VECTOR_SHIFT ; + dest_mode =test_bit (MSI_ADDR_DEST_MODE_SHIFT , + (unsigned long *)&gsi_msg->msg.address_lo); + trig_mode =test_bit (MSI_DATA_TRIGGER_SHIFT , + (unsigned long *)&gsi_msg->msg.data); + delivery_mode =test_bit (MSI_DATA_DELIVERY_MODE_SHIFT , + (unsigned long *)&gsi_msg->msg.data); deliver_bitmask =kvm_ioapic_get_delivery_bitmask (ioapic, dest_id, dest_mode);/* IOAPIC delivery mode value is the same as MSI here */
@@ -316,17 +326,16 @@ static int assigned_device_update_msi(struct kvm *kvm, {int r; + adev->guest_irq = airq->guest_irq; +if (airq->flags &KVM_DEV_IRQ_ASSIGN_ENABLE_MSI ) {/* x86 don't care upper address of guest msi message addr */ adev->irq_requested_type |=KVM_ASSIGNED_DEV_GUEST_MSI ; adev->irq_requested_type &= ~KVM_ASSIGNED_DEV_GUEST_INTX ; - adev->guest_msi.address_lo = airq->guest_msi.addr_lo; - adev->guest_msi.data = airq->guest_msi.data; adev->ack_notifier.gsi =-1 ; }else if (msi2intx) { adev->irq_requested_type |=KVM_ASSIGNED_DEV_GUEST_INTX ; adev->irq_requested_type &= ~KVM_ASSIGNED_DEV_GUEST_MSI ; - adev->guest_irq = airq->guest_irq; adev->ack_notifier.gsi = airq->guest_irq; }else {/*
Signature · 1 line
1.5.4.5
PATCH v3 3/10 KVM: Improve MSI dispatch function
In reply to Sheng YangPrepare to merge with kvm_set_irq().
--- virt/kvm/kvm_main.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-)
virt/kvm/kvm_main.c +4 -4
@@ -87,7 +87,7 @@ static bool kvm_rebooting; #ifdef KVM_CAP_DEVICE_ASSIGNMENT #ifdef CONFIG_X86 -static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev) +static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev, u32 gsi) { int vcpu_id; struct kvm_vcpu *vcpu;
@@ -99,7 +99,7 @@ static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev)BUG_ON (!ioapic);mutex_lock (&dev->kvm->gsi_msg_lock); - gsi_msg =kvm_find_gsi_msg (dev->kvm, dev->guest_irq); + gsi_msg =kvm_find_gsi_msg (dev->kvm, gsi);if (!gsi_msg) {printk (KERN_WARNING "kvm: fail to find correlated gsi_msg\n" );return ;
@@ -143,7 +143,7 @@ static void assigned_device_msi_dispatch(struct kvm_assigned_dev_kernel *dev) } }#else -static void assigned_device_msi_dispatch (struct kvm_assigned_dev_kernel *dev) {} +static void assigned_device_msi_dispatch (struct kvm_assigned_dev_kernel *dev,u32 gsi) {}#endif static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
@@ -178,7 +178,7 @@ static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work) assigned_dev->guest_irq, 1); else if (assigned_dev->irq_requested_type &KVM_ASSIGNED_DEV_GUEST_MSI ) { -assigned_device_msi_dispatch (assigned_dev); +assigned_device_msi_dispatch (assigned_dev, assigned_dev->guest_irq);enable_irq (assigned_dev->host_irq); assigned_dev->host_irq_disabled = false; }
Signature · 1 line
1.5.4.5
PATCH v3 4/10 KVM: Using ioapic_irqchip() macro for kvm_set_irq
In reply to Sheng Yang--- virt/kvm/irq_comm.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
virt/kvm/irq_comm.c +1 -1
@@ -39,7 +39,7 @@ void kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level)* IOAPIC. So set the bit in both. The guest will ignore * writes to the unused one. */ -kvm_ioapic_set_irq (kvm->arch.vioapic, irq, !!(*irq_state)); +kvm_ioapic_set_irq (ioapic_irqchip (kvm), irq, !!(*irq_state));#ifdef CONFIG_X86 kvm_pic_set_irq (pic_irqchip (kvm), irq, !!(*irq_state));#endif
Signature · 1 line
1.5.4.5
PATCH v3 5/10 KVM: Merge MSI handling to kvm_set_irq
In reply to Sheng YangUsing kvm_set_irq to handle all interrupt injection.
--- include/linux/kvm_host.h | 2 +- virt/kvm/irq_comm.c | 98 +++++++++++++++++++++++++++++++++++++++------- virt/kvm/kvm_main.c | 77 +++--------------------------------- 3 files changed, 90 insertions(+), 87 deletions(-)
include/linux/kvm_host.h +1 -1
@@ -330,7 +330,7 @@ struct kvm_gsi_msg {struct hlist_node link; }; -void kvm_set_irq (struct kvm *kvm,int irq_source_id,int irq,int level); +void kvm_set_irq (struct kvm *kvm,int irq_source_id,u32 gsi,int level);void kvm_notify_acked_irq (struct kvm *kvm,unsigned gsi);void kvm_register_irq_ack_notifier (struct kvm *kvm,struct kvm_irq_ack_notifier *kian);
virt/kvm/irq_comm.c +83 -15
111 lines not on this page · raw message
virt/kvm/kvm_main.c +6 -71
98 lines not on this page · raw message
Signature · 1 line
1.5.4.5
Re: [PATCH 05/10] KVM: Merge MSI handling to kvm_set_irq
Quoted 48 lines · Sheng Yang
Using
kvm_set_irqto handle all interrupt injection./* This should be called with the
kvm->lockmutex held */virt/kvm/irq_comm.c +28
-14-void kvm_set_irq (struct kvm *kvm,int irq_source_id,int irq,int level) +void kvm_set_irq (struct kvm *kvm,int irq_source_id,u32 gsi,int level) { -unsigned long *irq_state = (unsigned long *)&kvm->arch.irq_states[irq]; - -/* Logical OR for level trig interrupt */ -if (level) -set_bit (irq_source_id, irq_state); -else -clear_bit (irq_source_id, irq_state); - -/* Not possible to detect if the guest uses the PIC or the -* IOAPIC. So set the bit in both. The guest will ignore -* writes to the unused one. -*/ -kvm_ioapic_set_irq (ioapic_irqchip (kvm), irq, !!(*irq_state)); +unsigned long *irq_state; +#ifdef CONFIG_X86 +int vcpu_id; +struct kvm_vcpu *vcpu; +struct kvm_ioapic *ioapic =ioapic_irqchip (kvm); +struct kvm_gsi_msg *gsi_msg; +int dest_id, vector, dest_mode, trig_mode, delivery_mode; +u32 deliver_bitmask; + +BUG_ON (!ioapic); +#endif + +if (!(gsi &KVM_GSI_MSG_MASK )) { +int irq = gsi; + + irq_state = (unsigned long *)&kvm->arch.irq_states[irq]; + +/* Logical OR for level trig interrupt */ +if (level) +set_bit (irq_source_id, irq_state); +else +clear_bit (irq_source_id, irq_state); + +/* Not possible to detect if the guest uses the PIC or the +* IOAPIC. So set the bit in both. The guest will ignore +* writes to the unused one. +*/
virt/kvm/irq_comm.c line 70
Quoted 10 lines
virt/kvm/irq_comm.c +8
-1+kvm_ioapic_set_irq (ioapic, irq, !!(*irq_state));#ifdef CONFIG_X86 -kvm_pic_set_irq (pic_irqchip (kvm), irq, !!(*irq_state)); +kvm_pic_set_irq (pic_irqchip (kvm), irq, !!(*irq_state)); +#endif +return ; + } + +#ifdef CONFIG_X86 +mutex_lock (&kvm->gsi_msg_lock);
The lock is already taken here?
Quoted 36 lines
virt/kvm/irq_comm.c +34
-0+ gsi_msg =kvm_find_gsi_msg (kvm, gsi); +mutex_unlock (&kvm->gsi_msg_lock); +if (!gsi_msg) { +printk (KERN_WARNING "kvm: fail to find correlated gsi_msg\n" ); +return ; + } + + dest_id = (gsi_msg->msg.address_lo &MSI_ADDR_DEST_ID_MASK ) + >>MSI_ADDR_DEST_ID_SHIFT ; + vector = (gsi_msg->msg.data &MSI_DATA_VECTOR_MASK ) + >>MSI_DATA_VECTOR_SHIFT ; + dest_mode =test_bit (MSI_ADDR_DEST_MODE_SHIFT , + (unsigned long *)&gsi_msg->msg.address_lo); + trig_mode =test_bit (MSI_DATA_TRIGGER_SHIFT , + (unsigned long *)&gsi_msg->msg.data); + delivery_mode =test_bit (MSI_DATA_DELIVERY_MODE_SHIFT , + (unsigned long *)&gsi_msg->msg.data); + deliver_bitmask =kvm_ioapic_get_delivery_bitmask (ioapic, + dest_id, dest_mode); +/* IOAPIC delivery mode value is the same as MSI here */ +switch (delivery_mode) { +case IOAPIC_LOWEST_PRIORITY : + vcpu =kvm_get_lowest_prio_vcpu (ioapic->kvm, vector, + deliver_bitmask); +if (vcpu !=NULL ) +kvm_apic_set_irq (vcpu, vector, trig_mode); +else +printk (KERN_INFO "kvm: null lowest priority vcpu!\n" ); +break ; +case IOAPIC_FIXED : +for (vcpu_id =0 ; deliver_bitmask !=0 ; vcpu_id++) { +if (!(deliver_bitmask & (1 << vcpu_id))) +continue ; + deliver_bitmask &= ~(1 << vcpu_id);
virt/kvm/irq_comm.c line 114
Quoted 10 lines
virt/kvm/irq_comm.c +8
-0+ vcpu = ioapic->kvm->vcpus[vcpu_id]; +if (vcpu) + kvm_apic_set_irq(vcpu, vector, trig_mode); + } +break ; +default : +printk (KERN_INFO "kvm: unsupported MSI delivery mode\n" ); + }#endif }
This looks very messy. Would be better to have the in-kernel irq structure contain a (*set_level)() callback that can take the appropriate action.
Also, the CONFIG_X86 worries me, can we have IA64 enable this as well?
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH 05/10] KVM: Merge MSI handling to kvm_set_irq
Quoted 52 lines · On Tuesday 30 December 2008 18:48:42 Avi Kivity wrote:
Quoted 51 lines · Sheng Yang
Using
kvm_set_irqto handle all interrupt injection./* This should be called with the
kvm->lockmutex held */virt/kvm/irq_comm.c +30
-15-void kvm_set_irq (struct kvm *kvm,int irq_source_id,int irq,int level) +void kvm_set_irq (struct kvm *kvm,int irq_source_id,u32 gsi,int level) { -unsigned long *irq_state = (unsigned long *)&kvm->arch.irq_states[irq]; - -/* Logical OR for level trig interrupt */ -if (level) -set_bit (irq_source_id, irq_state); -else -clear_bit (irq_source_id, irq_state); - -/* Not possible to detect if the guest uses the PIC or the -* IOAPIC. So set the bit in both. The guest will ignore -* writes to the unused one. -*/ -kvm_ioapic_set_irq (ioapic_irqchip (kvm), irq, !!(*irq_state)); +unsigned long *irq_state; +#ifdef CONFIG_X86 +int vcpu_id; +struct kvm_vcpu *vcpu; +struct kvm_ioapic *ioapic =ioapic_irqchip (kvm); +struct kvm_gsi_msg *gsi_msg; +int dest_id, vector, dest_mode, trig_mode, delivery_mode; +u32 deliver_bitmask; + +BUG_ON (!ioapic); +#endif + +if (!(gsi &KVM_GSI_MSG_MASK )) { +int irq = gsi; + + irq_state = (unsigned long *)&kvm->arch.irq_states[irq]; + +/* Logical OR for level trig interrupt */ +if (level) +set_bit (irq_source_id, irq_state); +else +clear_bit (irq_source_id, irq_state); + +/* Not possible to detect if the guest uses the PIC or the +* IOAPIC. So set the bit in both. The guest will ignore +* writes to the unused one. +*/ +kvm_ioapic_set_irq (ioapic, irq, !!(*irq_state));#ifdef CONFIG_X86 -kvm_pic_set_irq (pic_irqchip (kvm), irq, !!(*irq_state)); +kvm_pic_set_irq (pic_irqchip (kvm), irq, !!(*irq_state));
Quoted 8 lines
virt/kvm/irq_comm.c line 70
Quoted 6 lines
virt/kvm/irq_comm.c +6
-0+#endif +return ; + } + +#ifdef CONFIG_X86 +mutex_lock (&kvm->gsi_msg_lock);The lock is already taken here?
Um? For gsi_msg_lock?
Quoted 41 lines
Quoted 41 lines
virt/kvm/irq_comm.c +40
-0+ gsi_msg =kvm_find_gsi_msg (kvm, gsi); +mutex_unlock (&kvm->gsi_msg_lock); +if (!gsi_msg) { +printk (KERN_WARNING "kvm: fail to find correlated gsi_msg\n" ); +return ; + } + + dest_id = (gsi_msg->msg.address_lo &MSI_ADDR_DEST_ID_MASK ) + >>MSI_ADDR_DEST_ID_SHIFT ; + vector = (gsi_msg->msg.data &MSI_DATA_VECTOR_MASK ) + >>MSI_DATA_VECTOR_SHIFT ; + dest_mode =test_bit (MSI_ADDR_DEST_MODE_SHIFT , + (unsigned long *)&gsi_msg->msg.address_lo); + trig_mode =test_bit (MSI_DATA_TRIGGER_SHIFT , + (unsigned long *)&gsi_msg->msg.data); + delivery_mode =test_bit (MSI_DATA_DELIVERY_MODE_SHIFT , + (unsigned long *)&gsi_msg->msg.data); + deliver_bitmask =kvm_ioapic_get_delivery_bitmask (ioapic, + dest_id, dest_mode); +/* IOAPIC delivery mode value is the same as MSI here */ +switch (delivery_mode) { +case IOAPIC_LOWEST_PRIORITY : + vcpu =kvm_get_lowest_prio_vcpu (ioapic->kvm, vector, + deliver_bitmask); +if (vcpu !=NULL ) +kvm_apic_set_irq (vcpu, vector, trig_mode); +else +printk (KERN_INFO "kvm: null lowest priority vcpu!\n" ); +break ; +case IOAPIC_FIXED : +for (vcpu_id =0 ; deliver_bitmask !=0 ; vcpu_id++) { +if (!(deliver_bitmask & (1 << vcpu_id))) +continue ; + deliver_bitmask &= ~(1 << vcpu_id); + vcpu = ioapic->kvm->vcpus[vcpu_id]; +if (vcpu) +kvm_apic_set_irq (vcpu, vector, trig_mode); + } +break ; +default :
Quoted 8 lines
virt/kvm/irq_comm.c line 114
Quoted 4 lines
virt/kvm/irq_comm.c +2
-0+ printk(KERN_INFO "kvm: unsupported MSI delivery mode\n"); + }#endif }This looks very messy. Would be better to have the in-kernel irq structure contain a (*set_level)() callback that can take the appropriate action.
You means this part which would merged with ioapic, or something else?
Quoted 2 lines
Also, the
CONFIG_X86worries me, can we have IA64 enable this as well?
IA64 MSI enabling is on the task list, but it's pity we are too busy recently...
Signature · 2 lines
regards Yang, Sheng
Re: [PATCH 05/10] KVM: Merge MSI handling to kvm_set_irq
Quoted 7 lines · Sheng Yang
Quoted 4 lines
virt/kvm/irq_comm.c line 70
Quoted 2 lines
virt/kvm/irq_comm.c +1
-0+mutex_lock (&kvm->gsi_msg_lock);The lock is already taken here?
Um? For
gsi_msg_lock?
Sorry, my mistake. Will have to get used to all those locks.
Is there a way to avoid the lock? We're starting to complicate things...
Quoted 7 lines
Quoted 4 lines
This looks very messy. Would be better to have the in-kernel irq structure contain a (*set_level)() callback that can take the appropriate action.
You means this part which would merged with ioapic, or something else?
At the very least, separated into functions.
At best, kvm_set_irq() should become generic and just invoke callbacks supplied by the irqchip (pit, ioapic, msi).
Signature · 1 line
error compiling committee.c: too many arguments to function
Re: [PATCH 05/10] KVM: Merge MSI handling to kvm_set_irq
Quoted 10 lines · On Tuesday 30 December 2008 19:07:52 Avi Kivity wrote:
Quoted 5 lines · Sheng Yang
Quoted 3 lines
virt/kvm/irq_comm.c line 70
Quoted 1 line
virt/kvm/irq_comm.c +1
-0+mutex_lock (&kvm->gsi_msg_lock);The lock is already taken here?
Um? For
gsi_msg_lock?Sorry, my mistake. Will have to get used to all those locks.
Is there a way to avoid the lock? We're starting to complicate things...
Well, one list, one lock... But it's not very performance affect one, so maybe I can try kvm->lock...
Quoted 11 lines
Quoted 6 lines
Quoted 4 lines
This looks very messy. Would be better to have the in-kernel irq structure contain a (*set_level)() callback that can take the appropriate action.
You means this part which would merged with ioapic, or something else?
At the very least, separated into functions.
At best,
kvm_set_irq()should become generic and just invoke callbacks supplied by the irqchip (pit, ioapic, msi).
OK, I would separate it first. Callback function, a little later...
Signature · 2 lines
regards Yang, Sheng
PATCH v3 6/10 KVM: Split IOAPIC structure
In reply to Sheng YangPrepared for reuse ioapic_redir_entry for MSI.
--- include/linux/kvm_types.h | 17 +++++++++++++++++ virt/kvm/ioapic.c | 6 +++--- virt/kvm/ioapic.h | 17 +---------------- 3 files changed, 21 insertions(+), 19 deletions(-)
include/linux/kvm_types.h +17 -0
@@ -52,4 +52,21 @@ struct kvm_pio_request { int rep; }; +union kvm_ioapic_redirect_entry { +u64 bits; +struct { +u8 vector; +u8 delivery_mode:3 ; +u8 dest_mode:1 ; +u8 delivery_status:1 ; +u8 polarity:1 ; +u8 remote_irr:1 ; +u8 trig_mode:1 ; +u8 mask:1 ; +u8 reserve:7 ; +u8 reserved[4 ]; +u8 dest_id; + } fields; +}; +#endif /* __KVM_TYPES_H__ */
virt/kvm/ioapic.c +3 -3
@@ -85,7 +85,7 @@ static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic,static void ioapic_service (struct kvm_ioapic *ioapic,unsigned int idx) { -union ioapic_redir_entry *pent; +union kvm_ioapic_redirect_entry *pent; pent = &ioapic->redirtbl[idx];
@@ -272,7 +272,7 @@ void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level) {u32 old_irr = ioapic->irr;u32 mask =1 << irq; -union ioapic_redir_entry entry; +union kvm_ioapic_redirect_entry entry;if (irq >=0 && irq <IOAPIC_NUM_PINS ) { entry = ioapic->redirtbl[irq];
@@ -291,7 +291,7 @@ void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level)static void __kvm_ioapic_update_eoi (struct kvm_ioapic *ioapic,int gsi,int trigger_mode) { -union ioapic_redir_entry *ent; +union kvm_ioapic_redirect_entry *ent; ent = &ioapic->redirtbl[gsi];
virt/kvm/ioapic.h +1 -16
@@ -40,22 +40,7 @@ struct kvm_ioapic {u32 id;u32 irr;u32 pad; -union ioapic_redir_entry { -u64 bits; -struct { -u8 vector; -u8 delivery_mode:3 ; -u8 dest_mode:1 ; -u8 delivery_status:1 ; -u8 polarity:1 ; -u8 remote_irr:1 ; -u8 trig_mode:1 ; -u8 mask:1 ; -u8 reserve:7 ; -u8 reserved[4 ]; -u8 dest_id; - } fields; - } redirtbl[IOAPIC_NUM_PINS ]; +union kvm_ioapic_redirect_entry redirtbl[IOAPIC_NUM_PINS ];struct kvm_io_device dev;struct kvm *kvm;void (*ack_notifier)(void *opaque,int irq);
Signature · 1 line
1.5.4.5
PATCH v3 7/10 KVM: Unified the delivery of IOAPIC and MSI
In reply to Sheng Yang--- include/linux/kvm_host.h | 3 ++ virt/kvm/ioapic.c | 84 +++++++++++++++++----------------------------- virt/kvm/irq_comm.c | 75 ++++++++++++++++++++++++---------------- 3 files changed, 79 insertions(+), 83 deletions(-)
include/linux/kvm_host.h +3 -0
@@ -330,6 +330,9 @@ struct kvm_gsi_msg {struct hlist_node link; }; +void kvm_get_intr_delivery_bitmask (struct kvm_ioapic *ioapic, +union kvm_ioapic_redirect_entry *entry, +u32 *deliver_bitmask);void kvm_set_irq (struct kvm *kvm,int irq_source_id,u32 gsi,int level);void kvm_notify_acked_irq (struct kvm *kvm,unsigned gsi); void kvm_register_irq_ack_notifier(struct kvm *kvm,
virt/kvm/ioapic.c +31 -53
106 lines not on this page · raw message
virt/kvm/irq_comm.c +45 -30
98 lines not on this page · raw message
Signature · 1 line
1.5.4.5
PATCH v3 8/10 KVM: Change API of kvm_ioapic_get_delivery_bitmask
In reply to Sheng YangIn order to use with bit ops.
--- virt/kvm/ioapic.c | 17 ++++++++--------- virt/kvm/ioapic.h | 4 ++-- virt/kvm/irq_comm.c | 5 +++-- 3 files changed, 13 insertions(+), 13 deletions(-)
virt/kvm/ioapic.c +8 -9
@@ -153,22 +153,22 @@ static void ioapic_inj_nmi(struct kvm_vcpu *vcpu)kvm_vcpu_kick (vcpu); } -u32 kvm_ioapic_get_delivery_bitmask (struct kvm_ioapic *ioapic,u8 dest, -u8 dest_mode) +void kvm_ioapic_get_delivery_bitmask (struct kvm_ioapic *ioapic,u8 dest, +u8 dest_mode,u32 *mask) { -u32 mask =0 ;int i;struct kvm *kvm = ioapic->kvm;struct kvm_vcpu *vcpu;ioapic_debug ("dest %d dest_mode %d\n" , dest, dest_mode); + *mask =0 ;if (dest_mode ==0 ) {/* Physical mode. */ if (dest ==0xFF ) {/* Broadcast. */ for (i =0 ; i <KVM_MAX_VCPUS ; ++i)if (kvm->vcpus[i] && kvm->vcpus[i]->arch.apic) - mask |=1 << i; -return mask; + *mask |=1 << i; +return ; }for (i =0 ; i <KVM_MAX_VCPUS ; ++i) { vcpu = kvm->vcpus[i];
@@ -176,7 +176,7 @@ u32 kvm_ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest,continue ;if (kvm_apic_match_physical_addr (vcpu->arch.apic, dest)) {if (vcpu->arch.apic) - mask =1 << i; + *mask =1 << i;break ; } }
@@ -187,10 +187,9 @@ u32 kvm_ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest,continue ;if (vcpu->arch.apic &&kvm_apic_match_logical_addr (vcpu->arch.apic, dest)) - mask |=1 << vcpu->vcpu_id; + *mask |=1 << vcpu->vcpu_id; } -ioapic_debug ("mask %x\n" , mask); -return mask; +ioapic_debug ("mask %x\n" , *mask); }static int ioapic_deliver (struct kvm_ioapic *ioapic,int irq)
virt/kvm/ioapic.h +2 -2
@@ -70,7 +70,7 @@ void kvm_ioapic_update_eoi(struct kvm *kvm, int vector, int trigger_mode);int kvm_ioapic_init (struct kvm *kvm);void kvm_ioapic_set_irq (struct kvm_ioapic *ioapic,int irq,int level);void kvm_ioapic_reset (struct kvm_ioapic *ioapic); -u32 kvm_ioapic_get_delivery_bitmask (struct kvm_ioapic *ioapic,u8 dest, -u8 dest_mode); +void kvm_ioapic_get_delivery_bitmask (struct kvm_ioapic *ioapic,u8 dest, +u8 dest_mode,u32 *mask);#endif
virt/kvm/irq_comm.c +3 -2
@@ -35,8 +35,9 @@ void kvm_get_intr_delivery_bitmask(struct kvm_ioapic *ioapic, {struct kvm_vcpu *vcpu; - *deliver_bitmask =kvm_ioapic_get_delivery_bitmask (ioapic, - entry->fields.dest_id, entry->fields.dest_mode); +kvm_ioapic_get_delivery_bitmask (ioapic, entry->fields.dest_id, + entry->fields.dest_mode, + deliver_bitmask);switch (entry->fields.delivery_mode) {case IOAPIC_LOWEST_PRIORITY : vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm,
Signature · 1 line
1.5.4.5
PATCH v3 9/10 KVM: Update intr delivery func to accept unsigned long* bitmap
In reply to Sheng YangWould be used with bit ops, and would be easily extended if KVM_MAX_VCPUS is increased.
--- arch/x86/kvm/lapic.c | 8 ++++---- include/linux/kvm_host.h | 2 +- virt/kvm/ioapic.c | 4 ++-- virt/kvm/ioapic.h | 4 ++-- virt/kvm/irq_comm.c | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-)
arch/x86/kvm/lapic.c +4 -4
@@ -403,7 +403,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, }static struct kvm_lapic *kvm_apic_round_robin (struct kvm *kvm,u8 vector, -unsigned long bitmap) +unsigned long *bitmap) {int last;int next;
@@ -415,7 +415,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, do { if (++next == KVM_MAX_VCPUS) next = 0; - if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap)) + if (kvm->vcpus[next] == NULL || !test_bit(next, bitmap)) continue; apic = kvm->vcpus[next]->arch.apic; if (apic && apic_enabled(apic))
@@ -431,7 +431,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, }struct kvm_vcpu *kvm_get_lowest_prio_vcpu (struct kvm *kvm,u8 vector, -unsigned long bitmap) +unsigned long *bitmap) {struct kvm_lapic *apic;
@@ -502,7 +502,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) }if (delivery_mode ==APIC_DM_LOWEST ) { - target =kvm_get_lowest_prio_vcpu (vcpu->kvm, vector, lpr_map); + target =kvm_get_lowest_prio_vcpu (vcpu->kvm, vector, &lpr_map);if (target !=NULL )__apic_accept_irq (target->arch.apic, delivery_mode, vector, level, trig_mode);
include/linux/kvm_host.h +1 -1
@@ -332,7 +332,7 @@ struct kvm_gsi_msg {void kvm_get_intr_delivery_bitmask (struct kvm_ioapic *ioapic,union kvm_ioapic_redirect_entry *entry, -u32 *deliver_bitmask); +unsigned long *deliver_bitmask);void kvm_set_irq (struct kvm *kvm,int irq_source_id,u32 gsi,int level);void kvm_notify_acked_irq (struct kvm *kvm,unsigned gsi); void kvm_register_irq_ack_notifier(struct kvm *kvm,
virt/kvm/ioapic.c +2 -2
@@ -154,7 +154,7 @@ static void ioapic_inj_nmi(struct kvm_vcpu *vcpu) }void kvm_ioapic_get_delivery_bitmask (struct kvm_ioapic *ioapic,u8 dest, -u8 dest_mode,u32 *mask) +u8 dest_mode,unsigned long *mask) {int i;struct kvm *kvm = ioapic->kvm;
@@ -195,7 +195,7 @@ void kvm_ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest,static int ioapic_deliver (struct kvm_ioapic *ioapic,int irq) {union kvm_ioapic_redirect_entry entry = ioapic->redirtbl[irq]; -u32 deliver_bitmask; +unsigned long deliver_bitmask;struct kvm_vcpu *vcpu;int vcpu_id, r =0 ;
virt/kvm/ioapic.h +2 -2
@@ -65,12 +65,12 @@ static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) }struct kvm_vcpu *kvm_get_lowest_prio_vcpu (struct kvm *kvm,u8 vector, -unsigned long bitmap); +unsigned long *bitmap);void kvm_ioapic_update_eoi (struct kvm *kvm,int vector,int trigger_mode);int kvm_ioapic_init (struct kvm *kvm);void kvm_ioapic_set_irq (struct kvm_ioapic *ioapic,int irq,int level);void kvm_ioapic_reset (struct kvm_ioapic *ioapic);void kvm_ioapic_get_delivery_bitmask (struct kvm_ioapic *ioapic,u8 dest, -u8 dest_mode,u32 *mask); +u8 dest_mode,unsigned long *mask);#endif
virt/kvm/irq_comm.c +3 -3
@@ -31,7 +31,7 @@void kvm_get_intr_delivery_bitmask (struct kvm_ioapic *ioapic,union kvm_ioapic_redirect_entry *entry, -u32 *deliver_bitmask) +unsigned long *deliver_bitmask) {struct kvm_vcpu *vcpu;
@@ -41,7 +41,7 @@ void kvm_get_intr_delivery_bitmask(struct kvm_ioapic *ioapic,switch (entry->fields.delivery_mode) {case IOAPIC_LOWEST_PRIORITY : vcpu =kvm_get_lowest_prio_vcpu (ioapic->kvm, - entry->fields.vector, *deliver_bitmask); + entry->fields.vector, deliver_bitmask); *deliver_bitmask =1 << vcpu->vcpu_id;break ;case IOAPIC_FIXED :
@@ -63,7 +63,7 @@ void kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 gsi, int level)struct kvm_ioapic *ioapic =ioapic_irqchip (kvm);struct kvm_gsi_msg *gsi_msg;union kvm_ioapic_redirect_entry entry; -u32 deliver_bitmask; +unsigned long deliver_bitmask;BUG_ON (!ioapic);#endif
Signature · 1 line
1.5.4.5
PATCH v3 10/10 KVM: bit ops for deliver_bitmap
In reply to Sheng YangIt's also convenient when we extend KVM supported vcpu number in the future.
--- arch/x86/kvm/lapic.c | 7 ++++--- virt/kvm/ioapic.c | 24 +++++++++++++----------- virt/kvm/irq_comm.c | 16 ++++++++-------- 3 files changed, 25 insertions(+), 22 deletions(-)
arch/x86/kvm/lapic.c +4 -3
@@ -477,9 +477,10 @@ static void apic_send_ipi(struct kvm_lapic *apic)struct kvm_vcpu *target;struct kvm_vcpu *vcpu; -unsigned long lpr_map =0 ; +DECLARE_BITMAP (lpr_map,KVM_MAX_VCPUS );int i; +bitmap_zero (lpr_map,KVM_MAX_VCPUS ); apic_debug("icr_high 0x%x, icr_low 0x%x, " "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n",
@@ -494,7 +495,7 @@ static void apic_send_ipi(struct kvm_lapic *apic)if (vcpu->arch.apic &&apic_match_dest (vcpu, apic, short_hand, dest, dest_mode)) {if (delivery_mode ==APIC_DM_LOWEST ) -set_bit (vcpu->vcpu_id, &lpr_map); +set_bit (vcpu->vcpu_id, lpr_map);else __apic_accept_irq (vcpu->arch.apic, delivery_mode, vector, level, trig_mode);
@@ -502,7 +503,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) }if (delivery_mode ==APIC_DM_LOWEST ) { - target =kvm_get_lowest_prio_vcpu (vcpu->kvm, vector, &lpr_map); + target =kvm_get_lowest_prio_vcpu (vcpu->kvm, vector, lpr_map);if (target !=NULL )__apic_accept_irq (target->arch.apic, delivery_mode, vector, level, trig_mode);
virt/kvm/ioapic.c +13 -11
@@ -195,7 +195,7 @@ void kvm_ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest,static int ioapic_deliver (struct kvm_ioapic *ioapic,int irq) {union kvm_ioapic_redirect_entry entry = ioapic->redirtbl[irq]; -unsigned long deliver_bitmask; +DECLARE_BITMAP (deliver_bitmask,KVM_MAX_VCPUS );struct kvm_vcpu *vcpu;int vcpu_id, r =0 ;
@@ -205,22 +205,24 @@ static int ioapic_deliver(struct kvm_ioapic *ioapic, int irq) entry.fields.delivery_mode, entry.fields.vector, entry.fields.trig_mode); -kvm_get_intr_delivery_bitmask (ioapic, &entry, &deliver_bitmask); -if (!deliver_bitmask) { -ioapic_debug ("no target on destination\n" ); -return 0 ; - } +bitmap_zero (deliver_bitmask,KVM_MAX_VCPUS );/* Always delivery PIT interrupt to vcpu 0 */ #ifdef CONFIG_X86 if (irq ==0 ) - deliver_bitmask =1 <<0 ; +set_bit (0 , deliver_bitmask); + else#endif +kvm_get_intr_delivery_bitmask (ioapic, &entry, deliver_bitmask); + +if (find_first_bit (deliver_bitmask,KVM_MAX_VCPUS ) >=KVM_MAX_VCPUS ) { +ioapic_debug ("no target on destination\n" ); +return 0 ; + } -for (vcpu_id =0 ; deliver_bitmask !=0 ; vcpu_id++) { -if (!(deliver_bitmask & (1 << vcpu_id))) -continue ; - deliver_bitmask &= ~(1 << vcpu_id); +while ((vcpu_id =find_first_bit (deliver_bitmask,KVM_MAX_VCPUS )) + <KVM_MAX_VCPUS ) { +clear_bit (vcpu_id, deliver_bitmask); vcpu = ioapic->kvm->vcpus[vcpu_id];if (vcpu) { if (entry.fields.delivery_mode ==
virt/kvm/irq_comm.c +8 -8
@@ -42,7 +42,7 @@ void kvm_get_intr_delivery_bitmask(struct kvm_ioapic *ioapic,case IOAPIC_LOWEST_PRIORITY : vcpu =kvm_get_lowest_prio_vcpu (ioapic->kvm, entry->fields.vector, deliver_bitmask); - *deliver_bitmask =1 << vcpu->vcpu_id; +set_bit (vcpu->vcpu_id, deliver_bitmask);break ;case IOAPIC_FIXED :case IOAPIC_NMI :
@@ -63,11 +63,12 @@ void kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 gsi, int level)struct kvm_ioapic *ioapic =ioapic_irqchip (kvm);struct kvm_gsi_msg *gsi_msg;union kvm_ioapic_redirect_entry entry; -unsigned long deliver_bitmask; +DECLARE_BITMAP (deliver_bitmask,KVM_MAX_VCPUS );BUG_ON (!ioapic);#endif +bitmap_zero (deliver_bitmask,KVM_MAX_VCPUS );if (!(gsi &KVM_GSI_MSG_MASK )) {int irq = gsi;
@@ -111,16 +112,15 @@ void kvm_set_irq(struct kvm *kvm, int irq_source_id, u32 gsi, int level) entry.fields.delivery_mode =test_bit (MSI_DATA_DELIVERY_MODE_SHIFT , (unsigned long *)&gsi_msg->msg.data); -kvm_get_intr_delivery_bitmask (ioapic, &entry, &deliver_bitmask); +kvm_get_intr_delivery_bitmask (ioapic, &entry, deliver_bitmask); -if (!deliver_bitmask) { +if (find_first_bit (deliver_bitmask,KVM_MAX_VCPUS ) >=KVM_MAX_VCPUS ) {printk (KERN_WARNING "kvm: no destination for MSI delivery!" );return ; } -for (vcpu_id =0 ; deliver_bitmask !=0 ; vcpu_id++) { -if (!(deliver_bitmask & (1 << vcpu_id))) -continue ; - deliver_bitmask &= ~(1 << vcpu_id); +while ((vcpu_id =find_first_bit (deliver_bitmask,KVM_MAX_VCPUS )) + <KVM_MAX_VCPUS ) { +clear_bit (vcpu_id, deliver_bitmask); vcpu = ioapic->kvm->vcpus[vcpu_id];if (vcpu) kvm_apic_set_irq(vcpu, entry.fields.vector,
Signature · 1 line
1.5.4.5
Quoted 6 lines · On Tuesday 30 December 2008 13:55:52 Sheng Yang wrote:
Update from v2:
Addgsi_msg_pending_bitmap, in order to support MSI-X mutiply interrupts.And this one depends on the former "Device assignment code clean up and MSI disable support" patchset.
Sorry, a little chaos here. The gsi_msg_pending_bitmap is in MSI-X patchset...
So this one should be almost the same as v2. Just resend.
Signature · 2 lines
regards Yang, Sheng