Skip to contentSkip to conversation

enable PCI multiple-segments for pass-through device

PATCH Applied Zhai, Edwin 4 participants 7 messages Jan 29, 2010 – Feb 2, 2010

Subsystems (3)
x86 KVM KVM x86
lore
Share & tools
lore

Zhai, Edwin

author
#0 lore raw

PATCH 1/1 enable PCI multiple-segments for pass-through device

These 2 patches enable optional parameter(default 0) - PCI segment(or domain) besides BDF, when assigning PCI device to guest.

Signed-off-by Zhai Edwin <[email protected]>
Signature · 2 lines
best rgds,
edwin
virt/kvm/assigned-dev.c +3 -1
Index: linux-2.6/virt/kvm/assigned-dev.c
===================================================================
@@ -526,7 +526,8 @@ static int kvm_vm_ioctl_assign_device(st
 		r = -ENOMEM;
 		goto out;
 	}
-	dev = pci_get_bus_and_slot(assigned_dev->busnr,
+	dev = pci_get_domain_bus_and_slot(assigned_dev->segnr,
+				   assigned_dev->busnr,
 				   assigned_dev->devfn);
 	if (!dev) {
 		printk(KERN_INFO "%s: host device not found\n", __func__);

@@ -548,6 +549,7 @@ static int kvm_vm_ioctl_assign_device(st
 	pci_reset_function(dev);
 
 	match->assigned_dev_id = assigned_dev->assigned_dev_id;
+	match->host_segnr = assigned_dev->segnr;
 	match->host_busnr = assigned_dev->busnr;
 	match->host_devfn = assigned_dev->devfn;
 	match->flags = assigned_dev->flags;
virt/kvm/iommu.c +6 -3
Index: linux-2.6/virt/kvm/iommu.c
===================================================================
@@ -106,7 +106,8 @@ int kvm_assign_device(struct kvm *kvm,
 
 	r = iommu_attach_device(domain, &pdev->dev);
 	if (r) {
-		printk(KERN_ERR "assign device %x:%x.%x failed",
+		printk(KERN_ERR "assign device %x:%x:%x.%x failed",
+			pci_domain_nr(pdev->bus),
 			pdev->bus->number,
 			PCI_SLOT(pdev->devfn),
 			PCI_FUNC(pdev->devfn));

@@ -127,7 +128,8 @@ int kvm_assign_device(struct kvm *kvm,
 			goto out_unmap;
 	}
 
-	printk(KERN_DEBUG "assign device: host bdf = %x:%x:%x\n",
+	printk(KERN_DEBUG "assign device %x:%x:%x.%x\n",
+		assigned_dev->host_segnr,
 		assigned_dev->host_busnr,
 		PCI_SLOT(assigned_dev->host_devfn),
 		PCI_FUNC(assigned_dev->host_devfn));

@@ -154,7 +156,8 @@ int kvm_deassign_device(struct kvm *kvm,
 
 	iommu_detach_device(domain, &pdev->dev);
 
-	printk(KERN_DEBUG "deassign device: host bdf = %x:%x:%x\n",
+	printk(KERN_DEBUG "deassign device %x:%x:%x.%x\n",
+		assigned_dev->host_segnr,
 		assigned_dev->host_busnr,
 		PCI_SLOT(assigned_dev->host_devfn),
 		PCI_FUNC(assigned_dev->host_devfn));
include/linux/kvm.h +2 -1
Index: linux-2.6/include/linux/kvm.h
===================================================================
@@ -691,8 +691,9 @@ struct kvm_assigned_pci_dev {
 	__u32 busnr;
 	__u32 devfn;
 	__u32 flags;
+	__u32 segnr;
 	union {
-		__u32 reserved[12];
+		__u32 reserved[11];
 	};
 };
include/linux/kvm_host.h +1 -0
Index: linux-2.6/include/linux/kvm_host.h
===================================================================
@@ -400,6 +400,7 @@ struct kvm_assigned_dev_kernel {
 	struct work_struct interrupt_work;
 	struct list_head list;
 	int assigned_dev_id;
+	int host_segnr;
 	int host_busnr;
 	int host_devfn;
 	unsigned int entries_nr;
hw/device-assignment.c +29 -27

163 lines not on this page · raw message

hw/device-assignment.h +2 -0
Index: qemu-kvm/hw/device-assignment.h
===================================================================
@@ -37,6 +37,7 @@
 #define PCI_DEVFN(slot, func)   ((((slot) & 0x1f) << 3) | ((func) & 0x07))
 
 typedef struct PCIHostDevice {
+    int seg;
     int bus;
     int dev;
     int func;

@@ -82,6 +83,7 @@ typedef struct AssignedDevice {
     PCIDevRegions real_device;
     int run;
     int girq;
+    unsigned int h_segnr;
     unsigned char h_busnr;
     unsigned int h_devfn;
     int irq_requested_type;
hw/pci.c +34 -6
Index: qemu-kvm/hw/pci.c
===================================================================
@@ -479,21 +479,48 @@ static int pci_parse_devaddr(const char 
 }
 
 /*
- * Parse device bdf in device assignment command:
+ * Parse device seg and bdf in device assignment command:
  *
- * -pcidevice host=bus:dev.func
+ * -pcidevice host=[seg:]bus:dev.func
  *
- * Parse <bus>:<slot>.<func> return -1 on error
+ * Parse [seg:]<bus>:<slot>.<func> return -1 on error
  */
-int pci_parse_host_devaddr(const char *addr, int *busp,
+int pci_parse_host_devaddr(const char *addr, int *segp, int *busp,
                            int *slotp, int *funcp)
 {
     const char *p;
     char *e;
     int val;
-    int bus = 0, slot = 0, func = 0;
+    int seg = 0, bus = 0, slot = 0, func = 0;
 
+    /* parse optional seg */
     p = addr;
+    val = 0;
+    while (1) {
+        p = strchr(p, ':');
+        if (p) {
+            val++;
+            p++;
+        } else
+            break;
+    }
+    if (val <= 0 || val > 2)
+        return -1;
+
+    p = addr;
+    if (val == 2) {
+        val = strtoul(p, &e, 16);
+        if (e == p)
+            return -1;
+        if (*e == ':') {
+            seg = val;
+            p = e + 1;
+        }
+    } else
+        seg = 0;
+
+
+    /* parse bdf */
     val = strtoul(p, &e, 16);
     if (e == p)
 	return -1;

@@ -515,12 +542,13 @@ int pci_parse_host_devaddr(const char *a
     } else
 	return -1;
 
-    if (bus > 0xff || slot > 0x1f || func > 0x7)
+    if (seg > 0xffff || bus > 0xff || slot > 0x1f || func > 0x7)
 	return -1;
 
     if (*e)
 	return -1;
 
+    *segp = seg;
     *busp = bus;
     *slotp = slot;
     *funcp = func;
hw/pci.h +1 -1
Index: qemu-kvm/hw/pci.h
===================================================================
@@ -275,7 +275,7 @@ PCIBus *pci_get_bus_devfn(int *devfnp, c
 int pci_read_devaddr(Monitor *mon, const char *addr, int *domp, int *busp,
                      unsigned *slotp);
 
-int pci_parse_host_devaddr(const char *addr, int *busp,
+int pci_parse_host_devaddr(const char *addr, int *segp, int *busp,
                            int *slotp, int *funcp);
 
 void pci_info(Monitor *mon);
kvm/include/linux/kvm.h +2 -1
Index: qemu-kvm/kvm/include/linux/kvm.h
===================================================================
@@ -686,8 +686,9 @@ struct kvm_assigned_pci_dev {
 	__u32 busnr;
 	__u32 devfn;
 	__u32 flags;
+	__u32 segnr;
 	union {
-		__u32 reserved[12];
+		__u32 reserved[11];
 	};
 };
Acked-by on v1 1/1
#1 lore raw
Quoted 4 lines · * Zhai, Edwin ([email protected]) wrote:

These 2 patches enable optional parameter(default 0) - PCI segment(or domain) besides BDF, when assigning PCI device to guest.

Signed-off-by Zhai Edwin <[email protected]>

Looks good. For compatibility, an old userspace will zero the padding and keep the default 0 segment when running w/ a new kernel, a new userspace won't be able to pass a non-zero segment value to an old kernel. If that latter bit matters, need a capability to express this change. Looks like qemu-options.hx needs an update too. Otherwise...

Acked-by Chris Wright <[email protected]>
author
#2 lore raw

Wright,
Thanks for your comments. I have updated the qemu-options.hx and used following new CAP. Updated version is also attached.

hw/device-assignment.c +10 -0
Index: qemu-kvm/hw/device-assignment.c
===================================================================
@@ -809,6 +809,16 @@ static int assign_device(AssignedDevice
    struct kvm_assigned_pci_dev assigned_dev_data;
    int r;
 
+#ifdef KVM_CAP_PCI_SEGMENT
+    /* Only pass non-zero PCI segment to capable module */
+    if (!kvm_check_extension(kvm_state, KVM_CAP_PCI_SEGMENT) &&
+        dev->h_segnr) {
+        fprintf(stderr, "Can't assign device inside non-zero PCI segment "
+                "as this KVM module doesn't support it.\n");
+        return -ENODEV;
+    }
+#endif
+
    memset(&assigned_dev_data, 0, sizeof(assigned_dev_data));
    assigned_dev_data.assigned_dev_id  =
    calc_assigned_dev_id(dev->h_segnr, dev->h_busnr, dev->h_devfn);
kvm/include/linux/kvm.h +0 -0
Index: qemu-kvm/kvm/include/linux/kvm.h
===================================================================
@@ -498,6 +498,8 @@ struct kvm_ioeventfd {

#define KVM_CAP_S390_PSW 42
#define KVM_CAP_PPC_SEGSTATE 43

+#define KVM_CAP_PCI_SEGMENT 47
+
#ifdef KVM_CAP_IRQ_ROUTING

struct kvm_irq_routing_irqchip {

include/linux/kvm.h +0 -0
Index: linux-2.6/include/linux/kvm.h
===================================================================
@@ -500,6 +500,7 @@ struct kvm_ioeventfd {

#define KVM_CAP_HYPERV 44
#define KVM_CAP_HYPERV_VAPIC 45
#define KVM_CAP_HYPERV_SPIN 46
+#define KVM_CAP_PCI_SEGMENT 47

#ifdef KVM_CAP_IRQ_ROUTING

arch/x86/kvm/x86.c +1 -0
Index: linux-2.6/arch/x86/kvm/x86.c
===================================================================
@@ -1569,6 +1569,7 @@ int kvm_dev_ioctl_check_extension(long e
    case KVM_CAP_HYPERV:
    case KVM_CAP_HYPERV_VAPIC:
    case KVM_CAP_HYPERV_SPIN:
+    case KVM_CAP_PCI_SEGMENT:
        r = 1;
        break;
    case KVM_CAP_COALESCED_MMIO:
Quoted 5 lines · Chris Wright
Quoted 3 lines · * Zhai, Edwin ([email protected]) wrote:

These 2 patches enable optional parameter(default 0) - PCI segment(or domain) besides BDF, when assigning PCI device to guest.

Signed-off-by Zhai Edwin <[email protected]>
Quoted 8 lines

Looks good. For compatibility, an old userspace will zero the padding and keep the default 0 segment when running w/ a new kernel, a new userspace won't be able to pass a non-zero segment value to an old kernel. If that latter bit matters, need a capability to express this change. Looks like qemu-options.hx needs an update too. Otherwise...

Acked-by Chris Wright <[email protected]>
hw/device-assignment.c +39 -27

175 lines not on this page · raw message

hw/device-assignment.h +2 -0
Index: qemu-kvm/hw/device-assignment.h
===================================================================
@@ -37,6 +37,7 @@
 #define PCI_DEVFN(slot, func)   ((((slot) & 0x1f) << 3) | ((func) & 0x07))
 
 typedef struct PCIHostDevice {
+    int seg;
     int bus;
     int dev;
     int func;

@@ -82,6 +83,7 @@ typedef struct AssignedDevice {
     PCIDevRegions real_device;
     int run;
     int girq;
+    unsigned int h_segnr;
     unsigned char h_busnr;
     unsigned int h_devfn;
     int irq_requested_type;
hw/pci.c +34 -6
Index: qemu-kvm/hw/pci.c
===================================================================
@@ -480,21 +480,48 @@ static int pci_parse_devaddr(const char 
 }
 
 /*
- * Parse device bdf in device assignment command:
+ * Parse device seg and bdf in device assignment command:
  *
- * -pcidevice host=bus:dev.func
+ * -pcidevice host=[seg:]bus:dev.func
  *
- * Parse <bus>:<slot>.<func> return -1 on error
+ * Parse [seg:]<bus>:<slot>.<func> return -1 on error
  */
-int pci_parse_host_devaddr(const char *addr, int *busp,
+int pci_parse_host_devaddr(const char *addr, int *segp, int *busp,
                            int *slotp, int *funcp)
 {
     const char *p;
     char *e;
     int val;
-    int bus = 0, slot = 0, func = 0;
+    int seg = 0, bus = 0, slot = 0, func = 0;
 
+    /* parse optional seg */
     p = addr;
+    val = 0;
+    while (1) {
+        p = strchr(p, ':');
+        if (p) {
+            val++;
+            p++;
+        } else
+            break;
+    }
+    if (val <= 0 || val > 2)
+        return -1;
+
+    p = addr;
+    if (val == 2) {
+        val = strtoul(p, &e, 16);
+        if (e == p)
+            return -1;
+        if (*e == ':') {
+            seg = val;
+            p = e + 1;
+        }
+    } else
+        seg = 0;
+
+
+    /* parse bdf */
     val = strtoul(p, &e, 16);
     if (e == p)
 	return -1;

@@ -516,12 +543,13 @@ int pci_parse_host_devaddr(const char *a
     } else
 	return -1;
 
-    if (bus > 0xff || slot > 0x1f || func > 0x7)
+    if (seg > 0xffff || bus > 0xff || slot > 0x1f || func > 0x7)
 	return -1;
 
     if (*e)
 	return -1;
 
+    *segp = seg;
     *busp = bus;
     *slotp = slot;
     *funcp = func;
hw/pci.h +1 -1
Index: qemu-kvm/hw/pci.h
===================================================================
@@ -276,7 +276,7 @@ PCIBus *pci_get_bus_devfn(int *devfnp, c
 int pci_read_devaddr(Monitor *mon, const char *addr, int *domp, int *busp,
                      unsigned *slotp);
 
-int pci_parse_host_devaddr(const char *addr, int *busp,
+int pci_parse_host_devaddr(const char *addr, int *segp, int *busp,
                            int *slotp, int *funcp);
 
 void pci_info(Monitor *mon);
kvm/include/linux/kvm.h +4 -1
Index: qemu-kvm/kvm/include/linux/kvm.h
===================================================================
@@ -498,6 +498,8 @@ struct kvm_ioeventfd {
 #define KVM_CAP_S390_PSW 42
 #define KVM_CAP_PPC_SEGSTATE 43
 
+#define KVM_CAP_PCI_SEGMENT 47
+
 #ifdef KVM_CAP_IRQ_ROUTING
 
 struct kvm_irq_routing_irqchip {

@@ -691,8 +693,9 @@ struct kvm_assigned_pci_dev {
 	__u32 busnr;
 	__u32 devfn;
 	__u32 flags;
+	__u32 segnr;
 	union {
-		__u32 reserved[12];
+		__u32 reserved[11];
 	};
 };
qemu-options.hx +1 -1
Index: qemu-kvm/qemu-options.hx
===================================================================
@@ -1977,7 +1977,7 @@ DEF("no-kvm-pit-reinjection", 0, QEMU_OP
     "                disable KVM kernel mode PIT interrupt reinjection\n")
 #if defined(TARGET_I386) || defined(TARGET_X86_64) || defined(TARGET_IA64) || defined(__linux__)
 DEF("pcidevice", HAS_ARG, QEMU_OPTION_pcidevice,
-    "-pcidevice host=bus:dev.func[,dma=none][,name=string]\n"
+    "-pcidevice host=[seg:]bus:dev.func[,dma=none][,name=string]\n"
     "                expose a PCI device to the guest OS\n"
     "                dma=none: don't perform any dma translations (default is to use an iommu)\n"
     "                'string' is used in log output\n")
virt/kvm/assigned-dev.c +3 -1
Index: linux-2.6/virt/kvm/assigned-dev.c
===================================================================
@@ -526,7 +526,8 @@ static int kvm_vm_ioctl_assign_device(st
 		r = -ENOMEM;
 		goto out;
 	}
-	dev = pci_get_bus_and_slot(assigned_dev->busnr,
+	dev = pci_get_domain_bus_and_slot(assigned_dev->segnr,
+				   assigned_dev->busnr,
 				   assigned_dev->devfn);
 	if (!dev) {
 		printk(KERN_INFO "%s: host device not found\n", __func__);

@@ -548,6 +549,7 @@ static int kvm_vm_ioctl_assign_device(st
 	pci_reset_function(dev);
 
 	match->assigned_dev_id = assigned_dev->assigned_dev_id;
+	match->host_segnr = assigned_dev->segnr;
 	match->host_busnr = assigned_dev->busnr;
 	match->host_devfn = assigned_dev->devfn;
 	match->flags = assigned_dev->flags;
virt/kvm/iommu.c +6 -3
Index: linux-2.6/virt/kvm/iommu.c
===================================================================
@@ -106,7 +106,8 @@ int kvm_assign_device(struct kvm *kvm,
 
 	r = iommu_attach_device(domain, &pdev->dev);
 	if (r) {
-		printk(KERN_ERR "assign device %x:%x.%x failed",
+		printk(KERN_ERR "assign device %x:%x:%x.%x failed",
+			pci_domain_nr(pdev->bus),
 			pdev->bus->number,
 			PCI_SLOT(pdev->devfn),
 			PCI_FUNC(pdev->devfn));

@@ -127,7 +128,8 @@ int kvm_assign_device(struct kvm *kvm,
 			goto out_unmap;
 	}
 
-	printk(KERN_DEBUG "assign device: host bdf = %x:%x:%x\n",
+	printk(KERN_DEBUG "assign device %x:%x:%x.%x\n",
+		assigned_dev->host_segnr,
 		assigned_dev->host_busnr,
 		PCI_SLOT(assigned_dev->host_devfn),
 		PCI_FUNC(assigned_dev->host_devfn));

@@ -154,7 +156,8 @@ int kvm_deassign_device(struct kvm *kvm,
 
 	iommu_detach_device(domain, &pdev->dev);
 
-	printk(KERN_DEBUG "deassign device: host bdf = %x:%x:%x\n",
+	printk(KERN_DEBUG "deassign device %x:%x:%x.%x\n",
+		assigned_dev->host_segnr,
 		assigned_dev->host_busnr,
 		PCI_SLOT(assigned_dev->host_devfn),
 		PCI_FUNC(assigned_dev->host_devfn));
include/linux/kvm.h +3 -1
Index: linux-2.6/include/linux/kvm.h
===================================================================
@@ -500,6 +500,7 @@ struct kvm_ioeventfd {
 #define KVM_CAP_HYPERV 44
 #define KVM_CAP_HYPERV_VAPIC 45
 #define KVM_CAP_HYPERV_SPIN 46
+#define KVM_CAP_PCI_SEGMENT 47
 
 #ifdef KVM_CAP_IRQ_ROUTING
 

@@ -694,8 +695,9 @@ struct kvm_assigned_pci_dev {
 	__u32 busnr;
 	__u32 devfn;
 	__u32 flags;
+	__u32 segnr;
 	union {
-		__u32 reserved[12];
+		__u32 reserved[11];
 	};
 };
include/linux/kvm_host.h +1 -0
Index: linux-2.6/include/linux/kvm_host.h
===================================================================
@@ -400,6 +400,7 @@ struct kvm_assigned_dev_kernel {
 	struct work_struct interrupt_work;
 	struct list_head list;
 	int assigned_dev_id;
+	int host_segnr;
 	int host_busnr;
 	int host_devfn;
 	unsigned int entries_nr;
arch/x86/kvm/x86.c +1 -0
Index: linux-2.6/arch/x86/kvm/x86.c
===================================================================
@@ -1569,6 +1569,7 @@ int kvm_dev_ioctl_check_extension(long e
 	case KVM_CAP_HYPERV:
 	case KVM_CAP_HYPERV_VAPIC:
 	case KVM_CAP_HYPERV_SPIN:
+	case KVM_CAP_PCI_SEGMENT:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
Applied
#3 lore raw
Quoted 3 lines · Edwin

Wright,
Thanks for your comments. I have updated the qemu-options.hx and used following new CAP. Updated version is also attached.

Applied, thanks.

Please send one patch per email, with patch inlined, in the future.

#4 lore raw
Quoted 9 lines · Marcelo Tosatti
Quoted 3 lines · Edwin

Wright,
Thanks for your comments. I have updated the qemu-options.hx and used following new CAP. Updated version is also attached.

Applied, thanks.

Please send one patch per email, with patch inlined, in the future.

This commit doesn't build without KVM_CAP_PCI_SEGMENT, please fix.

Jan

author
#5 lore raw
Quoted 11 lines · Jan Kiszka
Quoted 6 lines · Marcelo Tosatti

Applied, thanks.

Please send one patch per email, with patch inlined, in the future.

This commit doesn't build without KVM_CAP_PCI_SEGMENT, please fix.

Jan,
What does it mean? Can we update all files except the "kvm.h"?

Thanks,

Quoted 3 lines

Jan

Signature · 2 lines
best rgds,
edwin
#6 lore raw
Quoted 11 lines · * Zhai, Edwin ([email protected]) wrote:
Quoted 8 lines · Jan Kiszka
Quoted 4 lines · Marcelo Tosatti

Applied, thanks.

Please send one patch per email, with patch inlined, in the future.

This commit doesn't build without KVM_CAP_PCI_SEGMENT, please fix.

What does it mean? Can we update all files except the "kvm.h"?

I think Marcelo already fixed...

ed880109 Fix device-assignment.c compilation without KVM_CAP_PCI_SEGMENT