Skip to contentSkip to conversation

qemu-kvm: Fixes and improvements for device assigment

PATCH Applied Jan Kiszka 4 participants 11 messages Nov 2, 2010 – Nov 3, 2010 3 messages not in the archive KVM

lore
Share & tools
lore

Jan Kiszka

author
#0 lore raw

cover qemu-kvm: Fixes and improvements for device assigment

Mostly usability fixes, but also a patch to allow using devices that have problems with MSIs.

Jan Kiszka (4):
pci-assign: Convert iommu property to booleam
pci-assign: Allow to disable MSI perference for host IRQ
pci-assign: Issue warning when running w/o IOMMU
pci-assign: Remove broken -pcidevice and pci_add host

 hmp-commands.hx        |    2 +-
 hw/device-assignment.c |   81 ++++++++---------------------------------------
 hw/device-assignment.h |   15 ++++-----
 hw/ipf.c               |    6 ---
 hw/pc.c                |    6 ---
 hw/pci-hotplug.c       |   22 -------------
 qemu-options.hx        |    5 ---
 vl.c                   |   14 --------
 8 files changed, 22 insertions(+), 129 deletions(-)
author
#1 lore raw

PATCH 1/4 pci-assign: Convert iommu property to booleam

Defining the iommu property as an integer opens the door for confusion (Is it an index or a switch?). Fix this by converting it to a bit property that can be on or off, nothing else.

---
 hw/device-assignment.c |    7 ++++---
 hw/device-assignment.h |    6 +++++-
 2 files changed, 9 insertions(+), 4 deletions(-)
hw/device-assignment.c +4 -3
index 2605bd1..349e864 100644
@@ -902,7 +902,7 @@ static int assign_device(AssignedDevice *dev)
 
 #ifdef KVM_CAP_IOMMU
     /* We always enable the IOMMU unless disabled on the command line */
-    if (dev->use_iommu) {
+    if (dev->features & ASSIGNED_DEVICE_USE_IOMMU_MASK) {
         if (!kvm_check_extension(kvm_state, KVM_CAP_IOMMU)) {
             fprintf(stderr, "No IOMMU found.  Unable to assign device \"%s\"\n",
                     dev->dev.qdev.id);
@@ -911,7 +911,7 @@ static int assign_device(AssignedDevice *dev)
         assigned_dev_data.flags |= KVM_DEV_ASSIGN_ENABLE_IOMMU;
     }
 #else
-    dev->use_iommu = 0;
+    dev->features &= ~ASSIGNED_DEVICE_USE_IOMMU_MASK;
 #endif
 
     r = kvm_assign_pci_device(kvm_context, &assigned_dev_data);
@@ -1538,7 +1538,8 @@ static PCIDeviceInfo assign_info = {
     .config_write = assigned_dev_pci_write_config,
     .qdev.props   = (Property[]) {
         DEFINE_PROP("host", AssignedDevice, host, qdev_prop_hostaddr, PCIHostDevice),
-        DEFINE_PROP_UINT32("iommu", AssignedDevice, use_iommu, 1),
+        DEFINE_PROP_BIT("iommu", AssignedDevice, features,
+                        ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
         DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
         DEFINE_PROP_END_OF_LIST(),
     },
hw/device-assignment.h +5 -1
index 2f5fa17..4eb5835 100644
@@ -74,10 +74,14 @@ typedef struct {
     PCIRegion *region;
 } AssignedDevRegion;
 
+#define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
+
+#define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+
 typedef struct AssignedDevice {
     PCIDevice dev;
     PCIHostDevice host;
-    uint32_t use_iommu;
+    uint32_t features;
     int intpin;
     uint8_t debug_flags;
     AssignedDevRegion v_addrs[PCI_NUM_REGIONS - 1];
Signature · 1 line
1.7.1
author
#2 lore raw

PATCH 2/4 pci-assign: Allow to disable MSI perference for host IRQ

In reply to Jan Kiszka

Some devices (e.g. the ath9k) claim to support MSI but actually do not work when this is enabled. We must not blindly switch such devices to MSI but rather provide the user a way to pass control back to the guest driver. This can be done by turning the new property "prefer_msi" off (default remains on).

---
 hw/device-assignment.c |    5 ++++-
 hw/device-assignment.h |    2 ++
 2 files changed, 6 insertions(+), 1 deletions(-)
hw/device-assignment.c +4 -1
index 349e864..73d8afd 100644
@@ -964,7 +964,8 @@ static int assign_irq(AssignedDevice *dev)
     }
 
     assigned_irq_data.flags = KVM_DEV_IRQ_GUEST_INTX;
-    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
+    if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK &&
+        dev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
         assigned_irq_data.flags |= KVM_DEV_IRQ_HOST_MSI;
     else
         assigned_irq_data.flags |= KVM_DEV_IRQ_HOST_INTX;
@@ -1541,6 +1542,8 @@ static PCIDeviceInfo assign_info = {
         DEFINE_PROP_BIT("iommu", AssignedDevice, features,
                         ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
         DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
+        DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
+                        ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
         DEFINE_PROP_END_OF_LIST(),
     },
 };
hw/device-assignment.h +2 -0
index 4eb5835..56b7076 100644
@@ -75,8 +75,10 @@ typedef struct {
 } AssignedDevRegion;
 
 #define ASSIGNED_DEVICE_USE_IOMMU_BIT	0
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT	1
 
 #define ASSIGNED_DEVICE_USE_IOMMU_MASK	(1 << ASSIGNED_DEVICE_USE_IOMMU_BIT)
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK	(1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
 
 typedef struct AssignedDevice {
     PCIDevice dev;
Signature · 1 line
1.7.1
#3 lore raw

Re: [PATCH 2/4] pci-assign: Allow to disable MSI perference for host IRQ

Quoted 27 lines · Jan Kiszka

Some devices (e.g. the ath9k) claim to support MSI but actually do not work when this is enabled. We must not blindly switch such devices to MSI but rather provide the user a way to pass control back to the guest driver. This can be done by turning the new property "prefer_msi" off (default remains on).

---
 hw/device-assignment.c |    5 ++++-
 hw/device-assignment.h |    2 ++
 2 files changed, 6 insertions(+), 1 deletions(-)
hw/device-assignment.c +2 -1
index 349e864..73d8afd 100644
@@ -964,7 +964,8 @@ static int assign_irq(AssignedDevice *dev)
     }
 
     assigned_irq_data.flags = KVM_DEV_IRQ_GUEST_INTX;
-    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
+    if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK &&
+        dev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
         assigned_irq_data.flags |= KVM_DEV_IRQ_HOST_MSI;
     else
         assigned_irq_data.flags |= KVM_DEV_IRQ_HOST_INTX;

hw/device-assignment.c static PCIDeviceInfo assign_info = { · line 1546

Quoted 5 lines
hw/device-assignment.c +2 -0
@@ -1541,6 +1542,8 @@ static PCIDeviceInfo assign_info = {
         DEFINE_PROP_BIT("iommu", AssignedDevice, features,
                         ASSIGNED_DEVICE_USE_IOMMU_BIT, true),
         DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
+        DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
+                        ASSIGNED_DEVICE_PREFER_MSI_BIT, true),

Suggest to define the feature bit properties next to each other.

Quoted 3 lines
hw/device-assignment.c +0 -0
         DEFINE_PROP_END_OF_LIST(),
     },
 };

[...]

author
#4 lore raw

PATCH 3/4 pci-assign: Issue warning when running w/o IOMMU

In reply to Jan Kiszka
---
 hw/device-assignment.c |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
hw/device-assignment.c +6 -0
index 73d8afd..d8c565c 100644
@@ -913,6 +913,12 @@ static int assign_device(AssignedDevice *dev)
 #else
     dev->features &= ~ASSIGNED_DEVICE_USE_IOMMU_MASK;
 #endif
+    if (!(dev->features & ASSIGNED_DEVICE_USE_IOMMU_MASK)) {
+        fprintf(stderr,
+                "WARNING: Assigning a device without IOMMU protection can "
+                "cause host memory corruption if the device issues DMA write "
+                "requests!\n");
+    }
 
     r = kvm_assign_pci_device(kvm_context, &assigned_dev_data);
     if (r < 0) {
Signature · 1 line
1.7.1
author
#5 lore raw

PATCH 4/4 pci-assign: Remove broken -pcidevice and pci_add host

In reply to Jan Kiszka

These qemu-kvm-only interfaces were broken by b560a9ab9b, but no one complained loud enough to get them fixed again. As we have properly working "-device pci-assign"/"device_add" and we won't push this upstream anyway, there is likely no point in restoring the interface.

CC Markus Armbruster <[email protected]>
CC Daniel P. Berrange <[email protected]>
---
 hmp-commands.hx        |    2 +-
 hw/device-assignment.c |   63 ------------------------------------------------
 hw/device-assignment.h |    7 -----
 hw/ipf.c               |    6 ----
 hw/pc.c                |    6 ----
 hw/pci-hotplug.c       |   22 ----------------
 qemu-options.hx        |    5 ----
 vl.c                   |   14 ----------
 8 files changed, 1 insertions(+), 124 deletions(-)
hmp-commands.hx +1 -1
index 9406496..ba6de28 100644
@@ -806,7 +806,7 @@ ETEXI
     {
         .name       = "pci_add",
         .args_type  = "pci_addr:s,type:s,opts:s?",
-        .params     = "auto|[[<domain>:]<bus>:]<slot> nic|storage|host [[vlan=n][,macaddr=addr][,model=type]] [file=file][,if=type][,bus=nr]... [host=02:00.0[,name=string][,dma=none]",
+        .params     = "auto|[[<domain>:]<bus>:]<slot> nic|storage [[vlan=n][,macaddr=addr][,model=type]] [file=file][,if=type][,bus=nr]...",
         .help       = "hot-add PCI device",
         .mhandler.cmd = pci_device_hot_add,
     },
hw/device-assignment.c +0 -63

69 lines not on this page · raw message

hw/device-assignment.h +0 -7
index 56b7076..c94a730 100644
@@ -114,13 +114,6 @@ typedef struct AssignedDevice {
     QLIST_ENTRY(AssignedDevice) next;
 } AssignedDevice;
 
-QemuOpts *add_assigned_device(const char *arg);
-void add_assigned_devices(PCIBus *bus, const char **devices, int n_devices);
 void assigned_dev_update_irqs(void);
 
-#define MAX_DEV_ASSIGN_CMDLINE 8
-
-extern const char *assigned_devices[MAX_DEV_ASSIGN_CMDLINE];
-extern int assigned_devices_index;
-
 #endif              /* __DEVICE_ASSIGNMENT_H__ */
hw/ipf.c +0 -6
index 21cff72..10c21eb 100644
@@ -635,12 +635,6 @@ static void ipf_init1(ram_addr_t ram_size,
 	    unit_id++;
 	}
     }
-
-#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
-    if (kvm_enabled())
-	add_assigned_devices(pci_bus, assigned_devices, assigned_devices_index);
-#endif /* CONFIG_KVM_DEVICE_ASSIGNMENT */
-
 }
 
 static void ipf_init_pci(ram_addr_t ram_size,
hw/pc.c +0 -6
index b624873..3bf3862 100644
@@ -1105,10 +1105,4 @@ void pc_pci_device_init(PCIBus *pci_bus)
 
         extboot_init(info->bdrv);
     }
-
-#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
-    if (kvm_enabled()) {
-        add_assigned_devices(pci_bus, assigned_devices, assigned_devices_index);
-    }
-#endif /* CONFIG_KVM_DEVICE_ASSIGNMENT */
 }
hw/pci-hotplug.c +0 -22
index b370b9b..a2d4bb2 100644
@@ -230,24 +230,6 @@ static PCIDevice *qemu_pci_hot_add_storage(Monitor *mon,
     return dev;
 }
 
-#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
-static PCIDevice *qemu_pci_hot_assign_device(Monitor *mon,
-                                             const char *devaddr,
-                                             const char *opts_str)
-{
-    QemuOpts *opts;
-    DeviceState *dev;
-
-    opts = add_assigned_device(opts_str);
-    if (opts == NULL) {
-        monitor_printf(mon, "Error adding device; check syntax\n");
-        return NULL;
-    }
-    dev = qdev_device_add(opts);
-    return DO_UPCAST(PCIDevice, qdev, dev);
-}
-#endif /* CONFIG_KVM_DEVICE_ASSIGNMENT */
-
 void pci_device_hot_add(Monitor *mon, const QDict *qdict)
 {
     PCIDevice *dev = NULL;
@@ -271,10 +253,6 @@ void pci_device_hot_add(Monitor *mon, const QDict *qdict)
         dev = qemu_pci_hot_add_nic(mon, pci_addr, opts);
     } else if (strcmp(type, "storage") == 0) {
         dev = qemu_pci_hot_add_storage(mon, pci_addr, opts);
-#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
-    } else if (strcmp(type, "host") == 0) {
-        dev = qemu_pci_hot_assign_device(mon, pci_addr, opts);
-#endif /* CONFIG_KVM_DEVICE_ASSIGNMENT */
     } else {
         monitor_printf(mon, "invalid type: %s\n", type);
     }
qemu-options.hx +0 -5
index 1c9a1e5..699b653 100644
@@ -2278,11 +2278,6 @@ DEF("no-kvm-pit-reinjection", 0, QEMU_OPTION_no_kvm_pit_reinjection,
     "-no-kvm-pit-reinjection\n"
     "                disable KVM kernel mode PIT interrupt reinjection\n",
     QEMU_ARCH_I386)
-DEF("pcidevice", HAS_ARG, QEMU_OPTION_pcidevice,
-    "-pcidevice host=[seg:]bus:dev.func[,dma=none][,name=string]\n"
-    "                expose a PCI device to the guest OS\n"
-    "                dma=none: don't perform any dma translations (default is to use an iommu)\n"
-    "                'string' is used in log output\n", QEMU_ARCH_I386)
 DEF("enable-nesting", 0, QEMU_OPTION_enable_nesting,
     "-enable-nesting enable support for running a VM inside the VM (AMD only)\n", QEMU_ARCH_I386)
 DEF("nvram", HAS_ARG, QEMU_OPTION_nvram,
vl.c +0 -14
index 42617c2..b5da216 100644
@@ -204,8 +204,6 @@ int win2k_install_hack = 0;
 int rtc_td_hack = 0;
 int usb_enabled = 0;
 int singlestep = 0;
-const char *assigned_devices[MAX_DEV_ASSIGN_CMDLINE];
-int assigned_devices_index;
 int smp_cpus = 1;
 int max_cpus = 0;
 int smp_cores = 1;
@@ -1885,8 +1883,6 @@ int main(int argc, char **argv, char **envp)
         node_cpumask[i] = 0;
     }
 
-    assigned_devices_index = 0;
-
     nb_numa_nodes = 0;
     nb_nics = 0;
 
@@ -2502,16 +2498,6 @@ int main(int argc, char **argv, char **envp)
 		break;
 	    }
 #endif
-#if defined(TARGET_I386) || defined(TARGET_X86_64) || defined(TARGET_IA64) || defined(__linux__)
-            case QEMU_OPTION_pcidevice:
-		if (assigned_devices_index >= MAX_DEV_ASSIGN_CMDLINE) {
-                    fprintf(stderr, "Too many assigned devices\n");
-                    exit(1);
-		}
-		assigned_devices[assigned_devices_index] = optarg;
-		assigned_devices_index++;
-                break;
-#endif
             case QEMU_OPTION_usb:
                 usb_enabled = 1;
                 break;
Signature · 1 line
1.7.1
#6 lore raw

Re: [PATCH 4/4] pci-assign: Remove broken -pcidevice and pci_add host

Quoted 4 lines · Jan Kiszka

These qemu-kvm-only interfaces were broken by b560a9ab9b, but no one complained loud enough to get them fixed again. As we have properly working "-device pci-assign"/"device_add" and we won't push this upstream anyway, there is likely no point in restoring the interface.

Agree. Dan wrote re libvirt:

As of libvirt >= 0.8.1 & QEMU >= 0.12.x we use switched to using -device for everything. Older libvirt versions had rather broken checking for PCI device topology, so I think it is fine to require libvirt >= 0.8.1 for latest QEMU releases if users want PCI dev assignment. Thus -pcidevice and pci_add can both be killed from our POV.

Note that this patch keeps pci_add nic|storage around. Unlikely to make it upstream.

author
#7 lore raw

Re: [PATCH 4/4] pci-assign: Remove broken -pcidevice and pci_add host

Quoted 16 lines · Markus Armbruster
Quoted 4 lines · Jan Kiszka

These qemu-kvm-only interfaces were broken by b560a9ab9b, but no one complained loud enough to get them fixed again. As we have properly working "-device pci-assign"/"device_add" and we won't push this upstream anyway, there is likely no point in restoring the interface.

Agree. Dan wrote re libvirt:

As of libvirt >= 0.8.1 & QEMU >= 0.12.x we use switched to using -device for everything. Older libvirt versions had rather broken checking for PCI device topology, so I think it is fine to require libvirt >= 0.8.1 for latest QEMU releases if users want PCI dev assignment. Thus -pcidevice and pci_add can both be killed from our POV.

Quoted 2 lines

Note that this patch keeps pci_add nic|storage around. Unlikely to make it upstream.

pci_add is already in upstream.

Jan

Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux
#8 lore raw

Re: [PATCH 4/4] pci-assign: Remove broken -pcidevice and pci_add host

Quoted 17 lines · Jan Kiszka
Quoted 16 lines · Markus Armbruster
Quoted 4 lines · Jan Kiszka

These qemu-kvm-only interfaces were broken by b560a9ab9b, but no one complained loud enough to get them fixed again. As we have properly working "-device pci-assign"/"device_add" and we won't push this upstream anyway, there is likely no point in restoring the interface.

Agree. Dan wrote re libvirt:

As of libvirt >= 0.8.1 & QEMU >= 0.12.x we use switched to using -device for everything. Older libvirt versions had rather broken checking for PCI device topology, so I think it is fine to require libvirt >= 0.8.1 for latest QEMU releases if users want PCI dev assignment. Thus -pcidevice and pci_add can both be killed from our POV.

Quoted 4 lines
Quoted 2 lines

Note that this patch keeps pci_add nic|storage around. Unlikely to make it upstream.

pci_add is already in upstream.

You're right. Can't think straight today.

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

#12 lore raw

Re: [PATCH 5/4] pci-assign: Use PCI-2.3-based shared legacy interrupts by default

Quoted 42 lines · Jan Kiszka
Quoted 32 lines · Michael S. Tsirkin
Quoted 31 lines · Jan Kiszka

From: Jan Kiszka <[email protected]>

Make use of the new KVM feature that allows legacy interrupt sharing for PCI-2.3-compliant devices. Exclusive mode (with IRQ masking at interrupt controller level) can be re-enabled via disabling the property "pci_2_3".

---
 hw/device-assignment.c |    7 +++++++
 hw/device-assignment.h |    2 ++
 2 files changed, 9 insertions(+), 0 deletions(-)
hw/device-assignment.c +5 -0
index 0bbf9d9..5c7e958 100644
@@ -970,6 +970,11 @@ static int assign_irq(AssignedDevice *dev)
     }
 
     assigned_irq_data.flags = KVM_DEV_IRQ_GUEST_INTX;
+#ifdef KVM_CAP_PCI_2_3
+    if (dev->features & ASSIGNED_DEVICE_USE_PCI_2_3_MASK) {
+        assigned_irq_data.flags |= KVM_DEV_IRQ_PCI_2_3;
+    }
+#endif /* KVM_CAP_PCI_2_3 */
     if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK &&
         dev->cap.available & ASSIGNED_DEVICE_CAP_MSI)
         assigned_irq_data.flags |= KVM_DEV_IRQ_HOST_MSI;
@@ -1550,6 +1555,8 @@ static PCIDeviceInfo assign_info = {
         DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
Quoted 8 lines

hw/device-assignment.c static PCIDeviceInfo assign_info = { · line 1559

Quoted 4 lines
hw/device-assignment.c +2 -0
         DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
                         ASSIGNED_DEVICE_PREFER_MSI_BIT, true),
+        DEFINE_PROP_BIT("pci_2_3", AssignedDevice, features,
+                        ASSIGNED_DEVICE_USE_PCI_2_3_BIT, true),

I'd prefer some other name that hints this is a kvm feature. Reason is, we might want to have pci_2_3 to mean that INTx is supported *on the guest side*.

Quoted 5 lines

I did not get that use case yet, but I could call it "host_pci_2_3" if preferred. Or provide a better name.

Jan

Sounds good.

Signature · 1 line
MST
#13 lore raw
In reply to Jan Kiszka
Quoted 9 lines · Jan Kiszka

Mostly usability fixes, but also a patch to allow using devices that have problems with MSIs.

Jan Kiszka (4):
pci-assign: Convert iommu property to booleam
pci-assign: Allow to disable MSI perference for host IRQ
pci-assign: Issue warning when running w/o IOMMU
pci-assign: Remove broken -pcidevice and pci_add host

Quoted 9 lines
 hmp-commands.hx        |    2 +-
 hw/device-assignment.c |   81 ++++++++---------------------------------------
 hw/device-assignment.h |   15 ++++-----
 hw/ipf.c               |    6 ---
 hw/pc.c                |    6 ---
 hw/pci-hotplug.c       |   22 -------------
 qemu-options.hx        |    5 ---
 vl.c                   |   14 --------
 8 files changed, 22 insertions(+), 129 deletions(-)

Applied, thanks.