Skip to contentSkip to conversation

VIRTIO-IOMMU probe request support and MSI bypass on ARM

PATCH Superseded Eric Auger 3 participants 15 messages Jun 11, 2020 – Jun 23, 2020

v3 of 7 Superseded Latest is v7 →

lore
Share & tools
lore

Eric Auger

author
#0 lore raw

v3 cover VIRTIO-IOMMU probe request support and MSI bypass on ARM

By default the virtio-iommu translates MSI transactions. This behavior is inherited from ARM SMMU. However the virt machine code knows where the MSI doorbells are, so we can easily declare those regions as VIRTIO_IOMMU_RESV_MEM_T_MSI. With that setting the guest iommu subsystem will not need to map MSIs. This setup will simplify the VFIO integration.

In this series, the ITS or GICV2M doorbells are declared as HW MSI regions to be bypassed by the VIRTIO-IOMMU.

This also paves the way to the x86 integration where the MSI region, [0xFEE00000,0xFEEFFFFF], will be exposed by the q35 machine. However this will be handled in a separate series when not-DT support gets resolved.

Best Regards

Eric

This series can be found at:
https://github.com/eauger/qemu/tree/v5.0.0-virtio-iommu-msi-bypass-v3

History:

v2 -> v3:
- Introduce VIRT_MSI_CTRL_NONE in VirtMSIControllerType
- do not fill the remainder of the probe buffer

v1 -> v2:
- check which MSI controller is in use and advertise the corresponding MSI doorbell
- managed for both ITS and GICv2M
- various fixes spotted by Peter and Jean-Philippe, see individual logs

v1: Most of those patches were respinned from
[PATCH for-5.0 v11 00/20] VIRTIO-IOMMU device
except the last one which is new

Eric Auger (5):
qdev: Introduce DEFINE_PROP_RESERVED_REGION
virtio-iommu: Implement RESV_MEM probe request
virtio-iommu: Handle reserved regions in the translation process
virtio-iommu-pci: Add array of Interval properties hw/arm/virt: Let the virtio-iommu bypass MSIs

 include/exec/memory.h            |   6 ++
 include/hw/arm/virt.h            |   7 ++
 include/hw/qdev-properties.h     |   3 +
 include/hw/virtio/virtio-iommu.h |   2 +
 include/qemu/typedefs.h          |   1 +
 hw/arm/virt.c                    |  18 +++++
 hw/core/qdev-properties.c        |  89 +++++++++++++++++++++++++
 hw/virtio/virtio-iommu-pci.c     |   3 +
 hw/virtio/virtio-iommu.c         | 111 +++++++++++++++++++++++++++++--
 hw/virtio/trace-events           |   1 +
 10 files changed, 237 insertions(+), 4 deletions(-)
Signature · 1 line
2.20.1
author
#1 lore raw

PATCH v3 1/5 qdev: Introduce DEFINE_PROP_RESERVED_REGION

Introduce a new property defining a reserved region:
<low address>, <high address>, <type>.

This will be used to encode reserved IOVA regions.

For instance, in virtio-iommu use case, reserved IOVA regions will be passed by the machine code to the virtio-iommu-pci device (an array of those). The type of the reserved region will match the virtio_iommu_probe_resv_mem subtype value:
- VIRTIO_IOMMU_RESV_MEM_T_RESERVED (0)
- VIRTIO_IOMMU_RESV_MEM_T_MSI (1)

on PC/Q35 machine, this will be used to inform the virtio-iommu-pci device it should bypass the MSI region. The reserved region will be: 0xfee00000, 0xfeefffff, 1.

On ARM, we can declare the ITS MSI doorbell as an MSI region to prevent MSIs from being mapped on guest side.

---

v11 -> v12:
- rename into DEFINE_PROP_RESERVED_REGION
- do not use g_strsplit anymore, use endptr instead
- remove 0x references

---
 include/exec/memory.h        |  6 +++
 include/hw/qdev-properties.h |  3 ++
 include/qemu/typedefs.h      |  1 +
 hw/core/qdev-properties.c    | 89 ++++++++++++++++++++++++++++++++++++
 4 files changed, 99 insertions(+)
include/exec/memory.h +6 -0
index 3e00cdbbfa..3ee8224fa7 100644
@@ -57,6 +57,12 @@ struct MemoryRegionMmio {
     CPUWriteMemoryFunc *write[3];
 };
 
+struct ReservedRegion {
+    hwaddr low;
+    hwaddr high;
+    unsigned int type;
+};
+
 typedef struct IOMMUTLBEntry IOMMUTLBEntry;
 
 /* See address_space_translate: bit 0 is read, bit 1 is write.  */
include/hw/qdev-properties.h +3 -0
index f161604fb6..03bf850a7e 100644
@@ -19,6 +19,7 @@ extern const PropertyInfo qdev_prop_string;
 extern const PropertyInfo qdev_prop_chr;
 extern const PropertyInfo qdev_prop_tpm;
 extern const PropertyInfo qdev_prop_macaddr;
+extern const PropertyInfo qdev_prop_reserved_region;
 extern const PropertyInfo qdev_prop_on_off_auto;
 extern const PropertyInfo qdev_prop_multifd_compression;
 extern const PropertyInfo qdev_prop_losttickpolicy;
@@ -183,6 +184,8 @@ extern const PropertyInfo qdev_prop_pcie_link_width;
     DEFINE_PROP(_n, _s, _f, qdev_prop_drive_iothread, BlockBackend *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
     DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_RESERVED_REGION(_n, _s, _f)         \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_reserved_region, ReservedRegion)
 #define DEFINE_PROP_ON_OFF_AUTO(_n, _s, _f, _d) \
     DEFINE_PROP_SIGNED(_n, _s, _f, _d, qdev_prop_on_off_auto, OnOffAuto)
 #define DEFINE_PROP_MULTIFD_COMPRESSION(_n, _s, _f, _d) \
include/qemu/typedefs.h +1 -0
index ecf3cde26c..85c4f891f4 100644
@@ -59,6 +59,7 @@ typedef struct ISABus ISABus;
 typedef struct ISADevice ISADevice;
 typedef struct IsaDma IsaDma;
 typedef struct MACAddr MACAddr;
+typedef struct ReservedRegion ReservedRegion;
 typedef struct MachineClass MachineClass;
 typedef struct MachineState MachineState;
 typedef struct MemoryListener MemoryListener;
hw/core/qdev-properties.c +89 -0

101 lines not on this page · raw message

Signature · 1 line
2.20.1
Reviewed-by on v3 1/5
#2 lore raw

Re: [PATCH v3 1/5] qdev: Introduce DEFINE_PROP_RESERVED_REGION

Quoted 109 lines · Eric Auger

Introduce a new property defining a reserved region:
<low address>, <high address>, <type>.

This will be used to encode reserved IOVA regions.

For instance, in virtio-iommu use case, reserved IOVA regions will be passed by the machine code to the virtio-iommu-pci device (an array of those). The type of the reserved region will match the virtio_iommu_probe_resv_mem subtype value:
- VIRTIO_IOMMU_RESV_MEM_T_RESERVED (0)
- VIRTIO_IOMMU_RESV_MEM_T_MSI (1)

on PC/Q35 machine, this will be used to inform the virtio-iommu-pci device it should bypass the MSI region. The reserved region will be: 0xfee00000, 0xfeefffff, 1.

On ARM, we can declare the ITS MSI doorbell as an MSI region to prevent MSIs from being mapped on guest side.

---

v11 -> v12:
- rename into DEFINE_PROP_RESERVED_REGION
- do not use g_strsplit anymore, use endptr instead
- remove 0x references

---
 include/exec/memory.h        |  6 +++
 include/hw/qdev-properties.h |  3 ++
 include/qemu/typedefs.h      |  1 +
 hw/core/qdev-properties.c    | 89 ++++++++++++++++++++++++++++++++++++
 4 files changed, 99 insertions(+)
include/exec/memory.h +6 -0
index 3e00cdbbfa..3ee8224fa7 100644
@@ -57,6 +57,12 @@ struct MemoryRegionMmio {
     CPUWriteMemoryFunc *write[3];
 };
 
+struct ReservedRegion {
+    hwaddr low;
+    hwaddr high;
+    unsigned int type;
+};
+
 typedef struct IOMMUTLBEntry IOMMUTLBEntry;
 
 /* See address_space_translate: bit 0 is read, bit 1 is write.  */
include/hw/qdev-properties.h +3 -0
index f161604fb6..03bf850a7e 100644
@@ -19,6 +19,7 @@ extern const PropertyInfo qdev_prop_string;
 extern const PropertyInfo qdev_prop_chr;
 extern const PropertyInfo qdev_prop_tpm;
 extern const PropertyInfo qdev_prop_macaddr;
+extern const PropertyInfo qdev_prop_reserved_region;
 extern const PropertyInfo qdev_prop_on_off_auto;
 extern const PropertyInfo qdev_prop_multifd_compression;
 extern const PropertyInfo qdev_prop_losttickpolicy;
@@ -183,6 +184,8 @@ extern const PropertyInfo qdev_prop_pcie_link_width;
     DEFINE_PROP(_n, _s, _f, qdev_prop_drive_iothread, BlockBackend *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
     DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_RESERVED_REGION(_n, _s, _f)         \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_reserved_region, ReservedRegion)
 #define DEFINE_PROP_ON_OFF_AUTO(_n, _s, _f, _d) \
     DEFINE_PROP_SIGNED(_n, _s, _f, _d, qdev_prop_on_off_auto, OnOffAuto)
 #define DEFINE_PROP_MULTIFD_COMPRESSION(_n, _s, _f, _d) \
include/qemu/typedefs.h +1 -0
index ecf3cde26c..85c4f891f4 100644
@@ -59,6 +59,7 @@ typedef struct ISABus ISABus;
 typedef struct ISADevice ISADevice;
 typedef struct IsaDma IsaDma;
 typedef struct MACAddr MACAddr;
+typedef struct ReservedRegion ReservedRegion;
 typedef struct MachineClass MachineClass;
 typedef struct MachineState MachineState;
 typedef struct MemoryListener MemoryListener;
hw/core/qdev-properties.c +10 -0
index cc924815da..15b84adbee 100644
@@ -14,6 +14,7 @@
 #include "qapi/visitor.h"
 #include "chardev/char.h"
 #include "qemu/uuid.h"
+#include "qemu/cutils.h"
 
 void qdev_prop_set_after_realize(DeviceState *dev, const char *name,
                                   Error **errp)
@@ -577,6 +578,94 @@ const PropertyInfo qdev_prop_macaddr = {
     .set   = set_mac,
 };
 
+/* --- Reserved Region --- */
+
+/*
+ * accepted syntax version:
+ *   <low address>,<high address>,<type>
+ *   where low/high addresses are uint64_t in hexadecimal
+ *   and type is an unsigned integer in decimal
+ */
+static void get_reserved_region(Object *obj, Visitor *v, const char *name,

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 599

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+                                void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    ReservedRegion *rr = qdev_get_prop_ptr(dev, prop);
+    char buffer[64];
+    char *p = buffer;
+
+    snprintf(buffer, sizeof(buffer), "0x%"PRIx64",0x%"PRIx64",%u",
+             rr->low, rr->high, rr->type);

Matches existing practice in other getters. Nevertheless, I'd suggest something like

       n = snprintf(buffer, sizeof(buffer), ...);
       assert(n < sizeof(buffer);
Quoted 21 lines
hw/core/qdev-properties.c +21 -0
+
+    visit_type_str(v, name, &p, errp);
+}
+
+static void set_reserved_region(Object *obj, Visitor *v, const char *name,
+                                void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    ReservedRegion *rr = qdev_get_prop_ptr(dev, prop);
+    Error *local_err = NULL;
+    const char *endptr;
+    char *str;
+    int ret;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_str(v, name, &str, &local_err);

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 630

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    ret = qemu_strtou64(str, &endptr, 16, &rr->low);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region low addr");
+        error_append_hint(errp,
+                          "should be an address in hexadecimal\n");

Comes out like this:

qemu-system-x86_64: -device ...: Failed to decode reserved region low addr should be an address in hexadecimal

I'd capitalize the other way, to get

qemu-system-x86_64: -device ...: failed to decode reserved region low addr Should be an address in hexadecimal

Note: output is made up; I failed at figuring out how to use the new property. An example in PATCH 4's commit message might help.

Since the error message fails to mention @name, the user is left guessing unless "-device ..." contains just one reserved region parameter.

What about something like this:

error_setg(errp, "start address of reserved region '%s'"
" must be a hexadecimal integer", name);

No need to mess around with error_append_hint() then.

Same for the other error messages.

Quoted 12 lines
hw/core/qdev-properties.c +12 -0
+        goto out;
+    }
+    if (*endptr != ',') {
+        goto separator_error;
+    }
+
+    ret = qemu_strtou64(endptr + 1, &endptr, 16, &rr->high);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region high addr");
+        error_append_hint(errp,
+                          "should be an address in hexadecimal\n");
+        goto out;

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 652

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+    }
+    if (*endptr != ',') {
+        goto separator_error;
+    }
+
+    ret = qemu_strtoui(endptr + 1, &endptr, 10, &rr->type);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region type");
+        error_append_hint(errp, "should be an unsigned integer in decimal\n");
+    }

I dimly remember discussing the wisdom of numeric type here, dig, dig, ..., aha:

Subject: Re: [PATCH for-5.0 v11 12/20] qapi: Introduce DEFINE_PROP_INTERVAL
Date: Fri, 13 Dec 2019 11:03:02 +0100
Message-ID: <[email protected]>

>> So the "label" part of "<low address>,<high address>,label" is a number? > yes it is.
>>
>> Is a number appropriate for your use case, or would an enum be better? > I think a number is OK. There might be other types of reserved regions > in the future. Also if we want to allow somebody else to reuse that > property in another context, I would rather leave it open?

I'd prioritize the user interface over possible reuse (which might never happen). Mind, I'm not telling you using numbers is a bad user interface. In general, enums are nicer, but I don't know enough about this particular case.

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 656

Quoted 4 lines
hw/core/qdev-properties.c +4 -0
+    goto out;
+
+separator_error:
+    error_setg(errp, "reserved region fields must be separated with commas");

I'm not sure de-duplicating this error message is worth the extra goto.

Quoted 5 lines
hw/core/qdev-properties.c +5 -0
+out:
+    g_free(str);
+    return;
+}
+

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 671

Quoted 10 lines
hw/core/qdev-properties.c +7 -0
+const PropertyInfo qdev_prop_reserved_region = {
+    .name  = "reserved_region",
+    .description = "Reserved Region, example: 0xFEE00000,0xFEEFFFFF,0",
+    .get   = get_reserved_region,
+    .set   = set_reserved_region,
+};
+
 /* --- on/off/auto --- */
 
 const PropertyInfo qdev_prop_on_off_auto = {

Can't find anything that's actually wrong, so

author
#3 lore raw

Re: [PATCH v3 1/5] qdev: Introduce DEFINE_PROP_RESERVED_REGION

Hi Markus,

Quoted 125 lines · Markus Armbruster
Quoted 109 lines · Eric Auger

Introduce a new property defining a reserved region:
<low address>, <high address>, <type>.

This will be used to encode reserved IOVA regions.

For instance, in virtio-iommu use case, reserved IOVA regions will be passed by the machine code to the virtio-iommu-pci device (an array of those). The type of the reserved region will match the virtio_iommu_probe_resv_mem subtype value:
- VIRTIO_IOMMU_RESV_MEM_T_RESERVED (0)
- VIRTIO_IOMMU_RESV_MEM_T_MSI (1)

on PC/Q35 machine, this will be used to inform the virtio-iommu-pci device it should bypass the MSI region. The reserved region will be: 0xfee00000, 0xfeefffff, 1.

On ARM, we can declare the ITS MSI doorbell as an MSI region to prevent MSIs from being mapped on guest side.

---

v11 -> v12:
- rename into DEFINE_PROP_RESERVED_REGION
- do not use g_strsplit anymore, use endptr instead
- remove 0x references

---
 include/exec/memory.h        |  6 +++
 include/hw/qdev-properties.h |  3 ++
 include/qemu/typedefs.h      |  1 +
 hw/core/qdev-properties.c    | 89 ++++++++++++++++++++++++++++++++++++
 4 files changed, 99 insertions(+)
include/exec/memory.h +6 -0
index 3e00cdbbfa..3ee8224fa7 100644
@@ -57,6 +57,12 @@ struct MemoryRegionMmio {
     CPUWriteMemoryFunc *write[3];
 };
 
+struct ReservedRegion {
+    hwaddr low;
+    hwaddr high;
+    unsigned int type;
+};
+
 typedef struct IOMMUTLBEntry IOMMUTLBEntry;
 
 /* See address_space_translate: bit 0 is read, bit 1 is write.  */
include/hw/qdev-properties.h +3 -0
index f161604fb6..03bf850a7e 100644
@@ -19,6 +19,7 @@ extern const PropertyInfo qdev_prop_string;
 extern const PropertyInfo qdev_prop_chr;
 extern const PropertyInfo qdev_prop_tpm;
 extern const PropertyInfo qdev_prop_macaddr;
+extern const PropertyInfo qdev_prop_reserved_region;
 extern const PropertyInfo qdev_prop_on_off_auto;
 extern const PropertyInfo qdev_prop_multifd_compression;
 extern const PropertyInfo qdev_prop_losttickpolicy;
@@ -183,6 +184,8 @@ extern const PropertyInfo qdev_prop_pcie_link_width;
     DEFINE_PROP(_n, _s, _f, qdev_prop_drive_iothread, BlockBackend *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
     DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_RESERVED_REGION(_n, _s, _f)         \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_reserved_region, ReservedRegion)
 #define DEFINE_PROP_ON_OFF_AUTO(_n, _s, _f, _d) \
     DEFINE_PROP_SIGNED(_n, _s, _f, _d, qdev_prop_on_off_auto, OnOffAuto)
 #define DEFINE_PROP_MULTIFD_COMPRESSION(_n, _s, _f, _d) \
include/qemu/typedefs.h +1 -0
index ecf3cde26c..85c4f891f4 100644
@@ -59,6 +59,7 @@ typedef struct ISABus ISABus;
 typedef struct ISADevice ISADevice;
 typedef struct IsaDma IsaDma;
 typedef struct MACAddr MACAddr;
+typedef struct ReservedRegion ReservedRegion;
 typedef struct MachineClass MachineClass;
 typedef struct MachineState MachineState;
 typedef struct MemoryListener MemoryListener;
hw/core/qdev-properties.c +10 -0
index cc924815da..15b84adbee 100644
@@ -14,6 +14,7 @@
 #include "qapi/visitor.h"
 #include "chardev/char.h"
 #include "qemu/uuid.h"
+#include "qemu/cutils.h"
 
 void qdev_prop_set_after_realize(DeviceState *dev, const char *name,
                                   Error **errp)
@@ -577,6 +578,94 @@ const PropertyInfo qdev_prop_macaddr = {
     .set   = set_mac,
 };
 
+/* --- Reserved Region --- */
+
+/*
+ * accepted syntax version:
+ *   <low address>,<high address>,<type>
+ *   where low/high addresses are uint64_t in hexadecimal
+ *   and type is an unsigned integer in decimal
+ */
+static void get_reserved_region(Object *obj, Visitor *v, const char *name,

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 599

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+                                void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    ReservedRegion *rr = qdev_get_prop_ptr(dev, prop);
+    char buffer[64];
+    char *p = buffer;
+
+    snprintf(buffer, sizeof(buffer), "0x%"PRIx64",0x%"PRIx64",%u",
+             rr->low, rr->high, rr->type);

Matches existing practice in other getters. Nevertheless, I'd suggest something like

Quoted 2 lines
       n = snprintf(buffer, sizeof(buffer), ...);
       assert(n < sizeof(buffer);

OK

Quoted 38 lines
Quoted 22 lines
hw/core/qdev-properties.c +21 -0
+
+    visit_type_str(v, name, &p, errp);
+}
+
+static void set_reserved_region(Object *obj, Visitor *v, const char *name,
+                                void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    ReservedRegion *rr = qdev_get_prop_ptr(dev, prop);
+    Error *local_err = NULL;
+    const char *endptr;
+    char *str;
+    int ret;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_str(v, name, &str, &local_err);

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 630

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    ret = qemu_strtou64(str, &endptr, 16, &rr->low);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region low addr");
+        error_append_hint(errp,
+                          "should be an address in hexadecimal\n");

Comes out like this:

qemu-system-x86_64: -device ...: Failed to decode reserved region low addr should be an address in hexadecimal

Quoted 7 lines

I'd capitalize the other way, to get

qemu-system-x86_64: -device ...: failed to decode reserved region low addr Should be an address in hexadecimal

Note: output is made up; I failed at figuring out how to use the new property. An example in PATCH 4's commit message might help.

OK I will add one example. In practice in the virtio-iommu case the property is not really meant to be passed by the end-user but should be set by the machine code. However I have just tested from the cmd line and it looks using commas as separators is a bad idea because it collides with ',' separating properties. So if you're OK I will change the comma into ':'.

Quoted 10 lines

Since the error message fails to mention @name, the user is left guessing unless "-device ..." contains just one reserved region parameter.

What about something like this:

error_setg(errp, "start address of reserved region '%s'"
" must be a hexadecimal integer", name);

Sure. Given the fact the property is named reserved-region[n], this may be simplified into
error_setg(errp, "start address of '%s'"
" must be a hexadecimal integer",

Quoted 3 lines
                     name);

No need to mess around with error_append_hint() then.

OK

Quoted 41 lines

Same for the other error messages.

Quoted 12 lines
hw/core/qdev-properties.c +12 -0
+        goto out;
+    }
+    if (*endptr != ',') {
+        goto separator_error;
+    }
+
+    ret = qemu_strtou64(endptr + 1, &endptr, 16, &rr->high);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region high addr");
+        error_append_hint(errp,
+                          "should be an address in hexadecimal\n");
+        goto out;

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 652

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+    }
+    if (*endptr != ',') {
+        goto separator_error;
+    }
+
+    ret = qemu_strtoui(endptr + 1, &endptr, 10, &rr->type);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region type");
+        error_append_hint(errp, "should be an unsigned integer in decimal\n");
+    }

I dimly remember discussing the wisdom of numeric type here, dig, dig, ..., aha:

Subject: Re: [PATCH for-5.0 v11 12/20] qapi: Introduce DEFINE_PROP_INTERVAL
Date: Fri, 13 Dec 2019 11:03:02 +0100
Message-ID: <[email protected]>

>> So the "label" part of "<low address>,<high address>,label" is a number? > yes it is.
>>
>> Is a number appropriate for your use case, or would an enum be better? > I think a number is OK. There might be other types of reserved regions > in the future. Also if we want to allow somebody else to reuse that > property in another context, I would rather leave it open?

Quoted 4 lines

I'd prioritize the user interface over possible reuse (which might never happen). Mind, I'm not telling you using numbers is a bad user interface. In general, enums are nicer, but I don't know enough about this particular case.

Yep I remember too ;-) I left as it was because I think this property could be used for other use cases.

Quoted 22 lines

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 656

Quoted 5 lines
hw/core/qdev-properties.c +4 -0
+    goto out;
+
+separator_error:
+    error_setg(errp, "reserved region fields must be separated with commas");

I'm not sure de-duplicating this error message is worth the extra goto.

Quoted 14 lines
hw/core/qdev-properties.c +12 -0
+out:
+    g_free(str);
+    return;
+}
+
+const PropertyInfo qdev_prop_reserved_region = {
+    .name  = "reserved_region",
+    .description = "Reserved Region, example: 0xFEE00000,0xFEEFFFFF,0",
+    .get   = get_reserved_region,
+    .set   = set_reserved_region,
+};
+
 /* --- on/off/auto --- */
 
Quoted 5 lines

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 671

Quoted 1 line
hw/core/qdev-properties.c +0 -0
 const PropertyInfo qdev_prop_on_off_auto = {

Can't find anything that's actually wrong, so

Thanks

Eric

Quoted 1 line
#4 lore raw

Re: [PATCH v3 1/5] qdev: Introduce DEFINE_PROP_RESERVED_REGION

Quoted 25 lines · Auger Eric

Hi Markus,

Quoted 22 lines · Markus Armbruster
Quoted 20 lines · Eric Auger

Introduce a new property defining a reserved region:
<low address>, <high address>, <type>.

This will be used to encode reserved IOVA regions.

For instance, in virtio-iommu use case, reserved IOVA regions will be passed by the machine code to the virtio-iommu-pci device (an array of those). The type of the reserved region will match the virtio_iommu_probe_resv_mem subtype value:
- VIRTIO_IOMMU_RESV_MEM_T_RESERVED (0)
- VIRTIO_IOMMU_RESV_MEM_T_MSI (1)

on PC/Q35 machine, this will be used to inform the virtio-iommu-pci device it should bypass the MSI region. The reserved region will be: 0xfee00000, 0xfeefffff, 1.

On ARM, we can declare the ITS MSI doorbell as an MSI region to prevent MSIs from being mapped on guest side.

[...]

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
hw/core/qdev-properties.c +0 -0
index cc924815da..15b84adbee 100644

[...]

Quoted 38 lines
Quoted 38 lines
Quoted 17 lines
hw/core/qdev-properties.c +17 -0
+static void set_reserved_region(Object *obj, Visitor *v, const char *name,
+                                void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    ReservedRegion *rr = qdev_get_prop_ptr(dev, prop);
+    Error *local_err = NULL;
+    const char *endptr;
+    char *str;
+    int ret;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_str(v, name, &str, &local_err);

hw/core/qdev-properties.c const PropertyInfo qdev_prop_macaddr = { · line 630

Quoted 10 lines
hw/core/qdev-properties.c +10 -0
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    ret = qemu_strtou64(str, &endptr, 16, &rr->low);
+    if (ret) {
+        error_setg(errp, "Failed to decode reserved region low addr");
+        error_append_hint(errp,
+                          "should be an address in hexadecimal\n");

Comes out like this:

qemu-system-x86_64: -device ...: Failed to decode reserved region low addr should be an address in hexadecimal

I'd capitalize the other way, to get

qemu-system-x86_64: -device ...: failed to decode reserved region low addr Should be an address in hexadecimal

Quoted 8 lines
Quoted 2 lines

Note: output is made up; I failed at figuring out how to use the new property. An example in PATCH 4's commit message might help.

OK I will add one example. In practice in the virtio-iommu case the property is not really meant to be passed by the end-user but should be set by the machine code. However I have just tested from the cmd line and it looks using commas as separators is a bad idea because it collides with ',' separating properties. So if you're OK I will change the comma into ':'.

Please do.

[...]

#5 lore raw

Re: [PATCH v3 1/5] qdev: Introduce DEFINE_PROP_RESERVED_REGION

In reply to Auger Eric
Quoted 7 lines · Auger Eric

Hi Markus,

Quoted 4 lines · Markus Armbruster
Quoted 2 lines · Eric Auger

Introduce a new property defining a reserved region:
<low address>, <high address>, <type>.

[...]

Quoted 15 lines
Quoted 15 lines

I dimly remember discussing the wisdom of numeric type here, dig, dig, ..., aha:

Subject: Re: [PATCH for-5.0 v11 12/20] qapi: Introduce DEFINE_PROP_INTERVAL
Date: Fri, 13 Dec 2019 11:03:02 +0100
Message-ID: <[email protected]>

>> So the "label" part of "<low address>,<high address>,label" is a number? > yes it is.
>>
>> Is a number appropriate for your use case, or would an enum be better? > I think a number is OK. There might be other types of reserved regions > in the future. Also if we want to allow somebody else to reuse that > property in another context, I would rather leave it open?

Quoted 6 lines
Quoted 4 lines

I'd prioritize the user interface over possible reuse (which might never happen). Mind, I'm not telling you using numbers is a bad user interface. In general, enums are nicer, but I don't know enough about this particular case.

Yep I remember too ;-) I left as it was because I think this property could be used for other use cases.

YAGNI :)

A string would work, too, wouldn't it?

[...]

author
#6 lore raw

Re: [PATCH v3 1/5] qdev: Introduce DEFINE_PROP_RESERVED_REGION

Hi Markus,

Quoted 29 lines · Markus Armbruster
Quoted 7 lines · Auger Eric

Hi Markus,

Quoted 4 lines · Markus Armbruster
Quoted 2 lines · Eric Auger

Introduce a new property defining a reserved region:
<low address>, <high address>, <type>.

[...]

Quoted 19 lines
Quoted 19 lines

I dimly remember discussing the wisdom of numeric type here, dig, dig, ..., aha:

Subject: Re: [PATCH for-5.0 v11 12/20] qapi: Introduce DEFINE_PROP_INTERVAL
Date: Fri, 13 Dec 2019 11:03:02 +0100
Message-ID: <[email protected]>

>> So the "label" part of "<low address>,<high address>,label" is a number? > yes it is.
>>
>> Is a number appropriate for your use case, or would an enum be better? > I think a number is OK. There might be other types of reserved regions > in the future. Also if we want to allow somebody else to reuse that > property in another context, I would rather leave it open?

I'd prioritize the user interface over possible reuse (which might never happen). Mind, I'm not telling you using numbers is a bad user interface. In general, enums are nicer, but I don't know enough about this particular case.

Quoted 6 lines
Quoted 2 lines

Yep I remember too ;-) I left as it was because I think this property could be used for other use cases.

YAGNI :)

A string would work, too, wouldn't it?

:-)

Eric

Quoted 4 lines

[...]

author
#7 lore raw

PATCH v3 2/5 virtio-iommu: Implement RESV_MEM probe request

In reply to Eric Auger

This patch implements the PROBE request. At the moment, only THE RESV_MEM property is handled. The first goal is to report iommu wide reserved regions such as the MSI regions set by the machine code. On x86 this will be the IOAPIC MSI region, [0xFEE00000 - 0xFEEFFFFF], on ARM this may be the ITS doorbell.

In the future we may introduce per device reserved regions. This will be useful when protecting host assigned devices which may expose their own reserved regions

---

v2 -> v3:
- on probe, do not fill the reminder of the buffer with zeroes as the buffer was already zero initialized (Bharat)

v1 -> v2:
- move the unlock back to the same place
- remove the push label and factorize the code after the out label
- fix a bunch of cpu_to_leX according to the latest spec revision
- do not remove sizeof(last) from free space
- check the ep exists

---
 include/hw/virtio/virtio-iommu.h |  2 +
 hw/virtio/virtio-iommu.c         | 91 ++++++++++++++++++++++++++++++--
 hw/virtio/trace-events           |  1 +
 3 files changed, 90 insertions(+), 4 deletions(-)
include/hw/virtio/virtio-iommu.h +2 -0
index e653004d7c..49eb105cd8 100644
@@ -53,6 +53,8 @@ typedef struct VirtIOIOMMU {
     GHashTable *as_by_busptr;
     IOMMUPciBus *iommu_pcibus_by_bus_num[PCI_BUS_MAX];
     PCIBus *primary_bus;
+    ReservedRegion *reserved_regions;
+    uint32_t nb_reserved_regions;
     GTree *domains;
     QemuMutex mutex;
     GTree *endpoints;
hw/virtio/virtio-iommu.c +87 -4

144 lines not on this page · raw message

hw/virtio/trace-events +1 -0
index 6427a0047d..23109f69bb 100644
@@ -74,3 +74,4 @@ virtio_iommu_get_domain(uint32_t domain_id) "Alloc domain=%d"
 virtio_iommu_put_domain(uint32_t domain_id) "Free domain=%d"
 virtio_iommu_translate_out(uint64_t virt_addr, uint64_t phys_addr, uint32_t sid) "0x%"PRIx64" -> 0x%"PRIx64 " for sid=%d"
 virtio_iommu_report_fault(uint8_t reason, uint32_t flags, uint32_t endpoint, uint64_t addr) "FAULT reason=%d flags=%d endpoint=%d address =0x%"PRIx64
+virtio_iommu_fill_resv_property(uint32_t devid, uint8_t subtype, uint64_t start, uint64_t end) "dev= %d, type=%d start=0x%"PRIx64" end=0x%"PRIx64
Signature · 1 line
2.20.1
Reviewed-by on v3 2/5
#8 lore raw

Re: [PATCH v3 2/5] virtio-iommu: Implement RESV_MEM probe request

Quoted 7 lines · Eric Auger

This patch implements the PROBE request. At the moment, only THE RESV_MEM property is handled. The first goal is to report iommu wide reserved regions such as the MSI regions set by the machine code. On x86 this will be the IOAPIC MSI region, [0xFEE00000 - 0xFEEFFFFF], on ARM this may be the ITS doorbell.

Quoted 5 lines

In the future we may introduce per device reserved regions. This will be useful when protecting host assigned devices which may expose their own reserved regions

Looks good to me apart from one comment inconsistency (below)

Quoted 71 lines

---

v2 -> v3:
- on probe, do not fill the reminder of the buffer with zeroes as the buffer was already zero initialized (Bharat)

v1 -> v2:
- move the unlock back to the same place
- remove the push label and factorize the code after the out label
- fix a bunch of cpu_to_leX according to the latest spec revision
- do not remove sizeof(last) from free space
- check the ep exists

---
 include/hw/virtio/virtio-iommu.h |  2 +
 hw/virtio/virtio-iommu.c         | 91 ++++++++++++++++++++++++++++++--
 hw/virtio/trace-events           |  1 +
 3 files changed, 90 insertions(+), 4 deletions(-)
include/hw/virtio/virtio-iommu.h +2 -0
index e653004d7c..49eb105cd8 100644
@@ -53,6 +53,8 @@ typedef struct VirtIOIOMMU {
     GHashTable *as_by_busptr;
     IOMMUPciBus *iommu_pcibus_by_bus_num[PCI_BUS_MAX];
     PCIBus *primary_bus;
+    ReservedRegion *reserved_regions;
+    uint32_t nb_reserved_regions;
     GTree *domains;
     QemuMutex mutex;
     GTree *endpoints;
hw/virtio/virtio-iommu.c +24 -0
index 483883ec1d..be1527c1d6 100644
@@ -38,6 +38,7 @@
 
 /* Max size */
 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
+#define VIOMMU_PROBE_SIZE 512
 
 typedef struct VirtIOIOMMUDomain {
     uint32_t id;
@@ -378,6 +379,62 @@ static int virtio_iommu_unmap(VirtIOIOMMU *s,
     return ret;
 }
 
+static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
+                                               uint8_t *buf, size_t free)
+{
+    struct virtio_iommu_probe_resv_mem prop = {};
+    size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
+    int i;
+
+    total = size * s->nb_reserved_regions;
+
+    if (total > free) {
+        return -ENOSPC;
+    }
+
+    for (i = 0; i < s->nb_reserved_regions; i++) {
+        prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
+        prop.head.length = cpu_to_le16(length);
+        prop.subtype = s->reserved_regions[i].type;
+        prop.start = cpu_to_le64(s->reserved_regions[i].low);
+        prop.end = cpu_to_le64(s->reserved_regions[i].high);
+
+        memcpy(buf, &prop, size);
+
+        trace_virtio_iommu_fill_resv_property(ep, prop.subtype,

hw/virtio/virtio-iommu.c static int virtio_iommu_unmap(VirtIOIOMMU *s, · line 414

Quoted 10 lines
hw/virtio/virtio-iommu.c +10 -0
+                                              prop.start, prop.end);
+        buf += size;
+    }
+    return total;
+}
+
+/**
+ * virtio_iommu_probe - Fill the probe request buffer with
+ * the properties the device is able to return and add a NONE
+ * property at the end.

The NONE property doesn't exist anymore

Quoted 9 lines
hw/virtio/virtio-iommu.c +9 -0
+ */
+static int virtio_iommu_probe(VirtIOIOMMU *s,
+                              struct virtio_iommu_req_probe *req,
+                              uint8_t *buf)
+{
+    uint32_t ep_id = le32_to_cpu(req->endpoint);
+    size_t free = VIOMMU_PROBE_SIZE;
+    ssize_t count;
+

hw/virtio/virtio-iommu.c static int virtio_iommu_unmap(VirtIOIOMMU *s, · line 433

Quoted 10 lines
hw/virtio/virtio-iommu.c +10 -0
+    if (!virtio_iommu_mr(s, ep_id)) {
+        return VIRTIO_IOMMU_S_NOENT;
+    }
+
+    count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
+    if (count < 0) {
+        return VIRTIO_IOMMU_S_INVAL;
+    }
+    buf += count;
+    free -= count;

Nit: could be removed since we only support one property at the moment.

Quoted 99 lines
hw/virtio/virtio-iommu.c +34 -4
+
+    return VIRTIO_IOMMU_S_OK;
+}
+
 static int virtio_iommu_iov_to_req(struct iovec *iov,
                                    unsigned int iov_cnt,
                                    void *req, size_t req_sz)
@@ -407,15 +464,27 @@ virtio_iommu_handle_req(detach)
 virtio_iommu_handle_req(map)
 virtio_iommu_handle_req(unmap)
 
+static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
+                                     struct iovec *iov,
+                                     unsigned int iov_cnt,
+                                     uint8_t *buf)
+{
+    struct virtio_iommu_req_probe req;
+    int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
+
+    return ret ? ret : virtio_iommu_probe(s, &req, buf);
+}
+
 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
     struct virtio_iommu_req_head head;
     struct virtio_iommu_req_tail tail = {};
+    size_t output_size = sizeof(tail), sz;
     VirtQueueElement *elem;
     unsigned int iov_cnt;
     struct iovec *iov;
-    size_t sz;
+    void *buf = NULL;
 
     for (;;) {
         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
@@ -452,6 +521,17 @@ static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
         case VIRTIO_IOMMU_T_UNMAP:
             tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
             break;
+        case VIRTIO_IOMMU_T_PROBE:
+        {
+            struct virtio_iommu_req_tail *ptail;
+
+            output_size = s->config.probe_size + sizeof(tail);
+            buf = g_malloc0(output_size);
+
+            ptail = (struct virtio_iommu_req_tail *)
+                        (buf + s->config.probe_size);
+            ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
+        }
         default:
             tail.status = VIRTIO_IOMMU_S_UNSUPP;
         }
@@ -459,12 +539,13 @@ static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
 
 out:
         sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
-                          &tail, sizeof(tail));
-        assert(sz == sizeof(tail));
+                          buf ? buf : &tail, output_size);
+        assert(sz == output_size);
 
-        virtqueue_push(vq, elem, sizeof(tail));
+        virtqueue_push(vq, elem, sz);
         virtio_notify(vdev, vq);
         g_free(elem);
+        g_free(buf);
     }
 }
 
@@ -667,6 +748,7 @@ static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
     s->config.page_size_mask = TARGET_PAGE_MASK;
     s->config.input_range.end = -1UL;
     s->config.domain_range.end = 32;
+    s->config.probe_size = VIOMMU_PROBE_SIZE;
 
     virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
     virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
@@ -676,6 +758,7 @@ static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS);
     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
+    virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
 
     qemu_mutex_init(&s->mutex);
 
hw/virtio/trace-events +1 -0
index 6427a0047d..23109f69bb 100644
@@ -74,3 +74,4 @@ virtio_iommu_get_domain(uint32_t domain_id) "Alloc domain=%d"
 virtio_iommu_put_domain(uint32_t domain_id) "Free domain=%d"
 virtio_iommu_translate_out(uint64_t virt_addr, uint64_t phys_addr, uint32_t sid) "0x%"PRIx64" -> 0x%"PRIx64 " for sid=%d"
 virtio_iommu_report_fault(uint8_t reason, uint32_t flags, uint32_t endpoint, uint64_t addr) "FAULT reason=%d flags=%d endpoint=%d address =0x%"PRIx64
+virtio_iommu_fill_resv_property(uint32_t devid, uint8_t subtype, uint64_t start, uint64_t end) "dev= %d, type=%d start=0x%"PRIx64" end=0x%"PRIx64
Signature · 1 line
2.20.1
author
#9 lore raw

Re: [PATCH v3 2/5] virtio-iommu: Implement RESV_MEM probe request

Hi Jean-Philippe,

Quoted 8 lines · Jean-Philippe Brucker
Quoted 7 lines · Eric Auger

This patch implements the PROBE request. At the moment, only THE RESV_MEM property is handled. The first goal is to report iommu wide reserved regions such as the MSI regions set by the machine code. On x86 this will be the IOAPIC MSI region, [0xFEE00000 - 0xFEEFFFFF], on ARM this may be the ITS doorbell.

Quoted 7 lines
Quoted 5 lines

In the future we may introduce per device reserved regions. This will be useful when protecting host assigned devices which may expose their own reserved regions

Looks good to me apart from one comment inconsistency (below)

noted

Quoted 2 lines

Thanks!

Eric

Quoted 207 lines
Quoted 72 lines

---

v2 -> v3:
- on probe, do not fill the reminder of the buffer with zeroes as the buffer was already zero initialized (Bharat)

v1 -> v2:
- move the unlock back to the same place
- remove the push label and factorize the code after the out label
- fix a bunch of cpu_to_leX according to the latest spec revision
- do not remove sizeof(last) from free space
- check the ep exists

---
 include/hw/virtio/virtio-iommu.h |  2 +
 hw/virtio/virtio-iommu.c         | 91 ++++++++++++++++++++++++++++++--
 hw/virtio/trace-events           |  1 +
 3 files changed, 90 insertions(+), 4 deletions(-)
include/hw/virtio/virtio-iommu.h +2 -0
index e653004d7c..49eb105cd8 100644
@@ -53,6 +53,8 @@ typedef struct VirtIOIOMMU {
     GHashTable *as_by_busptr;
     IOMMUPciBus *iommu_pcibus_by_bus_num[PCI_BUS_MAX];
     PCIBus *primary_bus;
+    ReservedRegion *reserved_regions;
+    uint32_t nb_reserved_regions;
     GTree *domains;
     QemuMutex mutex;
     GTree *endpoints;
hw/virtio/virtio-iommu.c +24 -0
index 483883ec1d..be1527c1d6 100644
@@ -38,6 +38,7 @@
 
 /* Max size */
 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
+#define VIOMMU_PROBE_SIZE 512
 
 typedef struct VirtIOIOMMUDomain {
     uint32_t id;
@@ -378,6 +379,62 @@ static int virtio_iommu_unmap(VirtIOIOMMU *s,
     return ret;
 }
 
+static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
+                                               uint8_t *buf, size_t free)
+{
+    struct virtio_iommu_probe_resv_mem prop = {};
+    size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
+    int i;
+
+    total = size * s->nb_reserved_regions;
+
+    if (total > free) {
+        return -ENOSPC;
+    }
+
+    for (i = 0; i < s->nb_reserved_regions; i++) {
+        prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
+        prop.head.length = cpu_to_le16(length);
+        prop.subtype = s->reserved_regions[i].type;
+        prop.start = cpu_to_le64(s->reserved_regions[i].low);
+        prop.end = cpu_to_le64(s->reserved_regions[i].high);
+
+        memcpy(buf, &prop, size);
+
+        trace_virtio_iommu_fill_resv_property(ep, prop.subtype,

hw/virtio/virtio-iommu.c static int virtio_iommu_unmap(VirtIOIOMMU *s, · line 414

Quoted 10 lines
hw/virtio/virtio-iommu.c +10 -0
+                                              prop.start, prop.end);
+        buf += size;
+    }
+    return total;
+}
+
+/**
+ * virtio_iommu_probe - Fill the probe request buffer with
+ * the properties the device is able to return and add a NONE
+ * property at the end.

The NONE property doesn't exist anymore

Quoted 9 lines
hw/virtio/virtio-iommu.c +9 -0
+ */
+static int virtio_iommu_probe(VirtIOIOMMU *s,
+                              struct virtio_iommu_req_probe *req,
+                              uint8_t *buf)
+{
+    uint32_t ep_id = le32_to_cpu(req->endpoint);
+    size_t free = VIOMMU_PROBE_SIZE;
+    ssize_t count;
+

hw/virtio/virtio-iommu.c static int virtio_iommu_unmap(VirtIOIOMMU *s, · line 433

Quoted 10 lines
hw/virtio/virtio-iommu.c +10 -0
+    if (!virtio_iommu_mr(s, ep_id)) {
+        return VIRTIO_IOMMU_S_NOENT;
+    }
+
+    count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
+    if (count < 0) {
+        return VIRTIO_IOMMU_S_INVAL;
+    }
+    buf += count;
+    free -= count;

Nit: could be removed since we only support one property at the moment.

Quoted 99 lines
hw/virtio/virtio-iommu.c +34 -4
+
+    return VIRTIO_IOMMU_S_OK;
+}
+
 static int virtio_iommu_iov_to_req(struct iovec *iov,
                                    unsigned int iov_cnt,
                                    void *req, size_t req_sz)
@@ -407,15 +464,27 @@ virtio_iommu_handle_req(detach)
 virtio_iommu_handle_req(map)
 virtio_iommu_handle_req(unmap)
 
+static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
+                                     struct iovec *iov,
+                                     unsigned int iov_cnt,
+                                     uint8_t *buf)
+{
+    struct virtio_iommu_req_probe req;
+    int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
+
+    return ret ? ret : virtio_iommu_probe(s, &req, buf);
+}
+
 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
     struct virtio_iommu_req_head head;
     struct virtio_iommu_req_tail tail = {};
+    size_t output_size = sizeof(tail), sz;
     VirtQueueElement *elem;
     unsigned int iov_cnt;
     struct iovec *iov;
-    size_t sz;
+    void *buf = NULL;
 
     for (;;) {
         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
@@ -452,6 +521,17 @@ static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
         case VIRTIO_IOMMU_T_UNMAP:
             tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
             break;
+        case VIRTIO_IOMMU_T_PROBE:
+        {
+            struct virtio_iommu_req_tail *ptail;
+
+            output_size = s->config.probe_size + sizeof(tail);
+            buf = g_malloc0(output_size);
+
+            ptail = (struct virtio_iommu_req_tail *)
+                        (buf + s->config.probe_size);
+            ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
+        }
         default:
             tail.status = VIRTIO_IOMMU_S_UNSUPP;
         }
@@ -459,12 +539,13 @@ static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
 
 out:
         sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
-                          &tail, sizeof(tail));
-        assert(sz == sizeof(tail));
+                          buf ? buf : &tail, output_size);
+        assert(sz == output_size);
 
-        virtqueue_push(vq, elem, sizeof(tail));
+        virtqueue_push(vq, elem, sz);
         virtio_notify(vdev, vq);
         g_free(elem);
+        g_free(buf);
     }
 }
 
@@ -667,6 +748,7 @@ static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
     s->config.page_size_mask = TARGET_PAGE_MASK;
     s->config.input_range.end = -1UL;
     s->config.domain_range.end = 32;
+    s->config.probe_size = VIOMMU_PROBE_SIZE;
 
     virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
     virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
@@ -676,6 +758,7 @@ static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS);
     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
+    virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
 
     qemu_mutex_init(&s->mutex);
 
hw/virtio/trace-events +1 -0
index 6427a0047d..23109f69bb 100644
@@ -74,3 +74,4 @@ virtio_iommu_get_domain(uint32_t domain_id) "Alloc domain=%d"
 virtio_iommu_put_domain(uint32_t domain_id) "Free domain=%d"
 virtio_iommu_translate_out(uint64_t virt_addr, uint64_t phys_addr, uint32_t sid) "0x%"PRIx64" -> 0x%"PRIx64 " for sid=%d"
 virtio_iommu_report_fault(uint8_t reason, uint32_t flags, uint32_t endpoint, uint64_t addr) "FAULT reason=%d flags=%d endpoint=%d address =0x%"PRIx64
+virtio_iommu_fill_resv_property(uint32_t devid, uint8_t subtype, uint64_t start, uint64_t end) "dev= %d, type=%d start=0x%"PRIx64" end=0x%"PRIx64
Signature · 1 line
2.20.1
author
#10 lore raw

PATCH v3 3/5 virtio-iommu: Handle reserved regions in the translation process

In reply to Eric Auger

When translating an address we need to check if it belongs to a reserved virtual address range. If it does, there are 2 cases:

- it belongs to a RESERVED region: the guest should neither use this address in a MAP not instruct the end-point to DMA on them. We report an error

- It belongs to an MSI region: we bypass the translation.

---

v1 -> v2:
- use addr when testing addr belongs to the reserved region and use a block local variable

---
 hw/virtio/virtio-iommu.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
hw/virtio/virtio-iommu.c +20 -0
index be1527c1d6..03535a6a14 100644
@@ -604,6 +604,7 @@ static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
     uint32_t sid, flags;
     bool bypass_allowed;
     bool found;
+    int i;
 
     interval.low = addr;
     interval.high = addr + 1;
@@ -637,6 +638,25 @@ static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
         goto unlock;
     }
 
+    for (i = 0; i < s->nb_reserved_regions; i++) {
+        ReservedRegion *reg = &s->reserved_regions[i];
+
+        if (addr >= reg->low && addr <= reg->high) {
+            switch (reg->type) {
+            case VIRTIO_IOMMU_RESV_MEM_T_MSI:
+                entry.perm = flag;
+                break;
+            case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
+            default:
+                virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
+                                          VIRTIO_IOMMU_FAULT_F_ADDRESS,
+                                          sid, addr);
+                break;
+            }
+            goto unlock;
+        }
+    }
+
     if (!ep->domain) {
         if (!bypass_allowed) {
             error_report_once("%s %02x:%02x.%01x not attached to any domain",
Signature · 1 line
2.20.1
Reviewed-by on v3 3/5
#11 lore raw

Re: [PATCH v3 3/5] virtio-iommu: Handle reserved regions in the translation process

Quoted 12 lines · Eric Auger

When translating an address we need to check if it belongs to a reserved virtual address range. If it does, there are 2 cases:

- it belongs to a RESERVED region: the guest should neither use this address in a MAP not instruct the end-point to DMA on them. We report an error

- It belongs to an MSI region: we bypass the translation.

Quoted 50 lines

---

v1 -> v2:
- use addr when testing addr belongs to the reserved region and use a block local variable

---
 hw/virtio/virtio-iommu.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
hw/virtio/virtio-iommu.c +20 -0
index be1527c1d6..03535a6a14 100644
@@ -604,6 +604,7 @@ static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
     uint32_t sid, flags;
     bool bypass_allowed;
     bool found;
+    int i;
 
     interval.low = addr;
     interval.high = addr + 1;
@@ -637,6 +638,25 @@ static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
         goto unlock;
     }
 
+    for (i = 0; i < s->nb_reserved_regions; i++) {
+        ReservedRegion *reg = &s->reserved_regions[i];
+
+        if (addr >= reg->low && addr <= reg->high) {
+            switch (reg->type) {
+            case VIRTIO_IOMMU_RESV_MEM_T_MSI:
+                entry.perm = flag;
+                break;
+            case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
+            default:
+                virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
+                                          VIRTIO_IOMMU_FAULT_F_ADDRESS,
+                                          sid, addr);
+                break;
+            }
+            goto unlock;
+        }
+    }
+
     if (!ep->domain) {
         if (!bypass_allowed) {
             error_report_once("%s %02x:%02x.%01x not attached to any domain",
Signature · 1 line
2.20.1
author
#12 lore raw

PATCH v3 4/5 virtio-iommu-pci: Add array of Interval properties

In reply to Eric Auger

The machine may need to pass reserved regions to the virtio-iommu-pci device (such as the MSI window on x86). So let's add an array of Interval properties.

---

v12 -> v12:
- added Jean's R-b

---
 hw/virtio/virtio-iommu-pci.c | 3 +++
 1 file changed, 3 insertions(+)
hw/virtio/virtio-iommu-pci.c +3 -0
index 3dfbf55b47..44ae9ebc11 100644
@@ -33,6 +33,9 @@ struct VirtIOIOMMUPCI {
 
 static Property virtio_iommu_pci_properties[] = {
     DEFINE_PROP_UINT32("class", VirtIOPCIProxy, class_code, 0),
+    DEFINE_PROP_ARRAY("reserved-regions", VirtIOIOMMUPCI,
+                      vdev.nb_reserved_regions, vdev.reserved_regions,
+                      qdev_prop_reserved_region, ReservedRegion),
     DEFINE_PROP_END_OF_LIST(),
 };
 
Signature · 1 line
2.20.1
author
#13 lore raw

PATCH v3 5/5 hw/arm/virt: Let the virtio-iommu bypass MSIs

In reply to Eric Auger

At the moment the virtio-iommu translates MSI transactions. This behavior is inherited from ARM SMMU. The virt machine code knows where the guest MSI doorbells are so we can easily declare those regions as VIRTIO_IOMMU_RESV_MEM_T_MSI. With that setting the guest will not map MSIs through the IOMMU and those transactions will be simply bypassed.

Depending on which MSI controller is in use (ITS or GICV2M), we declare either:
- the ITS interrupt translation space (ITS_base + 0x10000), containing the GITS_TRANSLATOR or
- The GICV2M single frame, containing the MSI_SETSP_NS register.

---
v2 -> v3:
- Add a new value to VirtMSIControllerType

v1 -> v2:
- Test which MSI controller is instantiated
- If GICV2M is in use, declare its doorbell as an MSI doorbell too

---
 include/hw/arm/virt.h |  7 +++++++
 hw/arm/virt.c         | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+)
include/hw/arm/virt.h +7 -0
index 31878ddc72..a18b6b397b 100644
@@ -96,6 +96,12 @@ typedef enum VirtIOMMUType {
     VIRT_IOMMU_VIRTIO,
 } VirtIOMMUType;
 
+typedef enum VirtMSIControllerType {
+    VIRT_MSI_CTRL_NONE,
+    VIRT_MSI_CTRL_GICV2M,
+    VIRT_MSI_CTRL_ITS,
+} VirtMSIControllerType;
+
 typedef enum VirtGICType {
     VIRT_GIC_VERSION_MAX,
     VIRT_GIC_VERSION_HOST,
@@ -136,6 +142,7 @@ typedef struct {
     OnOffAuto acpi;
     VirtGICType gic_version;
     VirtIOMMUType iommu;
+    VirtMSIControllerType msi_controller;
     uint16_t virtio_iommu_bdf;
     struct arm_boot_info bootinfo;
     MemMapEntry *memmap;
hw/arm/virt.c +18 -0
index 37462a6f78..451b150459 100644
@@ -602,6 +602,7 @@ static void create_its(VirtMachineState *vms)
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, vms->memmap[VIRT_GIC_ITS].base);
 
     fdt_add_its_gic_node(vms);
+    vms->msi_controller = VIRT_MSI_CTRL_ITS;
 }
 
 static void create_v2m(VirtMachineState *vms)
@@ -622,6 +623,7 @@ static void create_v2m(VirtMachineState *vms)
     }
 
     fdt_add_v2m_gic_node(vms);
+    vms->msi_controller = VIRT_MSI_CTRL_GICV2M;
 }
 
 static void create_gic(VirtMachineState *vms)
@@ -2149,8 +2151,24 @@ out:
 static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev,
                                             DeviceState *dev, Error **errp)
 {
+    VirtMachineState *vms = VIRT_MACHINE(hotplug_dev);
+
     if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
         virt_memory_pre_plug(hotplug_dev, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI)) {
+        /* we declare a VIRTIO_IOMMU_RESV_MEM_T_MSI region */
+
+        if (vms->msi_controller == VIRT_MSI_CTRL_ITS) {
+            /* GITS_TRANSLATER page */
+            qdev_prop_set_uint32(dev, "len-reserved-regions", 1);
+            qdev_prop_set_string(dev, "reserved-regions[0]",
+                                 "0x8090000, 0x809FFFF, 1");
+        } else if (vms->msi_controller == VIRT_MSI_CTRL_GICV2M) {
+            /* MSI_SETSPI_NS page */
+            qdev_prop_set_uint32(dev, "len-reserved-regions", 1);
+            qdev_prop_set_string(dev, "reserved-regions[0]",
+                                 "0x8020000, 0x8020FFF, 1");
+        }
     }
 }
 
Signature · 1 line
2.20.1
Reviewed-by on v3 5/5
#14 lore raw

Re: [PATCH v3 5/5] hw/arm/virt: Let the virtio-iommu bypass MSIs

Quoted 7 lines · Eric Auger

At the moment the virtio-iommu translates MSI transactions. This behavior is inherited from ARM SMMU. The virt machine code knows where the guest MSI doorbells are so we can easily declare those regions as VIRTIO_IOMMU_RESV_MEM_T_MSI. With that setting the guest will not map MSIs through the IOMMU and those transactions will be simply bypassed.

Quoted 7 lines

Depending on which MSI controller is in use (ITS or GICV2M), we declare either:
- the ITS interrupt translation space (ITS_base + 0x10000), containing the GITS_TRANSLATOR or
- The GICV2M single frame, containing the MSI_SETSP_NS register.

Quoted 85 lines

---
v2 -> v3:
- Add a new value to VirtMSIControllerType

v1 -> v2:
- Test which MSI controller is instantiated
- If GICV2M is in use, declare its doorbell as an MSI doorbell too

---
 include/hw/arm/virt.h |  7 +++++++
 hw/arm/virt.c         | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+)
include/hw/arm/virt.h +7 -0
index 31878ddc72..a18b6b397b 100644
@@ -96,6 +96,12 @@ typedef enum VirtIOMMUType {
     VIRT_IOMMU_VIRTIO,
 } VirtIOMMUType;
 
+typedef enum VirtMSIControllerType {
+    VIRT_MSI_CTRL_NONE,
+    VIRT_MSI_CTRL_GICV2M,
+    VIRT_MSI_CTRL_ITS,
+} VirtMSIControllerType;
+
 typedef enum VirtGICType {
     VIRT_GIC_VERSION_MAX,
     VIRT_GIC_VERSION_HOST,
@@ -136,6 +142,7 @@ typedef struct {
     OnOffAuto acpi;
     VirtGICType gic_version;
     VirtIOMMUType iommu;
+    VirtMSIControllerType msi_controller;
     uint16_t virtio_iommu_bdf;
     struct arm_boot_info bootinfo;
     MemMapEntry *memmap;
hw/arm/virt.c +18 -0
index 37462a6f78..451b150459 100644
@@ -602,6 +602,7 @@ static void create_its(VirtMachineState *vms)
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, vms->memmap[VIRT_GIC_ITS].base);
 
     fdt_add_its_gic_node(vms);
+    vms->msi_controller = VIRT_MSI_CTRL_ITS;
 }
 
 static void create_v2m(VirtMachineState *vms)
@@ -622,6 +623,7 @@ static void create_v2m(VirtMachineState *vms)
     }
 
     fdt_add_v2m_gic_node(vms);
+    vms->msi_controller = VIRT_MSI_CTRL_GICV2M;
 }
 
 static void create_gic(VirtMachineState *vms)
@@ -2149,8 +2151,24 @@ out:
 static void virt_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev,
                                             DeviceState *dev, Error **errp)
 {
+    VirtMachineState *vms = VIRT_MACHINE(hotplug_dev);
+
     if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
         virt_memory_pre_plug(hotplug_dev, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI)) {
+        /* we declare a VIRTIO_IOMMU_RESV_MEM_T_MSI region */
+
+        if (vms->msi_controller == VIRT_MSI_CTRL_ITS) {
+            /* GITS_TRANSLATER page */
+            qdev_prop_set_uint32(dev, "len-reserved-regions", 1);
+            qdev_prop_set_string(dev, "reserved-regions[0]",
+                                 "0x8090000, 0x809FFFF, 1");
+        } else if (vms->msi_controller == VIRT_MSI_CTRL_GICV2M) {
+            /* MSI_SETSPI_NS page */
+            qdev_prop_set_uint32(dev, "len-reserved-regions", 1);
+            qdev_prop_set_string(dev, "reserved-regions[0]",
+                                 "0x8020000, 0x8020FFF, 1");
+        }
     }
 }
 
Signature · 1 line
2.20.1