Skip to contentSkip to conversation

s390x: support virtio-mem-pci

PATCH Superseded David Hildenbrand 3 participants 8 messages Jan 27, 2025

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover s390x: support virtio-mem-pci

This is based-on [1], which adds MSI-X support to virtio-balloon-pci, but can be applied independently.

Turns out it is fairly easy to get virtio-mem-pci running on s390x. We only have to add MSI-X support to virtio-mem-pci, and wire-up the (un)plugging in the machine.

Tried some simple stuff (hotplug/hotunplug/resize/reboot), and all seems to be working as expected.

The kernel in the VM needs both, CONFIG_VIRTIO_PCI and CONFIG_VIRTIO_MEM for it to work.

[1] https://lkml.kernel.org/r/[email protected]

Cc Eduardo Habkost <[email protected]>
Cc Marcel Apfelbaum <[email protected]>
Cc Richard Henderson <[email protected]>
Cc David Hildenbrand <[email protected]>
Cc Ilya Leoshkevich <[email protected]>
Cc Halil Pasic <[email protected]>
Cc Christian Borntraeger <[email protected]>
Cc Eric Farman <[email protected]>
Cc Thomas Huth <[email protected]>
Cc "Michael S. Tsirkin" <[email protected]>
Cc Cornelia Huck <[email protected]>
Cc Boris Fiuczynski <[email protected]>
Cc Michal Privoznik <[email protected]>

David Hildenbrand (2):
virtio-mem-pci: Allow setting nvectors, so we can use MSI-X s390x/s390-virtio-ccw: support plugging PCI-based virtio memory
devices

 hw/core/machine.c          |  3 +++
 hw/s390x/s390-virtio-ccw.c | 20 ++++++++++++++------
 hw/virtio/virtio-mem-pci.c | 12 ++++++++++++
 3 files changed, 29 insertions(+), 6 deletions(-)
Signature · 1 line
2.48.1
#1 lore raw

PATCH 1/2 virtio-mem-pci: Allow setting nvectors, so we can use MSI-X

Let's do it similar as virtio-balloon-pci. With this change, we can use virtio-mem-pci on s390x, although plugging will still fail until properly wired up in the machine.

---
 hw/core/machine.c          |  3 +++
 hw/virtio/virtio-mem-pci.c | 12 ++++++++++++
 2 files changed, 15 insertions(+)
hw/core/machine.c +3 -0
index 8f396ef803..5e1d9e12ef 100644
@@ -41,6 +41,9 @@ GlobalProperty hw_compat_9_2[] = {
     { "virtio-balloon-pci", "vectors", "0" },
     { "virtio-balloon-pci-transitional", "vectors", "0" },
     { "virtio-balloon-pci-non-transitional", "vectors", "0" },
+    { "virtio-mem-pci", "vectors", "0" },
+    { "virtio-mem-pci-transitional", "vectors", "0" },
+    { "virtio-mem-pci-non-transitional", "vectors", "0" },
 };
 const size_t hw_compat_9_2_len = G_N_ELEMENTS(hw_compat_9_2);
 
hw/virtio/virtio-mem-pci.c +12 -0
index 1b4e9a3284..6cc5f0fd3b 100644
@@ -22,6 +22,10 @@ static void virtio_mem_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
     VirtIOMEMPCI *mem_pci = VIRTIO_MEM_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&mem_pci->vdev);
 
+    if (vpci_dev->nvectors == DEV_NVECTORS_UNSPECIFIED) {
+        vpci_dev->nvectors = 2;
+    }
+
     virtio_pci_force_virtio_1(vpci_dev);
     qdev_realize(vdev, BUS(&vpci_dev->bus), errp);
 }
@@ -152,6 +156,13 @@ static void virtio_mem_pci_set_requested_size(Object *obj, Visitor *v,
     object_property_set(OBJECT(&pci_mem->vdev), name, v, errp);
 }
 
+static const Property virtio_mem_pci_class_properties[] = {
+    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
+                       DEV_NVECTORS_UNSPECIFIED),
+};
+
 static void virtio_mem_pci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -164,6 +175,7 @@ static void virtio_mem_pci_class_init(ObjectClass *klass, void *data)
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     pcidev_k->revision = VIRTIO_PCI_ABI_VERSION;
     pcidev_k->class_id = PCI_CLASS_OTHERS;
+    device_class_set_props(dc, virtio_mem_pci_class_properties);
 
     mdc->get_addr = virtio_mem_pci_get_addr;
     mdc->set_addr = virtio_mem_pci_set_addr;
Signature · 1 line
2.48.1
#2 lore raw

Re: [PATCH v1 1/2] virtio-mem-pci: Allow setting nvectors, so we can use MSI-X

Quoted 15 lines · David Hildenbrand

Let's do it similar as virtio-balloon-pci. With this change, we can use virtio-mem-pci on s390x, although plugging will still fail until properly wired up in the machine.

---
  hw/core/machine.c          |  3 +++
  hw/virtio/virtio-mem-pci.c | 12 ++++++++++++
  2 files changed, 15 insertions(+)
hw/core/machine.c +0 -0
index 8f396ef803..5e1d9e12ef 100644

hw/core/machine.c GlobalProperty hw_compat_9_2[] = { · line 46

Quoted 6 lines
hw/core/machine.c +3 -0
@@ -41,6 +41,9 @@ GlobalProperty hw_compat_9_2[] = {
      { "virtio-balloon-pci", "vectors", "0" },
      { "virtio-balloon-pci-transitional", "vectors", "0" },
      { "virtio-balloon-pci-non-transitional", "vectors", "0" },
+    { "virtio-mem-pci", "vectors", "0" },
+    { "virtio-mem-pci-transitional", "vectors", "0" },
+    { "virtio-mem-pci-non-transitional", "vectors", "0" },

Hi!

Do we need the entries fo the -transitional and -non-transitional devices? Since virtio-mem is rather new, I would expect that those should not exist here? E.g. virtio-balloon-pci.c has:

.transitional_name = "virtio-balloon-pci-transitional",
.non_transitional_name = "virtio-balloon-pci-non-transitional",

but there are no such entries in virtio-mem-pci.c ... ?

Thomas

Quoted 40 lines
hw/core/machine.c +0 -0
  };
  const size_t hw_compat_9_2_len = G_N_ELEMENTS(hw_compat_9_2);
 
hw/virtio/virtio-mem-pci.c +12 -0
index 1b4e9a3284..6cc5f0fd3b 100644
@@ -22,6 +22,10 @@ static void virtio_mem_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
      VirtIOMEMPCI *mem_pci = VIRTIO_MEM_PCI(vpci_dev);
      DeviceState *vdev = DEVICE(&mem_pci->vdev);
 
+    if (vpci_dev->nvectors == DEV_NVECTORS_UNSPECIFIED) {
+        vpci_dev->nvectors = 2;
+    }
+
      virtio_pci_force_virtio_1(vpci_dev);
      qdev_realize(vdev, BUS(&vpci_dev->bus), errp);
  }
@@ -152,6 +156,13 @@ static void virtio_mem_pci_set_requested_size(Object *obj, Visitor *v,
      object_property_set(OBJECT(&pci_mem->vdev), name, v, errp);
  }
 
+static const Property virtio_mem_pci_class_properties[] = {
+    DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
+                       DEV_NVECTORS_UNSPECIFIED),
+};
+
  static void virtio_mem_pci_class_init(ObjectClass *klass, void *data)
  {
      DeviceClass *dc = DEVICE_CLASS(klass);
@@ -164,6 +175,7 @@ static void virtio_mem_pci_class_init(ObjectClass *klass, void *data)
      set_bit(DEVICE_CATEGORY_MISC, dc->categories);
      pcidev_k->revision = VIRTIO_PCI_ABI_VERSION;
      pcidev_k->class_id = PCI_CLASS_OTHERS;
+    device_class_set_props(dc, virtio_mem_pci_class_properties);
 
      mdc->get_addr = virtio_mem_pci_get_addr;
      mdc->set_addr = virtio_mem_pci_set_addr;
#3 lore raw

Re: [PATCH v1 1/2] virtio-mem-pci: Allow setting nvectors, so we can use MSI-X

Quoted 25 lines · Thomas Huth
Quoted 15 lines · David Hildenbrand

Let's do it similar as virtio-balloon-pci. With this change, we can use virtio-mem-pci on s390x, although plugging will still fail until properly wired up in the machine.

---
   hw/core/machine.c          |  3 +++
   hw/virtio/virtio-mem-pci.c | 12 ++++++++++++
   2 files changed, 15 insertions(+)
hw/core/machine.c +0 -0
index 8f396ef803..5e1d9e12ef 100644

hw/core/machine.c GlobalProperty hw_compat_9_2[] = { · line 46

Quoted 6 lines
hw/core/machine.c +3 -0
@@ -41,6 +41,9 @@ GlobalProperty hw_compat_9_2[] = {
       { "virtio-balloon-pci", "vectors", "0" },
       { "virtio-balloon-pci-transitional", "vectors", "0" },
       { "virtio-balloon-pci-non-transitional", "vectors", "0" },
+    { "virtio-mem-pci", "vectors", "0" },
+    { "virtio-mem-pci-transitional", "vectors", "0" },
+    { "virtio-mem-pci-non-transitional", "vectors", "0" },
   Hi!
Quoted 8 lines

Do we need the entries fo the -transitional and -non-transitional devices? Since virtio-mem is rather new, I would expect that those should not exist here? E.g. virtio-balloon-pci.c has:

.transitional_name = "virtio-balloon-pci-transitional",
.non_transitional_name = "virtio-balloon-pci-non-transitional",

but there are no such entries in virtio-mem-pci.c ... ?

Oh, very good point. Yes, I think we can just drop this legacy stuff.

Thanks!

Signature · 3 lines
Cheers,

David / dhildenb
#4 lore raw

PATCH 2/2 s390x/s390-virtio-ccw: Support plugging PCI-based virtio memory devices

In reply to David Hildenbrand

Let's just wire it up, unlocking virtio-mem-pci support on s390x.

---
 hw/s390x/s390-virtio-ccw.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
hw/s390x/s390-virtio-ccw.c +14 -6
index 3af613d4e9..71f3443a53 100644
@@ -554,8 +554,7 @@ static void s390_machine_device_pre_plug(HotplugHandler *hotplug_dev,
     if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
         virtio_ccw_md_pre_plug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
-        error_setg(errp,
-                   "PCI-attached virtio based memory devices not supported");
+        virtio_md_pci_pre_plug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp);
     }
 }
 
@@ -566,7 +565,8 @@ static void s390_machine_device_plug(HotplugHandler *hotplug_dev,
 
     if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
         s390_cpu_plug(hotplug_dev, dev, errp);
-    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW) ||
+               object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
         /*
          * At this point, the device is realized and set all memdevs mapped, so
          * qemu_maxrampagesize() will pick up the page sizes of these memdevs
@@ -580,7 +580,11 @@ static void s390_machine_device_plug(HotplugHandler *hotplug_dev,
                        " initial memory");
             return;
         }
-        virtio_ccw_md_plug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
+        if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
+            virtio_ccw_md_plug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
+        } else {
+            virtio_md_pci_plug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp);
+        }
     }
 }
 
@@ -589,10 +593,12 @@ static void s390_machine_device_unplug_request(HotplugHandler *hotplug_dev,
 {
     if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
         error_setg(errp, "CPU hot unplug not supported on this machine");
-        return;
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
         virtio_ccw_md_unplug_request(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev),
                                      errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
+        virtio_md_pci_unplug_request(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev),
+                                     errp);
     }
 }
 
@@ -601,7 +607,9 @@ static void s390_machine_device_unplug(HotplugHandler *hotplug_dev,
 {
     if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
         virtio_ccw_md_unplug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
-     }
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
+        virtio_md_pci_unplug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp);
+    }
  }
 
 static CpuInstanceProperties s390_cpu_index_to_props(MachineState *ms,
Signature · 1 line
2.48.1
Reviewed-by on v1 2/2
#5 lore raw

Re: [PATCH v1 2/2] s390x/s390-virtio-ccw: Support plugging PCI-based virtio memory devices

Quoted 6 lines · David Hildenbrand

Let's just wire it up, unlocking virtio-mem-pci support on s390x.

---
  hw/s390x/s390-virtio-ccw.c | 20 ++++++++++++++------
  1 file changed, 14 insertions(+), 6 deletions(-)

Thanks for tackling this!

#6 lore raw

Re: [PATCH v1 2/2] s390x/s390-virtio-ccw: Support plugging PCI-based virtio memory devices

In reply to David Hildenbrand
Quoted 45 lines · David Hildenbrand

Let's just wire it up, unlocking virtio-mem-pci support on s390x.

---
 hw/s390x/s390-virtio-ccw.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
hw/s390x/s390-virtio-ccw.c +8 -4
index 3af613d4e9..71f3443a53 100644
@@ -554,8 +554,7 @@ static void s390_machine_device_pre_plug(HotplugHandler *hotplug_dev,
     if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
         virtio_ccw_md_pre_plug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
-        error_setg(errp,
-                   "PCI-attached virtio based memory devices not supported");
+        virtio_md_pci_pre_plug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp);
     }
 }
 
@@ -566,7 +565,8 @@ static void s390_machine_device_plug(HotplugHandler *hotplug_dev,
 
     if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
         s390_cpu_plug(hotplug_dev, dev, errp);
-    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW) ||
+               object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
         /*
          * At this point, the device is realized and set all memdevs mapped, so
          * qemu_maxrampagesize() will pick up the page sizes of these memdevs
@@ -580,7 +580,11 @@ static void s390_machine_device_plug(HotplugHandler *hotplug_dev,
                        " initial memory");
             return;
         }
-        virtio_ccw_md_plug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
+        if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
+            virtio_ccw_md_plug(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev), errp);
+        } else {
+            virtio_md_pci_plug(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev), errp);
+        }
     }
 }
 

hw/s390x/s390-virtio-ccw.c static void s390_machine_device_unplug_request(HotplugHandler *hotplug_dev, · line 592

Quoted 4 lines
hw/s390x/s390-virtio-ccw.c +0 -1
@@ -589,10 +593,12 @@ static void s390_machine_device_unplug_request(HotplugHandler *hotplug_dev,
 {
     if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
         error_setg(errp, "CPU hot unplug not supported on this machine");
-        return;

This looks suspicious. Do we really want to continue executing the rest of the function in this case (though there's nothing to execute currently)? OTOH, the statement can be put back should it be ever needed.

hw/s390x/s390-virtio-ccw.c static void s390_machine_device_unplug_request(HotplugHandler *hotplug_dev, · line 603

Quoted 8 lines
hw/s390x/s390-virtio-ccw.c +3 -0
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_CCW)) {
         virtio_ccw_md_unplug_request(VIRTIO_MD_CCW(dev), MACHINE(hotplug_dev),
                                      errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_MD_PCI)) {
+        virtio_md_pci_unplug_request(VIRTIO_MD_PCI(dev), MACHINE(hotplug_dev),
+                                     errp);
     }
 }

Michal

#7 lore raw

Re: [PATCH v1 2/2] s390x/s390-virtio-ccw: Support plugging PCI-based virtio memory devices

Quoted 11 lines

hw/s390x/s390-virtio-ccw.c static void s390_machine_device_unplug_request(HotplugHandler *hotplug_dev, · line 592

Quoted 7 lines
hw/s390x/s390-virtio-ccw.c +0 -1
@@ -589,10 +593,12 @@ static void s390_machine_device_unplug_request(HotplugHandler *hotplug_dev,
  {
      if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
          error_setg(errp, "CPU hot unplug not supported on this machine");
-        return;

This looks suspicious. Do we really want to continue executing the rest of the function in this case (though there's nothing to execute currently)?

These functions are usually a big switch-case statement, so the "return" is actually superfluous.

Note that this is already what we do with TYPE_CPU in s390_machine_device_plug().

Thanks!

Signature · 3 lines
Cheers,

David / dhildenb