Skip to contentSkip to conversation

hw/vfio/iommufd: Control dirty tracking for nesting parent HWPT

PATCH Applied Shameer Kolothum 3 participants 3 messages Apr 1 – Apr 3

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 hw/vfio/iommufd: Control dirty tracking for nesting parent HWPT

QEMU smmuv3 accel does not support live migration yet, so dirty tracking for the nesting parent HWPT is not useful.

Also, nested vIOMMU use cases can break on some platforms. For example, SMMUv3 with HTTU may advertise dirty tracking capability, but the kernel supports it only for stage-1. Requesting dirty tracking for a nesting parent HWPT (stage-2) can fail.

Add a vIOMMU flag to explicitly request dirty tracking for the nesting parent HWPT. For nested cases, dirty tracking is enabled only when requested by the vIOMMU.

Non-nested cases and Intel vIOMMU keep the existing behavior.

Fixes fc6dafb98cec ("hw/arm/smmuv3: Implement get_viommu_cap() callback")
---
 include/hw/core/iommu.h       |  2 ++
 include/hw/vfio/vfio-device.h |  1 +
 hw/i386/intel_iommu.c         |  7 +++++--
 hw/vfio/device.c              | 11 +++++++++++
 hw/vfio/iommufd.c             | 11 +++++++++--
 5 files changed, 28 insertions(+), 4 deletions(-)
include/hw/core/iommu.h +2 -0
index 86af315c15..cd59a367ce 100644
@@ -21,6 +21,8 @@ enum viommu_flags {
     /* vIOMMU needs nesting parent HWPT to create nested HWPT */
     VIOMMU_FLAG_WANT_NESTING_PARENT = BIT_ULL(0),
     VIOMMU_FLAG_PASID_SUPPORTED = BIT_ULL(1),
+    /* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */
+    VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2),
 };
 
 /* Host IOMMU quirks. Extracted from host IOMMU capabilities */
include/hw/vfio/vfio-device.h +1 -0
index 828a31c006..a95c5bf503 100644
@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer,
 void vfio_device_unprepare(VFIODevice *vbasedev);
 
 bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev);
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev);
 bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev,
                                                 uint32_t type, void *caps,
                                                 uint32_t size);
hw/i386/intel_iommu.c +5 -2
index 4924aa4375..5ec5ed82d7 100644
@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn)
 static uint64_t vtd_get_viommu_flags(void *opaque)
 {
     IntelIOMMUState *s = opaque;
-    uint64_t flags;
+    uint64_t flags = 0;
 
-    flags = s->fsts ? VIOMMU_FLAG_WANT_NESTING_PARENT : 0;
+    if (s->fsts) {
+        flags = VIOMMU_FLAG_WANT_NESTING_PARENT |
+                VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING;
+    }
 
     return flags;
 }
hw/vfio/device.c +11 -0
index 973fc35b59..8f7ae919a5 100644
@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev)
     vbasedev->bcontainer = NULL;
 }
 
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
+
+    if (vdev) {
+        return !!(pci_device_get_viommu_flags(PCI_DEVICE(vdev)) &
+                  VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING);
+    }
+    return false;
+}
+
 bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev)
 {
     VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
hw/vfio/iommufd.c +9 -2
index 131612eb83..231635cdc7 100644
@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
     ERRP_GUARD();
     IOMMUFDBackend *iommufd = vbasedev->iommufd;
     VFIOContainer *bcontainer = VFIO_IOMMU(container);
+    bool viommu_nesting, viommu_nesting_dirty;
     uint32_t type, flags = 0;
     uint64_t hw_caps;
     VendorCaps caps;
@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
         return false;
     }
 
+    viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev);
+    viommu_nesting_dirty =
+        vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev);
+
     if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) {
-        flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        if (!viommu_nesting || viommu_nesting_dirty) {
+            flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        }
     }
 
     /*
@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
      * force to create it so that it could be reused by vIOMMU to create
      * nested HWPT.
      */
-    if (vfio_device_get_viommu_flags_want_nesting(vbasedev)) {
+    if (viommu_nesting) {
         flags |= IOMMU_HWPT_ALLOC_NEST_PARENT;
 
         if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
Signature · 1 line
2.43.0
Reviewed-by on v1 1/1
#1 lore raw
Quoted 14 lines

-----Original Message-----
From: Shameer Kolothum <[email protected]>
Subject: [PATCH] hw/vfio/iommufd: Control dirty tracking for nesting parent HWPT

QEMU smmuv3 accel does not support live migration yet, so dirty tracking for the nesting parent HWPT is not useful.

Also, nested vIOMMU use cases can break on some platforms. For example, SMMUv3 with HTTU may advertise dirty tracking capability, but the kernel supports it only for stage-1. Requesting dirty tracking for a nesting parent HWPT (stage-2) can fail.

Quoted 8 lines

Add a vIOMMU flag to explicitly request dirty tracking for the nesting parent HWPT. For nested cases, dirty tracking is enabled only when requested by the vIOMMU.

Non-nested cases and Intel vIOMMU keep the existing behavior.

Fixes fc6dafb98cec ("hw/arm/smmuv3: Implement get_viommu_cap() callback")

Thanks
Zhenzhong

Quoted 119 lines

---

include/hw/core/iommu.h       |  2 ++
include/hw/vfio/vfio-device.h |  1 +
hw/i386/intel_iommu.c         |  7 +++++--
hw/vfio/device.c              | 11 +++++++++++
hw/vfio/iommufd.c             | 11 +++++++++--
5 files changed, 28 insertions(+), 4 deletions(-)
include/hw/core/iommu.h +2 -0
index 86af315c15..cd59a367ce 100644
@@ -21,6 +21,8 @@ enum viommu_flags {
    /* vIOMMU needs nesting parent HWPT to create nested HWPT */
    VIOMMU_FLAG_WANT_NESTING_PARENT = BIT_ULL(0),
    VIOMMU_FLAG_PASID_SUPPORTED = BIT_ULL(1),
+    /* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */
+    VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2),
 };
 
 /* Host IOMMU quirks. Extracted from host IOMMU capabilities */
include/hw/vfio/vfio-device.h +1 -0
index 828a31c006..a95c5bf503 100644
@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev,
 VFIOContainer *bcontainer,
 void vfio_device_unprepare(VFIODevice *vbasedev);
 
 bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev);
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice
 *vbasedev);
 bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev,

uint32_t type, void *caps, uint32_t size);

hw/i386/intel_iommu.c +5 -2
index 4924aa4375..5ec5ed82d7 100644
@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus
 *bus, void *opaque, int devfn)
 static uint64_t vtd_get_viommu_flags(void *opaque)
 {
    IntelIOMMUState *s = opaque;
-    uint64_t flags;
+    uint64_t flags = 0;
 
-    flags = s->fsts ? VIOMMU_FLAG_WANT_NESTING_PARENT : 0;
+    if (s->fsts) {
+        flags = VIOMMU_FLAG_WANT_NESTING_PARENT |
+                VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING;
+    }
 
    return flags;

}

hw/vfio/device.c +11 -0
index 973fc35b59..8f7ae919a5 100644
@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev)
    vbasedev->bcontainer = NULL;
 }
 
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
+
+    if (vdev) {
+        return !!(pci_device_get_viommu_flags(PCI_DEVICE(vdev)) &
+                  VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING);
+    }
+    return false;
+}
+
 bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev)
 {
    VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
hw/vfio/iommufd.c +1 -0
index 131612eb83..231635cdc7 100644
@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice
 *vbasedev,
    ERRP_GUARD();
    IOMMUFDBackend *iommufd = vbasedev->iommufd;
    VFIOContainer *bcontainer = VFIO_IOMMU(container);
+    bool viommu_nesting, viommu_nesting_dirty;
    uint32_t type, flags = 0;
    uint64_t hw_caps;
    VendorCaps caps;
hw/vfio/iommufd.c +7 -1
@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice
 *vbasedev,
        return false;
    }
 
+    viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev);
+    viommu_nesting_dirty =
+        vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev);
+
    if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) {
-        flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        if (!viommu_nesting || viommu_nesting_dirty) {
+            flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        }
    }
 
    /*
hw/vfio/iommufd.c +1 -1
@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice
 *vbasedev,
     * force to create it so that it could be reused by vIOMMU to create
     * nested HWPT.
     */
-    if (vfio_device_get_viommu_flags_want_nesting(vbasedev)) {
+    if (viommu_nesting) {
        flags |= IOMMU_HWPT_ALLOC_NEST_PARENT;
 

if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
--
2.43.0

#2 lore raw
In reply to Shameer Kolothum
Quoted 119 lines · Shameer Kolothum

QEMU smmuv3 accel does not support live migration yet, so dirty tracking for the nesting parent HWPT is not useful.

Also, nested vIOMMU use cases can break on some platforms. For example, SMMUv3 with HTTU may advertise dirty tracking capability, but the kernel supports it only for stage-1. Requesting dirty tracking for a nesting parent HWPT (stage-2) can fail.

Add a vIOMMU flag to explicitly request dirty tracking for the nesting parent HWPT. For nested cases, dirty tracking is enabled only when requested by the vIOMMU.

Non-nested cases and Intel vIOMMU keep the existing behavior.

Fixes fc6dafb98cec ("hw/arm/smmuv3: Implement get_viommu_cap() callback")
---
  include/hw/core/iommu.h       |  2 ++
  include/hw/vfio/vfio-device.h |  1 +
  hw/i386/intel_iommu.c         |  7 +++++--
  hw/vfio/device.c              | 11 +++++++++++
  hw/vfio/iommufd.c             | 11 +++++++++--
  5 files changed, 28 insertions(+), 4 deletions(-)
include/hw/core/iommu.h +2 -0
index 86af315c15..cd59a367ce 100644
@@ -21,6 +21,8 @@ enum viommu_flags {
      /* vIOMMU needs nesting parent HWPT to create nested HWPT */
      VIOMMU_FLAG_WANT_NESTING_PARENT = BIT_ULL(0),
      VIOMMU_FLAG_PASID_SUPPORTED = BIT_ULL(1),
+    /* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */
+    VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2),
  };
 
  /* Host IOMMU quirks. Extracted from host IOMMU capabilities */
include/hw/vfio/vfio-device.h +1 -0
index 828a31c006..a95c5bf503 100644
@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer,
  void vfio_device_unprepare(VFIODevice *vbasedev);
 
  bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev);
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev);
  bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev,
                                                  uint32_t type, void *caps,
                                                  uint32_t size);
hw/i386/intel_iommu.c +5 -2
index 4924aa4375..5ec5ed82d7 100644
@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn)
  static uint64_t vtd_get_viommu_flags(void *opaque)
  {
      IntelIOMMUState *s = opaque;
-    uint64_t flags;
+    uint64_t flags = 0;
 
-    flags = s->fsts ? VIOMMU_FLAG_WANT_NESTING_PARENT : 0;
+    if (s->fsts) {
+        flags = VIOMMU_FLAG_WANT_NESTING_PARENT |
+                VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING;
+    }
 
      return flags;
  }
hw/vfio/device.c +11 -0
index 973fc35b59..8f7ae919a5 100644
@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev)
      vbasedev->bcontainer = NULL;
  }
 
+bool vfio_device_get_viommu_flags_want_nesting_dirty(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
+
+    if (vdev) {
+        return !!(pci_device_get_viommu_flags(PCI_DEVICE(vdev)) &
+                  VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING);
+    }
+    return false;
+}
+
  bool vfio_device_get_viommu_flags_want_nesting(VFIODevice *vbasedev)
  {
      VFIOPCIDevice *vdev = vfio_pci_from_vfio_device(vbasedev);
hw/vfio/iommufd.c +8 -1
index 131612eb83..231635cdc7 100644
@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
      ERRP_GUARD();
      IOMMUFDBackend *iommufd = vbasedev->iommufd;
      VFIOContainer *bcontainer = VFIO_IOMMU(container);
+    bool viommu_nesting, viommu_nesting_dirty;
      uint32_t type, flags = 0;
      uint64_t hw_caps;
      VendorCaps caps;
@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
          return false;
      }
 
+    viommu_nesting = vfio_device_get_viommu_flags_want_nesting(vbasedev);
+    viommu_nesting_dirty =
+        vfio_device_get_viommu_flags_want_nesting_dirty(vbasedev);
+
      if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) {
-        flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        if (!viommu_nesting || viommu_nesting_dirty) {
+            flags |= IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
+        }
      }
 
      /*

hw/vfio/iommufd.c static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, · line 424

Quoted 8 lines
hw/vfio/iommufd.c +1 -1
@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
       * force to create it so that it could be reused by vIOMMU to create
       * nested HWPT.
       */
-    if (vfio_device_get_viommu_flags_want_nesting(vbasedev)) {
+    if (viommu_nesting) {
          flags |= IOMMU_HWPT_ALLOC_NEST_PARENT;
 
          if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,

Applied to vfio-next.

Thanks,

C.