hw/vfio/iommufd: Control dirty tracking for nesting parent HWPT
PATCH 1/1 hw/vfio/iommufd: Control dirty tracking for nesting parent HWPT
QEMU smmuv3 accel does not support live migration yet, so dirty tracking for the nesting parent HWPT is not useful.
Also, nested vIOMMU use cases can break on some platforms. For example, SMMUv3 with HTTU may advertise dirty tracking capability, but the kernel supports it only for stage-1. Requesting dirty tracking for a nesting parent HWPT (stage-2) can fail.
Add a vIOMMU flag to explicitly request dirty tracking for the nesting parent HWPT. For nested cases, dirty tracking is enabled only when requested by the vIOMMU.
Non-nested cases and Intel vIOMMU keep the existing behavior.
--- include/hw/core/iommu.h | 2 ++ include/hw/vfio/vfio-device.h | 1 + hw/i386/intel_iommu.c | 7 +++++-- hw/vfio/device.c | 11 +++++++++++ hw/vfio/iommufd.c | 11 +++++++++-- 5 files changed, 28 insertions(+), 4 deletions(-)
include/hw/core/iommu.h +2 -0
@@ -21,6 +21,8 @@ enum viommu_flags {/* vIOMMU needs nesting parent HWPT to create nested HWPT */ VIOMMU_FLAG_WANT_NESTING_PARENT =BIT_ULL (0 ),VIOMMU_FLAG_PASID_SUPPORTED =BIT_ULL (1 ), +/* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */ + VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2), };/* Host IOMMU quirks. Extracted from host IOMMU capabilities */
include/hw/vfio/vfio-device.h +1 -0
@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer,void vfio_device_unprepare (VFIODevice *vbasedev);bool vfio_device_get_viommu_flags_want_nesting (VFIODevice *vbasedev); +bool vfio_device_get_viommu_flags_want_nesting_dirty (VFIODevice *vbasedev);bool vfio_device_get_host_iommu_quirk_bypass_ro (VFIODevice *vbasedev,uint32_t type,void *caps,uint32_t size);
hw/i386/intel_iommu.c +5 -2
@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn)static uint64_t vtd_get_viommu_flags (void *opaque) {IntelIOMMUState *s = opaque; -uint64_t flags; +uint64_t flags =0 ; - flags = s->fsts ?VIOMMU_FLAG_WANT_NESTING_PARENT :0 ; +if (s->fsts) { + flags =VIOMMU_FLAG_WANT_NESTING_PARENT | +VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING ; + }return flags; }
hw/vfio/device.c +11 -0
@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev) vbasedev->bcontainer =NULL ; } +bool vfio_device_get_viommu_flags_want_nesting_dirty (VFIODevice *vbasedev) +{ +VFIOPCIDevice *vdev =vfio_pci_from_vfio_device (vbasedev); + +if (vdev) { +return !!(pci_device_get_viommu_flags (PCI_DEVICE (vdev)) & +VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING ); + } +return false; +} +bool vfio_device_get_viommu_flags_want_nesting (VFIODevice *vbasedev) {VFIOPCIDevice *vdev =vfio_pci_from_vfio_device (vbasedev);
hw/vfio/iommufd.c +9 -2
@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,ERRP_GUARD ();IOMMUFDBackend *iommufd = vbasedev->iommufd;VFIOContainer *bcontainer =VFIO_IOMMU (container); +bool viommu_nesting, viommu_nesting_dirty;uint32_t type, flags =0 ;uint64_t hw_caps;VendorCaps caps;
@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,return false; } + viommu_nesting =vfio_device_get_viommu_flags_want_nesting (vbasedev); + viommu_nesting_dirty = +vfio_device_get_viommu_flags_want_nesting_dirty (vbasedev); +if (hw_caps &IOMMU_HW_CAP_DIRTY_TRACKING ) { - flags =IOMMU_HWPT_ALLOC_DIRTY_TRACKING ; +if (!viommu_nesting || viommu_nesting_dirty) { + flags |=IOMMU_HWPT_ALLOC_DIRTY_TRACKING ; + } }/*
@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,* force to create it so that it could be reused by vIOMMU to create * nested HWPT. */ -if (vfio_device_get_viommu_flags_want_nesting (vbasedev)) { +if (viommu_nesting) { flags |=IOMMU_HWPT_ALLOC_NEST_PARENT ; if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
Signature · 1 line
2.43.0
Quoted 14 lines
-----Original Message-----
From: Shameer Kolothum <[email protected]>
Subject: [PATCH] hw/vfio/iommufd: Control dirty tracking for nesting parent HWPTQEMU smmuv3 accel does not support live migration yet, so dirty tracking for the nesting parent HWPT is not useful.
Also, nested vIOMMU use cases can break on some platforms. For example, SMMUv3 with HTTU may advertise dirty tracking capability, but the kernel supports it only for stage-1. Requesting dirty tracking for a nesting parent HWPT (stage-2) can fail.
Quoted 8 lines
Add a vIOMMU flag to explicitly request dirty tracking for the nesting parent HWPT. For nested cases, dirty tracking is enabled only when requested by the vIOMMU.
Non-nested cases and Intel vIOMMU keep the existing behavior.
Fixes fc6dafb98cec ("hw/arm/smmuv3: Implement get_viommu_cap() callback")Signed-off-by Shameer Kolothum <[email protected]>
Thanks
Zhenzhong
Quoted 119 lines
---
include/hw/core/iommu.h | 2 ++ include/hw/vfio/vfio-device.h | 1 + hw/i386/intel_iommu.c | 7 +++++-- hw/vfio/device.c | 11 +++++++++++ hw/vfio/iommufd.c | 11 +++++++++-- 5 files changed, 28 insertions(+), 4 deletions(-)include/hw/core/iommu.h +2
-0@@ -21,6 +21,8 @@ enum viommu_flags {/* vIOMMU needs nesting parent HWPT to create nested HWPT */ VIOMMU_FLAG_WANT_NESTING_PARENT =BIT_ULL (0 ),VIOMMU_FLAG_PASID_SUPPORTED =BIT_ULL (1 ), +/* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */ + VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2), };/* Host IOMMU quirks. Extracted from host IOMMU capabilities */ include/hw/vfio/vfio-device.h +1
-0@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev,VFIOContainer *bcontainer, void vfio_device_unprepare(VFIODevice *vbasedev);bool vfio_device_get_viommu_flags_want_nesting (VFIODevice *vbasedev); +bool vfio_device_get_viommu_flags_want_nesting_dirty (VFIODevice *vbasedev); bool vfio_device_get_host_iommu_quirk_bypass_ro(VFIODevice *vbasedev,
uint32_ttype, void *caps,uint32_tsize);hw/i386/intel_iommu.c +5
-2@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn)static uint64_t vtd_get_viommu_flags (void *opaque) {IntelIOMMUState *s = opaque; -uint64_t flags; +uint64_t flags =0 ; - flags = s->fsts ?VIOMMU_FLAG_WANT_NESTING_PARENT :0 ; +if (s->fsts) { + flags =VIOMMU_FLAG_WANT_NESTING_PARENT | +VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING ; + }return flags;}
hw/vfio/device.c +11
-0@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev) vbasedev->bcontainer =NULL ; } +bool vfio_device_get_viommu_flags_want_nesting_dirty (VFIODevice *vbasedev) +{ +VFIOPCIDevice *vdev =vfio_pci_from_vfio_device (vbasedev); + +if (vdev) { +return !!(pci_device_get_viommu_flags (PCI_DEVICE (vdev)) & +VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING ); + } +return false; +} +bool vfio_device_get_viommu_flags_want_nesting (VFIODevice *vbasedev) {VFIOPCIDevice *vdev =vfio_pci_from_vfio_device (vbasedev);hw/vfio/iommufd.c +1
-0@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,ERRP_GUARD ();IOMMUFDBackend *iommufd = vbasedev->iommufd;VFIOContainer *bcontainer =VFIO_IOMMU (container); +bool viommu_nesting, viommu_nesting_dirty;uint32_t type, flags =0 ;uint64_t hw_caps;VendorCaps caps;hw/vfio/iommufd.c +7
-1@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, return false; } + viommu_nesting =vfio_device_get_viommu_flags_want_nesting (vbasedev); + viommu_nesting_dirty = +vfio_device_get_viommu_flags_want_nesting_dirty (vbasedev); +if (hw_caps &IOMMU_HW_CAP_DIRTY_TRACKING ) { - flags =IOMMU_HWPT_ALLOC_DIRTY_TRACKING ; +if (!viommu_nesting || viommu_nesting_dirty) { + flags |=IOMMU_HWPT_ALLOC_DIRTY_TRACKING ; + } }/*hw/vfio/iommufd.c +1
-1@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice*vbasedev, * force to create it so that it could be reused by vIOMMU to create * nested HWPT. */ -if (vfio_device_get_viommu_flags_want_nesting (vbasedev)) { +if (viommu_nesting) { flags |=IOMMU_HWPT_ALLOC_NEST_PARENT ;if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
--
2.43.0
Cédric Le Goater
AppliedQuoted 119 lines · Shameer Kolothum
QEMU smmuv3 accel does not support live migration yet, so dirty tracking for the nesting parent HWPT is not useful.
Also, nested vIOMMU use cases can break on some platforms. For example, SMMUv3 with HTTU may advertise dirty tracking capability, but the kernel supports it only for stage-1. Requesting dirty tracking for a nesting parent HWPT (stage-2) can fail.
Add a vIOMMU flag to explicitly request dirty tracking for the nesting parent HWPT. For nested cases, dirty tracking is enabled only when requested by the vIOMMU.
Non-nested cases and Intel vIOMMU keep the existing behavior.
Fixes fc6dafb98cec ("hw/arm/smmuv3: Implement get_viommu_cap() callback")Signed-off-by Shameer Kolothum <[email protected]>--- include/hw/core/iommu.h | 2 ++ include/hw/vfio/vfio-device.h | 1 + hw/i386/intel_iommu.c | 7 +++++-- hw/vfio/device.c | 11 +++++++++++ hw/vfio/iommufd.c | 11 +++++++++-- 5 files changed, 28 insertions(+), 4 deletions(-)include/hw/core/iommu.h +2
-0@@ -21,6 +21,8 @@ enum viommu_flags {/* vIOMMU needs nesting parent HWPT to create nested HWPT */ VIOMMU_FLAG_WANT_NESTING_PARENT =BIT_ULL (0 ),VIOMMU_FLAG_PASID_SUPPORTED =BIT_ULL (1 ), +/* vIOMMU needs dirty tracking on the nesting parent HWPT for nested use */ + VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING = BIT_ULL(2), };/* Host IOMMU quirks. Extracted from host IOMMU capabilities */ include/hw/vfio/vfio-device.h +1
-0@@ -268,6 +268,7 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer,void vfio_device_unprepare (VFIODevice *vbasedev);bool vfio_device_get_viommu_flags_want_nesting (VFIODevice *vbasedev); +bool vfio_device_get_viommu_flags_want_nesting_dirty (VFIODevice *vbasedev);bool vfio_device_get_host_iommu_quirk_bypass_ro (VFIODevice *vbasedev,uint32_t type,void *caps,uint32_t size);hw/i386/intel_iommu.c +5
-2@@ -4831,9 +4831,12 @@ static void vtd_dev_unset_iommu_device(PCIBus *bus, void *opaque, int devfn)static uint64_t vtd_get_viommu_flags (void *opaque) {IntelIOMMUState *s = opaque; -uint64_t flags; +uint64_t flags =0 ; - flags = s->fsts ?VIOMMU_FLAG_WANT_NESTING_PARENT :0 ; +if (s->fsts) { + flags =VIOMMU_FLAG_WANT_NESTING_PARENT | +VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING ; + }return flags; }hw/vfio/device.c +11
-0@@ -522,6 +522,17 @@ void vfio_device_unprepare(VFIODevice *vbasedev) vbasedev->bcontainer =NULL ; } +bool vfio_device_get_viommu_flags_want_nesting_dirty (VFIODevice *vbasedev) +{ +VFIOPCIDevice *vdev =vfio_pci_from_vfio_device (vbasedev); + +if (vdev) { +return !!(pci_device_get_viommu_flags (PCI_DEVICE (vdev)) & +VIOMMU_FLAG_WANT_NESTING_DIRTY_TRACKING ); + } +return false; +} +bool vfio_device_get_viommu_flags_want_nesting (VFIODevice *vbasedev) {VFIOPCIDevice *vdev =vfio_pci_from_vfio_device (vbasedev);hw/vfio/iommufd.c +8
-1@@ -349,6 +349,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,ERRP_GUARD ();IOMMUFDBackend *iommufd = vbasedev->iommufd;VFIOContainer *bcontainer =VFIO_IOMMU (container); +bool viommu_nesting, viommu_nesting_dirty;uint32_t type, flags =0 ;uint64_t hw_caps;VendorCaps caps;@@ -402,8 +403,14 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,return false; } + viommu_nesting =vfio_device_get_viommu_flags_want_nesting (vbasedev); + viommu_nesting_dirty = +vfio_device_get_viommu_flags_want_nesting_dirty (vbasedev); +if (hw_caps &IOMMU_HW_CAP_DIRTY_TRACKING ) { - flags =IOMMU_HWPT_ALLOC_DIRTY_TRACKING ; +if (!viommu_nesting || viommu_nesting_dirty) { + flags |=IOMMU_HWPT_ALLOC_DIRTY_TRACKING ; + } }/*
hw/vfio/iommufd.c static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev, · line 424
Quoted 8 lines
hw/vfio/iommufd.c +1
-1@@ -411,7 +418,7 @@ static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,* force to create it so that it could be reused by vIOMMU to create * nested HWPT. */ -if (vfio_device_get_viommu_flags_want_nesting (vbasedev)) { +if (viommu_nesting) { flags |=IOMMU_HWPT_ALLOC_NEST_PARENT ; if (vfio_device_get_host_iommu_quirk_bypass_ro(vbasedev, type,
Applied to vfio-next.
Thanks,
C.