powerpc: iommu: Initial IOMMUFD support for PPC64
Subsystems (3)
RFC PATCH 1/1 powerpc: iommu: Initial IOMMUFD support for PPC64
The RFC attempts to implement the IOMMUFD support on PPC64 by adding new iommu_ops for paging domain. The existing platform domain continues to be the default domain for in-kernel use. The domain ownership transfer ensures the reset of iommu states for the new paging domain and in-kernel usage.
On PPC64, IOVA ranges are based on the type of the DMA window and their properties. Currently, there is no way to expose the attributes of the non-default 64-bit DMA window, which the platform supports. The platform allows the operating system to select the starting offset(at 4GiB or 512PiB default offset), pagesize and window size for the non-default 64-bit DMA window. For example, with VFIO, this is handled via VFIO_IOMMU_SPAPR_TCE_GET_INFO and VFIO_IOMMU_SPAPR_TCE_CREATE|REMOVE ioctls. While I am exploring the ways to expose and configure these DMA window attributes as per user input, any suggestions in this regard will be very helpful.
Currently existing vfio type1 specific vfio-compat driver even with this patch will not work for PPC64. I believe we need to have a separate "vfio-spapr-compat" driver to make it work.
So brief list of current open problems and ongoing reworks:
- Second DMA window support as mentioned above.
- KVM support.
- EEH support.
- The vfio compat driver for the spapr tce iommu.
- Multiple devices (multifunction, same/different iommu group checks, SRIOV VF assignment) support.
- Race conditions, device plug/unplug.
- self|tests.
The patch currently works for single device and exposes only the default DMA window of 1GB to the user. It has been tested for both PowerNV and pSeries machine tce iommu backends. The testing was done using a Qemu[1] and TCG guest having a NVME device passthrough. One can use the command like below to try:
qemu-system-ppc64 -machine pseries -accel tcg \
-device spapr-pci-host-bridge,index=1,id=pci.1,ddw=off \
-device vfio-pci,host=<hostdev>,id=hostdev0,\
bus=pci.1.0,addr=0x1,iommufd=iommufd0 \
-object iommufd,id=iommufd0 <...>
...
root:localhost# mount /dev/nvme0n1 /mnt root:localhost# ls /mnt
...
The current patch is based on linux kernel 6.19-rc6 tree.
References:
1 : https://github.com/shivaprasadbhat/qemu/tree/iommufd-wip
--- arch/powerpc/include/asm/iommu.h | 2 arch/powerpc/kernel/iommu.c | 181 +++++++++++++++++++++++++ arch/powerpc/platforms/powernv/pci-ioda-tce.c | 4 - arch/powerpc/platforms/powernv/pci-ioda.c | 4 - arch/powerpc/platforms/powernv/pci.h | 2 arch/powerpc/platforms/pseries/iommu.c | 6 - drivers/vfio/Kconfig | 4 - 7 files changed, 190 insertions(+), 13 deletions(-)
arch/powerpc/include/asm/iommu.h +1 -1
@@ -46,7 +46,7 @@ struct iommu_table_ops { long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs); + unsigned long attrs, bool is_phys);#ifdef CONFIG_IOMMU_API /* * Exchanges existing TCE with new TCE plus direction bits;
arch/powerpc/kernel/iommu.c +179 -2
212 lines not on this page · raw message
arch/powerpc/platforms/powernv/pci-ioda-tce.c +2 -2
@@ -123,10 +123,10 @@ static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc)int pnv_tce_build (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool is_phys) {u64 proto_tce =iommu_direction_to_tce_perm (direction); -u64 rpn =__pa (uaddr) >> tbl->it_page_shift; +u64 rpn = !is_phys ?__pa (uaddr) >> tbl->it_page_shift : uaddr >> tbl->it_page_shift;long i;if (proto_tce &TCE_PCI_WRITE )
arch/powerpc/platforms/powernv/pci-ioda.c +2 -2
@@ -1241,10 +1241,10 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,static int pnv_ioda2_tce_build (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool is_phys) {int ret =pnv_tce_build (tbl, index, npages, uaddr, direction, - attrs); + attrs, is_phys);if (!ret)pnv_pci_ioda2_tce_invalidate (tbl, index, npages);
arch/powerpc/platforms/powernv/pci.h +1 -1
@@ -300,7 +300,7 @@ extern void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,extern int pnv_tce_build (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs); +unsigned long attrs,bool is_phys);extern void pnv_tce_free (struct iommu_table *tbl,long index,long npages);extern int pnv_tce_xchg (struct iommu_table *tbl,long index,unsigned long *hpa,enum dma_data_direction *direction);
arch/powerpc/platforms/pseries/iommu.c +3 -3
@@ -122,7 +122,7 @@ static void iommu_pseries_free_group(struct iommu_table_group *table_group,static int tce_build_pSeries (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool false) {u64 proto_tce;__be64 *tcep;
@@ -250,7 +250,7 @@ static DEFINE_PER_CPU(__be64 *, tce_page);static int tce_buildmulti_pSeriesLP (struct iommu_table *tbl,long tcenum,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool is_phys) {u64 rc =0 ;u64 proto_tce;
@@ -287,7 +287,7 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,__this_cpu_write (tce_page, tcep); } - rpn =__pa (uaddr) >> tceshift; + rpn = !is_phys ?__pa (uaddr) >> tceshift : uaddr >> tceshift; proto_tce =TCE_PCI_READ ;if (direction !=DMA_TO_DEVICE ) proto_tce |=TCE_PCI_WRITE ;
drivers/vfio/Kconfig +2 -2
@@ -4,7 +4,7 @@ menuconfig VFIO select IOMMU_API depends on IOMMUFD || !IOMMUFD select INTERVAL_TREE - select VFIO_GROUP if SPAPR_TCE_IOMMU || IOMMUFD=n + select VFIO_GROUP if IOMMUFD=n select VFIO_DEVICE_CDEV if !VFIO_GROUP select VFIO_CONTAINER if IOMMUFD=n help
@@ -16,7 +16,7 @@ menuconfig VFIO if VFIO config VFIO_DEVICE_CDEV bool "Support for the VFIO cdev /dev/vfio/devices/vfioX" - depends on IOMMUFD && !SPAPR_TCE_IOMMU + depends on IOMMUFD default !VFIO_GROUP help The VFIO device cdev is another way for userspace to get device
Quoted 3 lines · Shivaprasad G Bhat
The RFC attempts to implement the IOMMUFD support on PPC64 by adding new
iommu_opsfor paging domain. The existing platform domain continues to be the default domain for in-kernel use.
It would be nice to see the platform domain go away and ppc use the normal dma-iommu.c stuff, but I don't think it is critical to making it work with iommufd.
Quoted 10 lines
On PPC64, IOVA ranges are based on the type of the DMA window and their properties. Currently, there is no way to expose the attributes of the non-default 64-bit DMA window, which the platform supports. The platform allows the operating system to select the starting offset(at 4GiB or 512PiB default offset), pagesize and window size for the non-default 64-bit DMA window. For example, with VFIO, this is handled via
VFIO_IOMMU_SPAPR_TCE_GET_INFOand VFIO_IOMMU_SPAPR_TCE_CREATE|REMOVE ioctls. While I am exploring the ways to expose and configure these DMA window attributes as per user input, any suggestions in this regard will be very helpful.
You can pass in driver specific information during HWPT creation, so any properties you need can be specified there.
Then you'd want to introduce a new domain op to get the apertures instead of the single range hard coded into the domain struct. The new op would be able to return a list. We can use this op to return apertures for sign extension page tables too.
Update iommufd to calculate the reserved regions by evaluating the whole list.
I think you'll find this pretty straight forward, I'd do it as a followup patch to this one.
Quoted 3 lines
Currently existing vfio type1 specific vfio-compat driver even with this patch will not work for PPC64. I believe we need to have a separate "vfio-spapr-compat" driver to make it work.
Yes, vfio-compat doesn't support the special spapr ioctls.
I don't think you need a new driver, just implement whatever they do with the existing interfaces, probably in its own .c file though.
However, I have no idea what is required to implement those ops, or if it is even possible.. It may be easier to just leave the old vfio stuff around instead of trying to compat it. The purpose of compat was to be able to build kernels without type1 at all. It isn't necessary to start using iommufd in new apps with the new interfaces.
Given you are mainly looking at a VMM that already will have iommufd support it may not be worthwhile.
arch/powerpc/kernel/iommu.c spapr_tce_blocked_iommu_attach_dev(struct iommu_domain *platform_domain, · line 1205
Quoted 6 lines
arch/powerpc/kernel/iommu.c +2
-0@@ -1201,7 +1201,15 @@ spapr_tce_blocked_iommu_attach_dev(struct iommu_domain *platform_domain,* also sets the dma_api ops */ table_group =iommu_group_get_iommudata (grp); + +if (old && old->type ==IOMMU_DOMAIN_DMA ) {
I'm trying to delete IOMMU_DOMAIN_DMA please don't use it in drivers.
Quoted 5 lines
arch/powerpc/kernel/iommu.c +0
-0static const struct iommu_ops spapr_tce_iommu_ops = { .default_domain = &spapr_tce_platform_domain, .blocked_domain = &spapr_tce_blocked_domain,@@ -1267,6 +1436,14 @@ static const struct iommu_ops spapr_tce_iommu_ops = { .probe_device = spapr_tce_iommu_probe_device,
arch/powerpc/kernel/iommu.c static const struct iommu_ops spapr_tce_iommu_ops = { · line 1446
Quoted 10 lines
arch/powerpc/kernel/iommu.c +8
-0.release_device = spapr_tce_iommu_release_device, .device_group = spapr_tce_iommu_device_group, + .domain_alloc_paging = spapr_tce_domain_alloc_paging, + .default_domain_ops = &(const struct iommu_domain_ops) { + .attach_dev = spapr_tce_iommu_attach_device, + .map_pages = spapr_tce_iommu_map_pages, + .unmap_pages = spapr_tce_iommu_unmap_pages, + .iova_to_phys = spapr_tce_iommu_iova_to_phys, + .free = spapr_tce_domain_free, + }
Please don't use default_domain_ops in a driver that is supporting multiple domain types and platform, it becomes confusing to guess which domain type those ops are linked to.
You should also implement the BLOCKING domain type to make VFIO work better
I wouldn't try to guess if this is right or not, but it looks pretty reasonable as a first start.
Jason
Hi Jason,
Thanks for reviewing this patch.
Quoted 7 lines · Jason Gunthorpe
Quoted 3 lines · Shivaprasad G Bhat
The RFC attempts to implement the IOMMUFD support on PPC64 by adding new
iommu_opsfor paging domain. The existing platform domain continues to be the default domain for in-kernel use.It would be nice to see the platform domain go away and ppc use the normal
dma-iommu.cstuff, but I don't think it is critical to making it work with iommufd.
I agree. I have started on this. I will send incremental changes
as follow-up after this.
Quoted 10 lines
Quoted 10 lines
On PPC64, IOVA ranges are based on the type of the DMA window and their properties. Currently, there is no way to expose the attributes of the non-default 64-bit DMA window, which the platform supports. The platform allows the operating system to select the starting offset(at 4GiB or 512PiB default offset), pagesize and window size for the non-default 64-bit DMA window. For example, with VFIO, this is handled via
VFIO_IOMMU_SPAPR_TCE_GET_INFOand VFIO_IOMMU_SPAPR_TCE_CREATE|REMOVE ioctls. While I am exploring the ways to expose and configure these DMA window attributes as per user input, any suggestions in this regard will be very helpful.
Quoted 2 lines
You can pass in driver specific information during HWPT creation, so any properties you need can be specified there.
Sure. I think IOMMU_GET_HW_INFO would be useful for getting the
platform supported configuration in this case.
Quoted 10 lines
Then you'd want to introduce a new domain op to get the apertures instead of the single range hard coded into the domain struct. The new op would be able to return a list. We can use this op to return apertures for sign extension page tables too.
Update iommufd to calculate the reserved regions by evaluating the whole list.
I think you'll find this pretty straight forward, I'd do it as a followup patch to this one.
Thanks. I will wait for that patch.
Quoted 8 lines
Quoted 4 lines
Currently existing vfio type1 specific vfio-compat driver even with this patch will not work for PPC64. I believe we need to have a separate "vfio-spapr-compat" driver to make it work.
Yes, vfio-compat doesn't support the special spapr ioctls.
I don't think you need a new driver, just implement whatever they do with the existing interfaces, probably in its own .c file though.
There are ioctl number conflicts like
# grep -n "VFIO_BASE + 1[89]" include/uapi/linux/vfio.h | grep define 940:#defineVFIO_DEVICE_BIND_IOMMUFD_IO(VFIO_TYPE, VFIO_BASE + 18) 976:#defineVFIO_DEVICE_ATTACH_IOMMUFD_PT_IO(VFIO_TYPE, VFIO_BASE + 19) 1833:#defineVFIO_IOMMU_SPAPR_UNREGISTER_MEMORY_IO(VFIO_TYPE, VFIO_BASE + 18) 1856:#defineVFIO_IOMMU_SPAPR_TCE_CREATE_IO(VFIO_TYPE, VFIO_BASE + 19) # grep -n "VFIO_BASE + 20" include/uapi/linux/vfio.h | grep define 999:#defineVFIO_DEVICE_DETACH_IOMMUFD_PT_IO(VFIO_TYPE, VFIO_BASE + 20) 1870:#defineVFIO_IOMMU_SPAPR_TCE_REMOVE_IO(VFIO_TYPE, VFIO_BASE + 20)
Quoted 8 lines
However, I have no idea what is required to implement those ops, or if it is even possible.. It may be easier to just leave the old vfio stuff around instead of trying to compat it. The purpose of compat was to be able to build kernels without type1 at all. It isn't necessary to start using iommufd in new apps with the new interfaces.
Given you are mainly looking at a VMM that already will have iommufd support it may not be worthwhile.
You are right. We do have some use cases beyond VMM, I will consider compat driver
only if it is helpful there.
Quoted 8 lines
arch/powerpc/kernel/iommu.c spapr_tce_blocked_iommu_attach_dev(struct iommu_domain *platform_domain, · line 1205
Quoted 6 lines
arch/powerpc/kernel/iommu.c +2
-0@@ -1201,7 +1201,15 @@ spapr_tce_blocked_iommu_attach_dev(struct iommu_domain *platform_domain,* also sets the dma_api ops */ table_group =iommu_group_get_iommudata (grp); + +if (old && old->type ==IOMMU_DOMAIN_DMA ) {I'm trying to delete
IOMMU_DOMAIN_DMAplease don't use it in drivers.
Sure.
Quoted 10 lines
Quoted 10 lines
arch/powerpc/kernel/iommu.c +3
-0static const struct iommu_ops spapr_tce_iommu_ops = { .default_domain = &spapr_tce_platform_domain, .blocked_domain = &spapr_tce_blocked_domain,@@ -1267,6 +1436,14 @@ static const struct iommu_ops spapr_tce_iommu_ops = { .probe_device = spapr_tce_iommu_probe_device, .release_device = spapr_tce_iommu_release_device, .device_group = spapr_tce_iommu_device_group, + .domain_alloc_paging = spapr_tce_domain_alloc_paging, + .default_domain_ops = &(const struct iommu_domain_ops ) { + .attach_dev = spapr_tce_iommu_attach_device,
Quoted 8 lines
arch/powerpc/kernel/iommu.c static const struct iommu_ops spapr_tce_iommu_ops = { · line 1446
Quoted 5 lines
arch/powerpc/kernel/iommu.c +5
-0+ .map_pages = spapr_tce_iommu_map_pages, + .unmap_pages = spapr_tce_iommu_unmap_pages, + .iova_to_phys = spapr_tce_iommu_iova_to_phys, + .free = spapr_tce_domain_free, + }Please don't use
default_domain_opsin a driver that is supporting multiple domain types and platform, it becomes confusing to guess which domain type those ops are linked to.
Sure.
Quoted 2 lines
You should also implement the BLOCKING domain type to make VFIO work better
I am not sure how this could help making VFIO better. May be, I am not able
to imagine the advantages with the current platform domain approach
in place. Could you please elaborate more on this?
Quoted 2 lines
I wouldn't try to guess if this is right or not, but it looks pretty reasonable as a first start.
Thanks, I will iterate this as RFC till i get to reasonable shape.
Regards,
Shivaprasad
Quoted 1 line
Jason
Quoted 5 lines · Shivaprasad G Bhat
Quoted 5 lines
Then you'd want to introduce a new domain op to get the apertures instead of the single range hard coded into the domain struct. The new op would be able to return a list. We can use this op to return apertures for sign extension page tables too.
Quoted 8 lines
Quoted 5 lines
Update iommufd to calculate the reserved regions by evaluating the whole list.
I think you'll find this pretty straight forward, I'd do it as a followup patch to this one.
Thanks. I will wait for that patch.
I think you will have to make it :)
Quoted 10 lines
There are ioctl number conflicts like
# grep -n "VFIO_BASE + 1[89]" include/uapi/linux/vfio.h | grep define 940:#defineVFIO_DEVICE_BIND_IOMMUFD_IO(VFIO_TYPE, VFIO_BASE + 18) 976:#defineVFIO_DEVICE_ATTACH_IOMMUFD_PT_IO(VFIO_TYPE, VFIO_BASE + 19) 1833:#defineVFIO_IOMMU_SPAPR_UNREGISTER_MEMORY_IO(VFIO_TYPE, VFIO_BASE + 18) 1856:#defineVFIO_IOMMU_SPAPR_TCE_CREATE_IO(VFIO_TYPE, VFIO_BASE + 19) # grep -n "VFIO_BASE + 20" include/uapi/linux/vfio.h | grep define 999:#defineVFIO_DEVICE_DETACH_IOMMUFD_PT_IO(VFIO_TYPE, VFIO_BASE + 20) 1870:#defineVFIO_IOMMU_SPAPR_TCE_REMOVE_IO(VFIO_TYPE, VFIO_BASE + 20)
It's Ok the compat codes will know what type it is operating in before it decodes any of those.
Quoted 2 lines
You are right. We do have some use cases beyond VMM, I will consider compat driver only if it is helpful there.
You can also use the type1 compat mode which will magically start working with PPC..
Quoted 6 lines
Quoted 2 lines
You should also implement the BLOCKING domain type to make VFIO work better
I am not sure how this could help making VFIO better. May be, I am not able to imagine the advantages with the current platform domain approach in place. Could you please elaborate more on this?
VFIO always uses a BLOCKED domain when it opens the device, then it changes to a paging domain. If the driver doesn't support a native BLOCKED domain then it allocates an empty page table and uses that.
A proper native BLOCKED domain has better error handling characteristics since it is not allowed to fail attach and it doesn't require allocation.
I think you will also find what you are doing easier if you push the iommu_domain down through the PPC iommu ops instead of retaining these unnecessary historical layers.
Jason
Quoted 6 lines · Jason Gunthorpe
Quoted 5 lines · Shivaprasad G Bhat
Quoted 5 lines
Then you'd want to introduce a new domain op to get the apertures instead of the single range hard coded into the domain struct. The new op would be able to return a list. We can use this op to return apertures for sign extension page tables too.
Quoted 8 lines
Quoted 7 lines
Quoted 5 lines
Update iommufd to calculate the reserved regions by evaluating the whole list.
I think you'll find this pretty straight forward, I'd do it as a followup patch to this one.
Thanks. I will wait for that patch.
I think you will have to make it :)
Sure. Thanks!
Quoted 11 lines
Quoted 11 lines
There are ioctl number conflicts like
# grep -n "VFIO_BASE + 1[89]" include/uapi/linux/vfio.h | grep define 940:#defineVFIO_DEVICE_BIND_IOMMUFD_IO(VFIO_TYPE, VFIO_BASE + 18) 976:#defineVFIO_DEVICE_ATTACH_IOMMUFD_PT_IO(VFIO_TYPE, VFIO_BASE + 19) 1833:#defineVFIO_IOMMU_SPAPR_UNREGISTER_MEMORY_IO(VFIO_TYPE, VFIO_BASE + 18) 1856:#defineVFIO_IOMMU_SPAPR_TCE_CREATE_IO(VFIO_TYPE, VFIO_BASE + 19) # grep -n "VFIO_BASE + 20" include/uapi/linux/vfio.h | grep define 999:#defineVFIO_DEVICE_DETACH_IOMMUFD_PT_IO(VFIO_TYPE, VFIO_BASE + 20) 1870:#defineVFIO_IOMMU_SPAPR_TCE_REMOVE_IO(VFIO_TYPE, VFIO_BASE + 20)
Quoted 2 lines
It's Ok the compat codes will know what type it is operating in before it decodes any of those.
I agree, the additional PPC specific ioctls can be selectively handled based
on arch.
Quoted 16 lines
Quoted 3 lines
You are right. We do have some use cases beyond VMM, I will consider compat driver only if it is helpful there.
You can also use the type1 compat mode which will magically start working with PPC..
Quoted 6 lines
Quoted 2 lines
You should also implement the BLOCKING domain type to make VFIO work better
I am not sure how this could help making VFIO better. May be, I am not able to imagine the advantages with the current platform domain approach in place. Could you please elaborate more on this?
VFIO always uses a BLOCKED domain when it opens the device, then it changes to a paging domain. If the driver doesn't support a native BLOCKED domain then it allocates an empty page table and uses that.
Quoted 7 lines
A proper native BLOCKED domain has better error handling characteristics since it is not allowed to fail attach and it doesn't require allocation.
I think you will also find what you are doing easier if you push the
iommu_domaindown through the PPC iommu ops instead of retaining these unnecessary historical layers.
I see there is some more history around this being discussed before on the patch here,
https://lore.kernel.org/all/[email protected]/
Will try to dig more and address this in future.
Thanks,
Shivaprasad
Quoted 1 line
Jason
Christophe Leroy (CS GROUP)
reviewerLe 27/01/2026 à 19:35, Shivaprasad G Bhat a écrit :
Quoted 47 lines
The RFC attempts to implement the IOMMUFD support on PPC64 by adding new
iommu_opsfor paging domain. The existing platform domain continues to be the default domain for in-kernel use. The domain ownership transfer ensures the reset of iommu states for the new paging domain and in-kernel usage.On PPC64, IOVA ranges are based on the type of the DMA window and their properties. Currently, there is no way to expose the attributes of the non-default 64-bit DMA window, which the platform supports. The platform allows the operating system to select the starting offset(at 4GiB or 512PiB default offset), pagesize and window size for the non-default 64-bit DMA window. For example, with VFIO, this is handled via
VFIO_IOMMU_SPAPR_TCE_GET_INFOand VFIO_IOMMU_SPAPR_TCE_CREATE|REMOVE ioctls. While I am exploring the ways to expose and configure these DMA window attributes as per user input, any suggestions in this regard will be very helpful.Currently existing vfio type1 specific vfio-compat driver even with this patch will not work for PPC64. I believe we need to have a separate "vfio-spapr-compat" driver to make it work.
So brief list of current open problems and ongoing reworks:
- Second DMA window support as mentioned above.
- KVM support.
- EEH support.
- The vfio compat driver for the spapr tce iommu.
- Multiple devices (multifunction, same/different iommu group checks, SRIOV VF assignment) support.
- Race conditions, device plug/unplug.
- self|tests.The patch currently works for single device and exposes only the default DMA window of 1GB to the user. It has been tested for both PowerNV and pSeries machine tce iommu backends. The testing was done using a Qemu[1] and TCG guest having a NVME device passthrough. One can use the command like below to try:
qemu-system-ppc64 -machine pseries -accel tcg \
(quoted diff) +0
-2-device spapr-pci-host-bridge,index=1,id=pci.1,ddw=off \ -device vfio-pci,host=<hostdev>,id=hostdev0,\bus=pci.1.0,addr=0x1,iommufd=iommufd0 \
-object iommufd,id=iommufd0 <...>
...
root:localhost# mount/dev/nvme0n1/mnt
root:localhost# ls /mnt
...
Quoted 1 line
The current patch is based on linux kernel 6.19-rc6 tree.
Getting the following build failure on linuxppc-dev patchwork with g5_defconfig or ppc64_defconfig:
Error: /linux/arch/powerpc/sysdev/dart_iommu.c:325:9: error: initialization of 'int (*)(struct iommu_table *, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int, bool)' {aka 'int (*)(struct iommu_table *, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int, _Bool)'} from incompatible pointer type 'int (*)(struct iommu_table *, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int)' [-Werror=incompatible-pointer-types]
.set = dart_build,
^~~~~~~~~~
/linux/arch/powerpc/sysdev/dart_iommu.c:325:9: note: (near initialization for 'iommu_dart_ops.set')
cc1: all warnings being treated as errors
make[5]: *** [/linux/scripts/Makefile.build:287: arch/powerpc/sysdev/dart_iommu.o] Error 1 make[4]: *** [/linux/scripts/Makefile.build:544: arch/powerpc/sysdev] Error 2
Christophe
Quoted 351 lines
Signed-off-by Shivaprasad G Bhat <[email protected]>--- arch/powerpc/include/asm/iommu.h | 2 arch/powerpc/kernel/iommu.c | 181 +++++++++++++++++++++++++ arch/powerpc/platforms/powernv/pci-ioda-tce.c | 4 - arch/powerpc/platforms/powernv/pci-ioda.c | 4 - arch/powerpc/platforms/powernv/pci.h | 2 arch/powerpc/platforms/pseries/iommu.c | 6 - drivers/vfio/Kconfig | 4 - 7 files changed, 190 insertions(+), 13 deletions(-)arch/powerpc/include/asm/iommu.h +1
-1@@ -46,7 +46,7 @@ struct iommu_table_ops { long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs); + unsigned long attrs, bool is_phys);#ifdef CONFIG_IOMMU_API /* * Exchanges existing TCE with new TCE plus direction bits; arch/powerpc/kernel/iommu.c +179
-2@@ -365,7 +365,7 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,/* Put the TCEs in the HW table */ build_fail = tbl->it_ops->set (tbl, entry, npages, (unsigned long )page & -IOMMU_PAGE_MASK (tbl), direction, attrs); +IOMMU_PAGE_MASK (tbl), direction, attrs, false);/* tbl->it_ops->set() only returns non-zero for transient errors. * Clean up the table bitmap in this case and return @@ -539,7 +539,7 @@ int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl,/* Insert into HW table */ build_fail = tbl->it_ops->set (tbl, entry, npages, vaddr &IOMMU_PAGE_MASK (tbl), - direction, attrs); + direction, attrs, false);if (unlikely (build_fail)) goto failure;@@ -1201,7 +1201,15 @@ spapr_tce_blocked_iommu_attach_dev(struct iommu_domain *platform_domain,* also sets the dma_api ops */ table_group =iommu_group_get_iommudata (grp); + +if (old && old->type ==IOMMU_DOMAIN_DMA ) { + ret = table_group->ops->unset_window (table_group,0 ); +if (ret) + goto exit; + } + ret = table_group->ops->take_ownership (table_group, dev); +exit:iommu_group_put (grp);return ret;@@ -1260,6 +1268,167 @@ static struct iommu_group *spapr_tce_iommu_device_group(struct device *dev)return hose->controller_ops.device_group (hose, pdev); } +struct ppc64_domain { +struct iommu_domain domain; +struct device *device;/* Make it a list */ +struct iommu_table *table; +spinlock_t list_lock; +struct rcu_head rcu; +}; + +static struct ppc64_domain *to_ppc64_domain (struct iommu_domain *dom) +{ + return container_of(dom, struct ppc64_domain, domain); +} + +static void spapr_tce_domain_free (struct iommu_domain *domain) +{ +struct ppc64_domain *ppc64_domain =to_ppc64_domain (domain); + +kfree (ppc64_domain); +} + +static const struct iommu_ops spapr_tce_iommu_ops; +static struct iommu_domain *spapr_tce_domain_alloc_paging (struct device *dev) +{ +struct iommu_group *grp =iommu_group_get (dev); +struct iommu_table_group *table_group; +struct ppc64_domain *ppc64_domain; +struct iommu_table *ptbl; +int ret =-1 ; + + table_group =iommu_group_get_iommudata (grp); + ppc64_domain =kzalloc (sizeof (*ppc64_domain),GFP_KERNEL ); +if (!ppc64_domain) +return NULL ; + +/* Just the default window hardcode for now */ + ret = table_group->ops->create_table (table_group,0 ,0xc ,0x40000000 ,1 , &ptbl); +iommu_tce_table_get (ptbl); + ppc64_domain->table = ptbl;/* REVISIT: Single device for now */ +if (!ppc64_domain->table) { +kfree (ppc64_domain); +iommu_tce_table_put (ptbl); +iommu_group_put (grp); +return NULL ; + } + + table_group->ops->set_window (table_group,0 , ptbl); +iommu_group_put (grp); + + ppc64_domain->domain.pgsize_bitmap =SZ_4K ; + ppc64_domain->domain.geometry.force_aperture = true; + ppc64_domain->domain.geometry.aperture_start =0 ; + ppc64_domain->domain.geometry.aperture_end =0x40000000 ;/*default window */ + ppc64_domain->domain.ops = spapr_tce_iommu_ops.default_domain_ops; + +spin_lock_init (&ppc64_domain->list_lock); + +return &ppc64_domain->domain; +} + +static size_t spapr_tce_iommu_unmap_pages (struct iommu_domain *domain, +unsigned long iova, +size_t pgsize,size_t pgcount, +struct iommu_iotlb_gather *gather) +{ +struct ppc64_domain *ppc64_domain =to_ppc64_domain (domain); +struct iommu_table *tbl = ppc64_domain->table; +unsigned long pgshift =__ffs (pgsize); +size_t size = pgcount << pgshift; +size_t mapped =0 ; +unsigned int tcenum; +int mask; + +if (pgsize !=SZ_4K ) +return -EINVAL ; + + size =PAGE_ALIGN (size); + + mask =IOMMU_PAGE_MASK (tbl); + tcenum = iova >> tbl->it_page_shift; + + tbl->it_ops->clear (tbl, tcenum, pgcount); + + mapped = pgsize * pgcount; + +return mapped; +} + +static phys_addr_t spapr_tce_iommu_iova_to_phys (struct iommu_domain *domain,dma_addr_t iova) +{ +struct ppc64_domain *ppc64_domain =to_ppc64_domain (domain); +struct iommu_table *tbl = ppc64_domain->table; +phys_addr_t paddr, rpn, tceval; +unsigned int tcenum; + + tcenum = iova >> tbl->it_page_shift; + tceval = tbl->it_ops->get (tbl, tcenum); + +/* Ignore the direction bits */ + rpn = tceval >> tbl->it_page_shift; + paddr = rpn << tbl->it_page_shift; + +return paddr; +} + +static int spapr_tce_iommu_map_pages (struct iommu_domain *domain, +unsigned long iova,phys_addr_t paddr, +size_t pgsize,size_t pgcount, +int prot,gfp_t gfp,size_t *mapped) +{ +struct ppc64_domain *ppc64_domain =to_ppc64_domain (domain); +enum dma_data_direction direction =DMA_BIDIRECTIONAL ; +struct iommu_table *tbl = ppc64_domain->table; +unsigned long pgshift =__ffs (pgsize); +size_t size = pgcount << pgshift; +unsigned int tcenum; +int ret; + +if (pgsize !=SZ_4K ) +return -EINVAL ; + +if (iova < ppc64_domain->domain.geometry.aperture_start || + (iova + size -1 ) > ppc64_domain->domain.geometry.aperture_end) +return -EINVAL ; + +if (!IS_ALIGNED (iova | paddr, pgsize)) +return -EINVAL ; + +if (!(prot &IOMMU_WRITE )) + direction =DMA_FROM_DEVICE ; + +if (!(prot &IOMMU_READ )) + direction =DMA_TO_DEVICE ; + + size =PAGE_ALIGN (size); + tcenum = iova >> tbl->it_page_shift; + +/* Put the TCEs in the HW table */ + ret = tbl->it_ops->set (tbl, tcenum, pgcount, + paddr, direction,0 , true); +if (!ret && mapped) + *mapped = pgsize; + +return 0 ; +} + +static int spapr_tce_iommu_attach_device (struct iommu_domain *domain, +struct device *dev,struct iommu_domain *old) +{ +struct ppc64_domain *ppc64_domain =to_ppc64_domain (domain); + +/* REVISIT */ +if (!domain) +return 0 ; + +/* REVISIT: Check table group, list handling */ + ppc64_domain->device = dev; + +return 0 ; +} + +static const struct iommu_ops spapr_tce_iommu_ops = { .default_domain = &spapr_tce_platform_domain, .blocked_domain = &spapr_tce_blocked_domain,@@ -1267,6 +1436,14 @@ static const struct iommu_ops spapr_tce_iommu_ops = { .probe_device = spapr_tce_iommu_probe_device, .release_device = spapr_tce_iommu_release_device, .device_group = spapr_tce_iommu_device_group, + .domain_alloc_paging = spapr_tce_domain_alloc_paging, + .default_domain_ops = &(const struct iommu_domain_ops ) { + .attach_dev = spapr_tce_iommu_attach_device, + .map_pages = spapr_tce_iommu_map_pages, + .unmap_pages = spapr_tce_iommu_unmap_pages, + .iova_to_phys = spapr_tce_iommu_iova_to_phys, + .free = spapr_tce_domain_free, + } };static struct attribute *spapr_tce_iommu_attrs[] = {arch/powerpc/platforms/powernv/pci-ioda-tce.c +2
-2@@ -123,10 +123,10 @@ static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc)int pnv_tce_build (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool is_phys) {u64 proto_tce =iommu_direction_to_tce_perm (direction); -u64 rpn =__pa (uaddr) >> tbl->it_page_shift; +u64 rpn = !is_phys ?__pa (uaddr) >> tbl->it_page_shift : uaddr >> tbl->it_page_shift;long i;if (proto_tce &TCE_PCI_WRITE )arch/powerpc/platforms/powernv/pci-ioda.c +2
-2@@ -1241,10 +1241,10 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,static int pnv_ioda2_tce_build (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool is_phys) {int ret =pnv_tce_build (tbl, index, npages, uaddr, direction, - attrs); + attrs, is_phys);if (!ret)pnv_pci_ioda2_tce_invalidate (tbl, index, npages);arch/powerpc/platforms/powernv/pci.h +1
-1@@ -300,7 +300,7 @@ extern void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,extern int pnv_tce_build (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs); +unsigned long attrs,bool is_phys);extern void pnv_tce_free (struct iommu_table *tbl,long index,long npages);extern int pnv_tce_xchg (struct iommu_table *tbl,long index,unsigned long *hpa,enum dma_data_direction *direction);arch/powerpc/platforms/pseries/iommu.c +3
-3@@ -122,7 +122,7 @@ static void iommu_pseries_free_group(struct iommu_table_group *table_group,static int tce_build_pSeries (struct iommu_table *tbl,long index,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool false) {u64 proto_tce;__be64 *tcep;@@ -250,7 +250,7 @@ static DEFINE_PER_CPU(__be64 *, tce_page);static int tce_buildmulti_pSeriesLP (struct iommu_table *tbl,long tcenum,long npages,unsigned long uaddr,enum dma_data_direction direction, -unsigned long attrs) +unsigned long attrs,bool is_phys) {u64 rc =0 ;u64 proto_tce;@@ -287,7 +287,7 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,__this_cpu_write (tce_page, tcep); } - rpn =__pa (uaddr) >> tceshift; + rpn = !is_phys ?__pa (uaddr) >> tceshift : uaddr >> tceshift; proto_tce =TCE_PCI_READ ;if (direction !=DMA_TO_DEVICE ) proto_tce |=TCE_PCI_WRITE ;drivers/vfio/Kconfig +2
-2@@ -4,7 +4,7 @@ menuconfig VFIO select IOMMU_API depends on IOMMUFD || !IOMMUFD select INTERVAL_TREE - select VFIO_GROUP if SPAPR_TCE_IOMMU || IOMMUFD=n + select VFIO_GROUP if IOMMUFD=n select VFIO_DEVICE_CDEV if !VFIO_GROUP select VFIO_CONTAINER if IOMMUFD=n help@@ -16,7 +16,7 @@ menuconfig VFIO if VFIO config VFIO_DEVICE_CDEV bool "Support for the VFIO cdev /dev/vfio/devices/vfioX" - depends on IOMMUFD && !SPAPR_TCE_IOMMU + depends on IOMMUFD default !VFIO_GROUP help The VFIO device cdev is another way for userspace to get device
Hi Christophe.
Thanks for trying the patch.
Quoted 8 lines · Christophe Leroy (CS GROUP)
Le 27/01/2026 à 19:35, Shivaprasad G Bhat a écrit :
Quoted 5 lines
The RFC attempts to implement the IOMMUFD support on PPC64 by adding new
iommu_opsfor paging domain. The existing platform domain continues to be the default domain for in-kernel use. The domain ownership transfer ensures the reset of iommu states for the new paging domain and in-kernel usage.
<snip/>
Quoted 9 lines
Quoted 6 lines
...
root:localhost# mount/dev/nvme0n1/mnt
root:localhost# ls /mnt
...The current patch is based on linux kernel 6.19-rc6 tree.
Getting the following build failure on linuxppc-dev patchwork with
g5_defconfigorppc64_defconfig:
Quoted 7 lines
Error: /linux/arch/powerpc/sysdev/dart_iommu.c:325:9: error: initialization of 'int (*)(struct
iommu_table*, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int, bool)' {aka 'int (*)(structiommu_table*, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int,_Bool)'} from incompatible pointer type 'int (*)(structiommu_table*, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int)' [-Werror=incompatible-pointer-types] .set =dart_build,
^~~~~~~~~~
/linux/arch/powerpc/sysdev/dart_iommu.c:325:9: note: (near initialization for 'iommu_dart_ops.set')
cc1: all warnings being treated as errors
make[5]: *** [/linux/scripts/Makefile.build:287:arch/powerpc/sysdev/dart_iommu.o] Error 1
make[4]: *** [/linux/scripts/Makefile.build:544:arch/powerpc/sysdev] Error 2
I was trying only pseries and powernv configs. I see the changes would break
pasemi and dart iommus.
The below diff should get it going,
===========================
arch/powerpc/platforms/pasemi/iommu.c +0 -0
@@ -77,7 +77,7 @@ static int iommu_table_iobmap_inited;
static int iobmap_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs)
+ unsigned long attrs, bool is_phys) {
u32 *ip;
u32 rpn;
arch/powerpc/sysdev/dart_iommu.c +0 -0
@@ -171,7 +171,7 @@ static void dart_flush(struct iommu_table *tbl)
static int dart_build(struct iommu_table *tbl, long index, long npages, unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs)
+ unsigned long attrs, bool is_phys) {
unsigned int *dp, *orig_dp;
unsigned int rpn;
===========================
I will take care of it in next version.
Thank you!
Regards,
Shivaprasad
Quoted 4 lines
Christophe
<snip/>
kernel test robot
build botkernel test robot noticed the following build errors:
Hi Shivaprasad,
[This is a private test report for your RFC patch.] kernel test robot noticed the following build errors:
[auto build test ERROR on powerpc/next]
[also build test ERROR on powerpc/fixes awilliam-vfio/next awilliam-vfio/for-linus linus/master v6.19-rc7 next-20260130] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Shivaprasad-G-Bhat/powerpc-iommu-Initial-IOMMUFD-support-for-PPC64/20260128-023838 base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next patch link: https://lore.kernel.org/r/176953894915.725.1102545144304639827.stgit%40linux.ibm.com patch subject: [RFC PATCH] powerpc: iommu: Initial IOMMUFD support for PPC64 config: powerpc64-randconfig-001-20260202 (https://download.01.org/0day-ci/archive/20260202/[email protected]/config) compiler: powerpc64-linux-gcc (GCC) 12.5.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260202/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All error/warnings (new ones prefixed by >>):
>> arch/powerpc/sysdev/dart_iommu.c:325:16: error: initialization of 'int (*)(struct iommu_table *, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int, bool)' {aka 'int (*)(struct iommu_table *, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int, _Bool)'} from incompatible pointer type 'int (*)(struct iommu_table *, long int, long int, long unsigned int, enum dma_data_direction, long unsigned int)' [-Werror=incompatible-pointer-types] 325 | .set = dart_build,
| ^~~~~~~~~~
arch/powerpc/sysdev/dart_iommu.c:325:16: note: (near initialization for 'iommu_dart_ops.set')
cc1: some warnings being treated as errors
--
arch/powerpc/kernel/iommu.c: In function 'spapr_tce_domain_alloc_paging':
>> arch/powerpc/kernel/iommu.c:1298:13: warning: variable 'ret' set but not used [-Wunused-but-set-variable]
1298 | int ret = -1;
| ^~~
arch/powerpc/kernel/iommu.c: In function 'spapr_tce_iommu_unmap_pages':
>> arch/powerpc/kernel/iommu.c:1341:14: warning: variable 'mask' set but not used [-Wunused-but-set-variable]
1341 | int mask;
| ^~~~vim +325 arch/powerpc/sysdev/dart_iommu.c
^1da177e4c3f41 arch/ppc64/kernel/u3_iommu.c Linus Torvalds 2005-04-16 323
da004c3600f52e arch/powerpc/sysdev/dart_iommu.c Alexey Kardashevskiy 2015-06-05 324 static struct iommu_table_ops iommu_dart_ops = {
da004c3600f52e arch/powerpc/sysdev/dart_iommu.c Alexey Kardashevskiy 2015-06-05 @325 .set = dart_build,
da004c3600f52e arch/powerpc/sysdev/dart_iommu.c Alexey Kardashevskiy 2015-06-05 326 .clear = dart_free,
da004c3600f52e arch/powerpc/sysdev/dart_iommu.c Alexey Kardashevskiy 2015-06-05 327 .flush = dart_flush,
da004c3600f52e arch/powerpc/sysdev/dart_iommu.c Alexey Kardashevskiy 2015-06-05 328 };da004c3600f52e arch/powerpc/sysdev/dart_iommu.c Alexey Kardashevskiy 2015-06-05 329
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki