Skip to contentSkip to conversation

iommu: Support IOMMU_RESV_SW_MSI with nesting

PATCH New Nicolin Chen 1 participant 5 messages Aug 7, 2024

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

v3 cover iommu: Support IOMMU_RESV_SW_MSI with nesting

The dma-iommu needs to find the correct domain for MSI mapping. With an IOMMU_DOMAIN_NESTED, the mapping resides in its parent paging domain.

Add a get_msi_mapping_domain op for drivers to return paging domains.

Add an iommufd selftest coverage for that, by doing a loopback test.

Add arm_smmu_get_msi_mapping_domain in the SMMUv3 driver so its nesting feature could work with MSI correctly.

This is based on top of the reserved-IOVA change:
https://lore.kernel.org/all/[email protected]/
And Jason's SMMUv3 nesting series:
https://lore.kernel.org/all/[email protected]/

This series is on Github:
https://github.com/nicolinc/iommufd/commits/iommufd_nesting_sw_msi/

[changelog]
v3:
* Refined PATCH-2 commit message
* Added domain->ops check in PATCH-2
* Added PATCH-4 to implement in SMMUv3 driver
v2:
https://lore.kernel.org/all/[email protected]/
* Resent with a proper bug fix.

Thanks
Nicolin

Nicolin Chen (3):
iommufd: Reorder include files
iommufd/selftest: Add coverage for IOMMU_RESV_SW_MSI iommu/arm-smmu-v3: Implement arm_smmu_get_msi_mapping_domain

Robin Murphy (1):
iommu/dma: Support MSIs through nested domains

 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 10 ++
 drivers/iommu/dma-iommu.c                     | 18 +++-
 drivers/iommu/iommufd/device.c                |  4 +-
 drivers/iommu/iommufd/fault.c                 |  4 +-
 drivers/iommu/iommufd/io_pagetable.c          |  8 +-
 drivers/iommu/iommufd/io_pagetable.h          |  2 +-
 drivers/iommu/iommufd/ioas.c                  |  2 +-
 drivers/iommu/iommufd/iommufd_private.h       |  9 +-
 drivers/iommu/iommufd/iommufd_test.h          |  6 +-
 drivers/iommu/iommufd/iova_bitmap.c           |  2 +-
 drivers/iommu/iommufd/main.c                  |  8 +-
 drivers/iommu/iommufd/pages.c                 | 10 +-
 drivers/iommu/iommufd/selftest.c              | 92 ++++++++++++++++++-
 include/linux/iommu.h                         |  4 +
 include/linux/iommufd.h                       |  4 +-
 include/uapi/linux/iommufd.h                  |  2 +-
 tools/testing/selftests/iommu/iommufd_utils.h |  9 ++
 17 files changed, 160 insertions(+), 34 deletions(-)
Signature · 1 line
2.43.0
#2 lore raw

PATCH v3 2/4 iommu/dma: Support MSIs through nested domains

From: Robin Murphy <[email protected]>

Currently, iommu-dma is the only place outside of IOMMUFD and drivers which might need to be aware of the stage 2 domain encapsulated within a nested domain. This would be still the RMR solution where we're using host-managed MSIs with an identity mapping at stage 1, where it is the underlying stage 2 domain which owns an MSI cookie and holds the corresponding dynamic mappings. Hook up the new op to resolve what we need from a nested domain.

---
 drivers/iommu/dma-iommu.c | 18 ++++++++++++++++--
 include/linux/iommu.h     |  4 ++++
 2 files changed, 20 insertions(+), 2 deletions(-)
drivers/iommu/dma-iommu.c +16 -2
index 7b1dfa0665df..5b21be04e04a 100644
@@ -1799,6 +1799,20 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
 	return NULL;
 }
 
+/*
+ * Nested domains may not have an MSI cookie or accept mappings, but they may
+ * be related to a domain which does, so we let them tell us what they need.
+ */
+static struct iommu_domain *iommu_dma_get_msi_mapping_domain(struct device *dev)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+
+	if (domain && domain->type == IOMMU_DOMAIN_NESTED &&
+	    domain->ops && domain->ops->get_msi_mapping_domain)
+		domain = domain->ops->get_msi_mapping_domain(domain);
+	return domain;
+}
+
 /**
  * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
  * @desc: MSI descriptor, will store the MSI page
@@ -1809,7 +1823,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
 int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
 {
 	struct device *dev = msi_desc_to_dev(desc);
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iommu_domain *domain = iommu_dma_get_msi_mapping_domain(dev);
 	struct iommu_dma_msi_page *msi_page;
 	static DEFINE_MUTEX(msi_prepare_lock); /* see below */
 
@@ -1842,7 +1856,7 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
 void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
 {
 	struct device *dev = msi_desc_to_dev(desc);
-	const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	const struct iommu_domain *domain = iommu_dma_get_msi_mapping_domain(dev);
 	const struct iommu_dma_msi_page *msi_page;
 
 	msi_page = msi_desc_get_iommu_cookie(desc);
include/linux/iommu.h +4 -0
index b0323290cb6c..20a40fd6c4ba 100644
@@ -684,6 +684,8 @@ struct iommu_ops {
  *                           specific mechanisms.
  * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
  * @free: Release the domain after use.
+ * @get_msi_mapping_domain: Return the related iommu_domain that should hold the
+ *                          MSI cookie and accept mapping(s).
  */
 struct iommu_domain_ops {
 	int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
@@ -713,6 +715,8 @@ struct iommu_domain_ops {
 				  unsigned long quirks);
 
 	void (*free)(struct iommu_domain *domain);
+	struct iommu_domain *
+		(*get_msi_mapping_domain)(struct iommu_domain *domain);
 };
 
 /**
Signature · 1 line
2.43.0
#1 lore raw

PATCH v3 1/4 iommufd: Reorder include files

In reply to Nicolin Chen

Reorder include files to alphabetic order to simplify maintenance, and separate local headers and global headers with a blank line.

No functional change intended.

---
 drivers/iommu/iommufd/device.c          |  4 ++--
 drivers/iommu/iommufd/fault.c           |  4 ++--
 drivers/iommu/iommufd/io_pagetable.c    |  8 ++++----
 drivers/iommu/iommufd/io_pagetable.h    |  2 +-
 drivers/iommu/iommufd/ioas.c            |  2 +-
 drivers/iommu/iommufd/iommufd_private.h |  9 +++++----
 drivers/iommu/iommufd/iommufd_test.h    |  2 +-
 drivers/iommu/iommufd/iova_bitmap.c     |  2 +-
 drivers/iommu/iommufd/main.c            |  8 ++++----
 drivers/iommu/iommufd/pages.c           | 10 +++++-----
 drivers/iommu/iommufd/selftest.c        |  8 ++++----
 include/linux/iommufd.h                 |  4 ++--
 include/uapi/linux/iommufd.h            |  2 +-
 13 files changed, 33 insertions(+), 32 deletions(-)
drivers/iommu/iommufd/device.c +2 -2
index bcbcd6d94062..5fd3dd420290 100644
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
  */
+#include <linux/iommu.h>
 #include <linux/iommufd.h>
 #include <linux/slab.h>
-#include <linux/iommu.h>
 #include <uapi/linux/iommufd.h>
-#include "../iommu-priv.h"
 
+#include "../iommu-priv.h"
 #include "io_pagetable.h"
 #include "iommufd_private.h"
 
drivers/iommu/iommufd/fault.c +2 -2
index a643d5c7c535..df03411c8728 100644
@@ -3,14 +3,14 @@
  */
 #define pr_fmt(fmt) "iommufd: " fmt
 
+#include <linux/anon_inodes.h>
 #include <linux/file.h>
 #include <linux/fs.h>
+#include <linux/iommufd.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/iommufd.h>
 #include <linux/pci.h>
 #include <linux/poll.h>
-#include <linux/anon_inodes.h>
 #include <uapi/linux/iommufd.h>
 
 #include "../iommu-priv.h"
drivers/iommu/iommufd/io_pagetable.c +4 -4
index 05fd9d3abf1b..bbbc8a044bcf 100644
@@ -8,17 +8,17 @@
  * The datastructure uses the iopt_pages to optimize the storage of the PFNs
  * between the domains and xarray.
  */
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/iommu.h>
 #include <linux/iommufd.h>
 #include <linux/lockdep.h>
-#include <linux/iommu.h>
 #include <linux/sched/mm.h>
-#include <linux/err.h>
 #include <linux/slab.h>
-#include <linux/errno.h>
 #include <uapi/linux/iommufd.h>
 
-#include "io_pagetable.h"
 #include "double_span.h"
+#include "io_pagetable.h"
 
 struct iopt_pages_list {
 	struct iopt_pages *pages;
drivers/iommu/iommufd/io_pagetable.h +1 -1
index 0ec3509b7e33..c61d74471684 100644
@@ -6,8 +6,8 @@
 #define __IO_PAGETABLE_H
 
 #include <linux/interval_tree.h>
-#include <linux/mutex.h>
 #include <linux/kref.h>
+#include <linux/mutex.h>
 #include <linux/xarray.h>
 
 #include "iommufd_private.h"
drivers/iommu/iommufd/ioas.c +1 -1
index 742248276548..82428e44a837 100644
@@ -3,8 +3,8 @@
  * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
  */
 #include <linux/interval_tree.h>
-#include <linux/iommufd.h>
 #include <linux/iommu.h>
+#include <linux/iommufd.h>
 #include <uapi/linux/iommufd.h>
 
 #include "io_pagetable.h"
drivers/iommu/iommufd/iommufd_private.h +5 -4
index 0bfacaf40c05..5d3768d77099 100644
@@ -4,13 +4,14 @@
 #ifndef __IOMMUFD_PRIVATE_H
 #define __IOMMUFD_PRIVATE_H
 
-#include <linux/rwsem.h>
-#include <linux/xarray.h>
-#include <linux/refcount.h>
-#include <linux/uaccess.h>
 #include <linux/iommu.h>
 #include <linux/iova_bitmap.h>
+#include <linux/refcount.h>
+#include <linux/rwsem.h>
+#include <linux/uaccess.h>
+#include <linux/xarray.h>
 #include <uapi/linux/iommufd.h>
+
 #include "../iommu-priv.h"
 
 struct iommu_domain;
drivers/iommu/iommufd/iommufd_test.h +1 -1
index acbbba1c6671..f4bc23a92f9a 100644
@@ -4,8 +4,8 @@
 #ifndef _UAPI_IOMMUFD_TEST_H
 #define _UAPI_IOMMUFD_TEST_H
 
-#include <linux/types.h>
 #include <linux/iommufd.h>
+#include <linux/types.h>
 
 enum {
 	IOMMU_TEST_OP_ADD_RESERVED = 1,
drivers/iommu/iommufd/iova_bitmap.c +1 -1
index b9e964b1ad5c..d90b9e253412 100644
@@ -3,10 +3,10 @@
  * Copyright (c) 2022, Oracle and/or its affiliates.
  * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
  */
+#include <linux/highmem.h>
 #include <linux/iova_bitmap.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
-#include <linux/highmem.h>
 
 #define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE)
 
drivers/iommu/iommufd/main.c +4 -4
index 83bbd7c5d160..b5f5d27ee963 100644
@@ -8,15 +8,15 @@
  */
 #define pr_fmt(fmt) "iommufd: " fmt
 
+#include <linux/bug.h>
 #include <linux/file.h>
 #include <linux/fs.h>
-#include <linux/module.h>
-#include <linux/slab.h>
+#include <linux/iommufd.h>
 #include <linux/miscdevice.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/bug.h>
+#include <linux/slab.h>
 #include <uapi/linux/iommufd.h>
-#include <linux/iommufd.h>
 
 #include "io_pagetable.h"
 #include "iommufd_private.h"
drivers/iommu/iommufd/pages.c +5 -5
index 117f644a0c5b..93d806c9c073 100644
@@ -45,16 +45,16 @@
  * last_iova + 1 can overflow. An iopt_pages index will always be much less than
  * ULONG_MAX so last_index + 1 cannot overflow.
  */
+#include <linux/highmem.h>
+#include <linux/iommu.h>
+#include <linux/iommufd.h>
+#include <linux/kthread.h>
 #include <linux/overflow.h>
 #include <linux/slab.h>
-#include <linux/iommu.h>
 #include <linux/sched/mm.h>
-#include <linux/highmem.h>
-#include <linux/kthread.h>
-#include <linux/iommufd.h>
 
-#include "io_pagetable.h"
 #include "double_span.h"
+#include "io_pagetable.h"
 
 #ifndef CONFIG_IOMMUFD_TEST
 #define TEMP_MEMORY_LIMIT 65536
drivers/iommu/iommufd/selftest.c +4 -4
index f95e32e29133..b60687f57bef 100644
@@ -3,13 +3,13 @@
  *
  * Kernel side components to support tools/testing/selftests/iommu
  */
-#include <linux/slab.h>
-#include <linux/iommu.h>
-#include <linux/xarray.h>
-#include <linux/file.h>
 #include <linux/anon_inodes.h>
 #include <linux/fault-inject.h>
+#include <linux/file.h>
+#include <linux/iommu.h>
 #include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/xarray.h>
 #include <uapi/linux/iommufd.h>
 
 #include "../iommu-priv.h"
include/linux/iommufd.h +2 -2
index ffc3a949f837..c2f2f6b9148e 100644
@@ -6,9 +6,9 @@
 #ifndef __LINUX_IOMMUFD_H
 #define __LINUX_IOMMUFD_H
 
-#include <linux/types.h>
-#include <linux/errno.h>
 #include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/types.h>
 
 struct device;
 struct iommufd_device;
include/uapi/linux/iommufd.h +1 -1
index f2d1677ddec4..d42e0471156f 100644
@@ -4,8 +4,8 @@
 #ifndef _UAPI_IOMMUFD_H
 #define _UAPI_IOMMUFD_H
 
-#include <linux/types.h>
 #include <linux/ioctl.h>
+#include <linux/types.h>
 
 #define IOMMUFD_TYPE (';')
 
Signature · 1 line
2.43.0
#3 lore raw

PATCH v3 3/4 iommufd/selftest: Add coverage for IOMMU_RESV_SW_MSI

In reply to Nicolin Chen

Define an IOMMU_RESV_SW_MSI region (within the mock aperture) as the ARM SMMU drivers do.

Implement the get_msi_mapping_domain nested domain op to allow dma-iommu to find the correct paging domain.

Add a new IOMMU_TEST_OP_MD_CHECK_SW_MSI to loopback test the MSI mapping using public dma-iommu and iommu helpers.

Note that iommu_fwspec_init is a must for iommu_dma_get_resv_regions().

---
 drivers/iommu/iommufd/iommufd_test.h          |  4 +
 drivers/iommu/iommufd/selftest.c              | 84 +++++++++++++++++++
 tools/testing/selftests/iommu/iommufd_utils.h |  9 ++
 3 files changed, 97 insertions(+)
drivers/iommu/iommufd/iommufd_test.h +4 -0
index f4bc23a92f9a..0bb30275a92f 100644
@@ -23,6 +23,7 @@ enum {
 	IOMMU_TEST_OP_DIRTY,
 	IOMMU_TEST_OP_MD_CHECK_IOTLB,
 	IOMMU_TEST_OP_TRIGGER_IOPF,
+	IOMMU_TEST_OP_MD_CHECK_SW_MSI,
 };
 
 enum {
@@ -135,6 +136,9 @@ struct iommu_test_cmd {
 			__u32 perm;
 			__u64 addr;
 		} trigger_iopf;
+		struct {
+			__u32 stdev_id;
+		} check_sw_msi;
 	};
 	__u32 last;
 };
drivers/iommu/iommufd/selftest.c +84 -0

135 lines not on this page · raw message

tools/testing/selftests/iommu/iommufd_utils.h +9 -0
index 40f6f14ce136..bc4556a48d82 100644
@@ -130,6 +130,13 @@ static int _test_cmd_mock_domain_flags(int fd, unsigned int ioas_id,
 static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id,
 					 __u32 *hwpt_id)
 {
+	struct iommu_test_cmd sw_msi = {
+		.size = sizeof(sw_msi),
+		.op = IOMMU_TEST_OP_MD_CHECK_SW_MSI,
+		.check_sw_msi = {
+			.stdev_id = stdev_id,
+		},
+	};
 	struct iommu_test_cmd cmd = {
 		.size = sizeof(cmd),
 		.op = IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE,
@@ -145,6 +152,8 @@ static int _test_cmd_mock_domain_replace(int fd, __u32 stdev_id, __u32 pt_id,
 		return ret;
 	if (hwpt_id)
 		*hwpt_id = cmd.mock_domain_replace.pt_id;
+	sw_msi.id = cmd.mock_domain_replace.pt_id;
+	assert(ioctl(fd, IOMMU_TEST_CMD, &sw_msi) == 0);
 	return 0;
 }
 
Signature · 1 line
2.43.0
#4 lore raw

PATCH v3 4/4 iommu/arm-smmu-v3: Implement arm_smmu_get_msi_mapping_domain

In reply to Nicolin Chen

In a 1-stage translation setup, a device is attached to a paging domain. In a 2-stage translation setup, a device is attached to a nested domain, which does not have the mappings for the MSI page but only an s2_parent paging domain pointer that holds the mappings.

Add arm_smmu_get_msi_mapping_domain in arm_smmu_nested_ops to return the correct paging domain.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 10 ++++++++++
 1 file changed, 10 insertions(+)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +10 -0
index 24836f3269b3..3962d5a55519 100644
@@ -3305,10 +3305,20 @@ static int arm_smmu_cache_invalidate_user(struct iommu_domain *domain,
 	return ret;
 }
 
+static struct iommu_domain *
+arm_smmu_get_msi_mapping_domain(struct iommu_domain *domain)
+{
+	struct arm_smmu_nested_domain *nested_domain =
+		container_of(domain, struct arm_smmu_nested_domain, domain);
+
+	return &nested_domain->s2_parent->domain;
+}
+
 static const struct iommu_domain_ops arm_smmu_nested_ops = {
 	.attach_dev = arm_smmu_attach_dev_nested,
 	.free = arm_smmu_domain_nested_free,
 	.cache_invalidate_user	= arm_smmu_cache_invalidate_user,
+	.get_msi_mapping_domain	= arm_smmu_get_msi_mapping_domain,
 };
 
 static struct iommu_domain *
Signature · 1 line
2.43.0