iommu/arm-smmu-v3: Support guest-level Realm VSMMU (Part-1)

index 3f7a6a833993b..a1836b19c1556 100644
@@ -236,4 +236,33 @@ static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule,
 	return res.a0;
 }
 
+static inline unsigned long rsi_vsmmu_get_info(phys_addr_t addr,
+					       phys_addr_t *top)
+{
+	struct arm_smccc_1_2_regs regs = {
+		.a0 = SMC_RSI_VSMMU_GET_INFO,
+		.a1 = addr,
+	};
+
+	arm_smccc_1_2_smc(&regs, &regs);
+	if (regs.a0 == RSI_SUCCESS && top)
+		*top = regs.a1;
+
+	return regs.a0;
+}
+
+static inline unsigned long rsi_arch_dev_activate(phys_addr_t base,
+						  unsigned long dev_type)
+{
+	struct arm_smccc_1_2_regs regs = {
+		.a0 = SMC_RSI_ARCH_DEV_ACTIVATE,
+		.a1 = base,
+		.a2 = dev_type,
+	};
+
+	arm_smccc_1_2_smc(&regs, &regs);
+
+	return regs.a0;
+}
+
 #endif /* __LINUX_ARM_RSI_CMDS_H_ */

index cca881ddfdbab..fff0073622153 100644
@@ -37,6 +37,7 @@
 #define RSI_ERROR_STATE		UL(2)
 #define RSI_INCOMPLETE		UL(3)
 #define RSI_ERROR_UNKNOWN	UL(4)
+#define RSI_ERROR_DEVICE	UL(5)
 
 #define SMC_RSI_FID(n)		ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,      \
 						   ARM_SMCCC_SMC_64,         \

@@ -190,4 +191,23 @@ struct realm_config {
  */
 #define SMC_RSI_HOST_CALL			SMC_RSI_FID(0x199)
 
+/*
+ * Get information about a VSMMU.
+ *
+ * arg1 == Base IPA of the VSMMU
+ * ret0 == Status / error
+ * ret1 == Top IPA of the VSMMU
+ */
+#define SMC_RSI_VSMMU_GET_INFO			SMC_RSI_FID(0x19A)
+
+#define RSI_ARCH_DEV_SMMUV3			UL(0)
+
+/*
+ * Activate an architectural device.
+ *
+ * arg1 == Base IPA of the architectural device
+ * arg2 == Architectural device type
+ * ret0 == Status / error
+ */
+#define SMC_RSI_ARCH_DEV_ACTIVATE		SMC_RSI_FID(0x19B)
 #endif /* __LINUX_ARM_SMCCC_RSI_H_ */

index 429e39f647cf9..01d7877d0b571 100644
@@ -604,6 +604,8 @@ struct device_physical_location {
  *		ancestor device.
  * @DEV_FLAG_OFFLINE_DISABLED: If set, the device is permanently online.
  * @DEV_FLAG_OFFLINE: Set after successful invocation of bus type's .offline().
+ * @DEV_FLAG_DMA_CC_PRIVATE: The device is able to access private (encrypted)
+ *		memory, no shared memory bouncing is required from the DMA API.
  * @DEV_FLAG_COUNT: Number of defined struct_device_flags.
  */
 enum struct_device_flags {

@@ -617,6 +619,7 @@ enum struct_device_flags {
 	DEV_FLAG_OF_NODE_REUSED = 7,
 	DEV_FLAG_OFFLINE_DISABLED = 8,
 	DEV_FLAG_OFFLINE = 9,
+	DEV_FLAG_DMA_CC_PRIVATE = 10,
 
 	DEV_FLAG_COUNT
 };

@@ -834,6 +837,7 @@ __create_dev_flag_accessors(dma_coherent, DEV_FLAG_DMA_COHERENT);
 __create_dev_flag_accessors(of_node_reused, DEV_FLAG_OF_NODE_REUSED);
 __create_dev_flag_accessors(offline_disabled, DEV_FLAG_OFFLINE_DISABLED);
 __create_dev_flag_accessors(offline, DEV_FLAG_OFFLINE);
+__create_dev_flag_accessors(dma_cc_private, DEV_FLAG_DMA_CC_PRIVATE);
 
 #undef __create_dev_flag_accessors
 

index efde0e5a58dc2..3cc38db4a7490 100644
@@ -218,6 +218,7 @@ void *dma_vmap_noncontiguous(struct device *dev, size_t size,
 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr);
 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma,
 		size_t size, struct sg_table *sgt);
+void dma_set_cc_private(struct device *dev, bool private);
 #else /* CONFIG_HAS_DMA */
 static inline dma_addr_t dma_map_page_attrs(struct device *dev,
 		struct page *page, size_t offset, size_t size,

@@ -357,6 +358,9 @@ static inline int dma_mmap_noncontiguous(struct device *dev,
 {
 	return -EINVAL;
 }
+static inline void dma_set_cc_private(struct device *dev, bool private)
+{
+}
 #endif /* CONFIG_HAS_DMA */
 
 #ifdef CONFIG_IOMMU_DMA

index 82e701648d604..ded585b39fb92 100644
@@ -501,6 +501,27 @@ static void dma_setup_need_sync(struct device *dev)
 static inline void dma_setup_need_sync(struct device *dev) { }
 #endif /* !CONFIG_DMA_NEED_SYNC */
 
+/**
+ * dma_set_cc_private - Set whether @dev can DMA to private memory
+ * @dev: device whose confidential-computing DMA access is being updated
+ * @private: whether the device can DMA to private memory
+ *
+ * Called when a device is known to be able to DMA to private memory.
+ * For architected CPU integrated devices their kernel drivers will
+ * self-accept using a driver specific validation. Other devices will
+ * have a userspace managed process.
+ *
+ * DMA mapping must not be active when this flag is changed. For CPU
+ * integrated devices the driver should self accept early during probe
+ * after validation. Other devices have this flag set automatically
+ * before probing.
+ */
+void dma_set_cc_private(struct device *dev, bool private)
+{
+	dev_assign_dma_cc_private(dev, private);
+}
+EXPORT_SYMBOL_GPL(dma_set_cc_private);
+
 /*
  * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems
  * that the intention is to allow exporting memory allocated via the

index daa31a1adf7b0..9d548e545f7a6 100644
@@ -134,6 +134,11 @@ static inline bool force_dma_unencrypted(struct device *dev)
 }
 #endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
 
+static inline bool dma_require_unencrypted(struct device *dev)
+{
+	return !dev_dma_cc_private(dev) && force_dma_unencrypted(dev);
+}
+
 static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
 		bool is_ram, unsigned long attrs)
 {

@@ -146,7 +151,7 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
 	 * requires unencrypted DMA addresses. Treat it as not DMA-capable
 	 * so the caller can fall back to a suitable SWIOTLB pool.
 	 */
-	if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev))
+	if (!(attrs & DMA_ATTR_CC_SHARED) && dma_require_unencrypted(dev))
 		return false;
 
 	if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&

index 25e9a86835fd3..29ea832b9f0af 100644
@@ -41,9 +41,9 @@ static inline struct page *dma_direct_to_page(struct device *dev,
 
 u64 dma_direct_get_required_mask(struct device *dev)
 {
-	bool require_decrypted = force_dma_unencrypted(dev);
+	bool require_unencrypted = dma_require_unencrypted(dev);
 	phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1;
-	u64 max_dma = phys_to_dma_direct(dev, phys, require_decrypted);
+	u64 max_dma = phys_to_dma_direct(dev, phys, require_unencrypted);
 
 	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
 }

@@ -72,8 +72,9 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit)
 
 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
 {
-	bool require_decrypted = force_dma_unencrypted(dev);
-	dma_addr_t dma_addr = phys_to_dma_direct(dev, phys, require_decrypted);
+	bool require_unencrypted = dma_require_unencrypted(dev);
+	dma_addr_t dma_addr = phys_to_dma_direct(dev, phys,
+						 require_unencrypted);
 
 	if (dma_addr == DMA_MAPPING_ERROR)
 		return false;

@@ -226,7 +227,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	struct page *page;
 	void *cpu_addr;
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_unencrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) {

@@ -385,7 +386,7 @@ void dma_direct_free(struct device *dev, size_t size,
 	 * If the allocation used decrypted/shared backing pages, restore
 	 * the encryption state on free.
 	 */
-	if (force_dma_unencrypted(dev))
+	if (dma_require_unencrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)

@@ -452,7 +453,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 	struct page *page;
 	void *cpu_addr;
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_unencrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))

@@ -496,7 +497,7 @@ void dma_direct_free_pages(struct device *dev, size_t size,
 	 * if the device had requested for an unencrypted buffer,
 	 * convert it to encrypted on free
 	 */
-	bool mark_mem_encrypted = force_dma_unencrypted(dev);
+	bool mark_mem_encrypted = dma_require_unencrypted(dev);
 
 	/* If page is not from an atomic pool, dma_free_from_pool_page() fails */
 	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&

@@ -665,7 +666,7 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
 	const pgoff_t pgoff_end = vma_end_pgoff(vma);
 	int ret = -ENXIO;
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_unencrypted(dev))
 		attrs |= DMA_ATTR_CC_SHARED;
 
 	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);

@@ -814,7 +815,7 @@ size_t dma_direct_max_mapping_size(struct device *dev)
 	/* If SWIOTLB is active, use its maximum mapping size */
 	if (is_swiotlb_active(dev) &&
 	    (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev) ||
-	     force_dma_unencrypted(dev)))
+	     dma_require_unencrypted(dev)))
 		return swiotlb_max_mapping_size(dev);
 
 	return SIZE_MAX;

index ded585b39fb92..cd405309358e7 100644
@@ -673,7 +673,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 		return NULL;
 	}
 
-	if (force_dma_unencrypted(dev))
+	if (dma_require_unencrypted(dev))
 		attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
 
 	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {

index 2c57b49ac7c06..29ecfdab22957 100644
@@ -1550,7 +1550,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
 
 		/* swiotlb pool is incorrect for this device */
-		if (unlikely(mem->cc_shared != force_dma_unencrypted(dev)))
+		if (unlikely(mem->cc_shared != dma_require_unencrypted(dev)))
 			return (phys_addr_t)DMA_MAPPING_ERROR;
 
 	} else if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {

index ef5043752a365..e8fb5ee59883e 100644
@@ -824,6 +824,8 @@ struct iommu_domain_ops {
  * @singleton_group: Used internally for drivers that have only one group
  * @max_pasids: number of supported PASIDs
  * @ready: set once iommu_device_register() has completed successfully
+ * @confidential: this instance runs inside a confidential VM. Set by the driver
+ *                before any dma allocation.
  */
 struct iommu_device {
 	struct list_head list;

@@ -833,6 +835,7 @@ struct iommu_device {
 	struct iommu_group *singleton_group;
 	u32 max_pasids;
 	bool ready;
+	bool confidential;
 };
 
 /**

@@ -892,6 +895,8 @@ struct dev_iommu {
 int iommu_device_register(struct iommu_device *iommu,
 			  const struct iommu_ops *ops,
 			  struct device *hwdev);
+void iommu_device_set_confidential(struct iommu_device *iommu,
+				   struct device *hwdev);
 void iommu_device_unregister(struct iommu_device *iommu);
 int  iommu_device_sysfs_add(struct iommu_device *iommu,
 			    struct device *parent,

@@ -1423,6 +1428,11 @@ static inline int iommu_device_register(struct iommu_device *iommu,
 	return -ENODEV;
 }
 
+static inline void iommu_device_set_confidential(struct iommu_device *iommu,
+						 struct device *hwdev)
+{
+}
+
 static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
 {
 	return NULL;

index b486b8bbd1fcd..ba98a0bb800a5 100644
@@ -17,6 +17,7 @@
 #include <linux/slab.h>
 #include <linux/errno.h>
 #include <linux/host1x_context_bus.h>
+#include <linux/dma-mapping.h>
 #include <linux/iommu.h>
 #include <linux/iommufd.h>
 #include <linux/idr.h>

@@ -313,6 +314,25 @@ int iommu_device_register(struct iommu_device *iommu,
 }
 EXPORT_SYMBOL_GPL(iommu_device_register);
 
+/**
+ * iommu_device_set_confidential - Mark an IOMMU instance as confidential
+ * @iommu: the IOMMU instance
+ * @hwdev: the struct device of @iommu, whose own DMA reaches private memory
+ *
+ * Declare that @iommu runs inside a confidential VM
+ *
+ * Call this before @iommu makes any DMA allocation of its own, and not merely
+ * before iommu_device_register(). Queues and tables allocated any earlier land
+ * in shared memory that the hypervisor can read.
+ */
+void iommu_device_set_confidential(struct iommu_device *iommu,
+				   struct device *hwdev)
+{
+	iommu->confidential = true;
+	dma_set_cc_private(hwdev, true);
+}
+EXPORT_SYMBOL_GPL(iommu_device_set_confidential);
+
 void iommu_device_unregister(struct iommu_device *iommu)
 {
 	for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++)

index 1f9f886b2c107..624466384b6e3 100644
@@ -8,6 +8,25 @@
 #include <linux/iommu-debug-pagealloc.h>
 #include <linux/msi.h>
 
+#ifdef CONFIG_ARCH_HAS_CC_PLATFORM
+void iommu_tdisp_enter_t0(struct device *dev);
+void iommu_tdisp_exit_t0(struct device *dev);
+#else
+static inline void iommu_tdisp_enter_t0(struct device *dev)
+{
+}
+
+static inline void iommu_tdisp_exit_t0(struct device *dev)
+{
+}
+#endif
+
+static inline bool iommu_using_t0_stream(struct device *dev)
+{
+	return dev->iommu->iommu_dev->confidential &&
+	       !dev->iommu->tdisp_t1;
+}
+
 static inline const struct iommu_ops *dev_iommu_ops(struct device *dev)
 {
 	/*

index e8fb5ee59883e..1b5817a7d0be9 100644
@@ -873,6 +873,9 @@ struct iommu_fault_param {
  * @max_pasids:  number of PASIDs this device can consume
  * @attach_deferred: the dma domain attachment is deferred
  * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs
+ * @tdisp_t1: device uses its confidential IOMMU's T=1 stream. A zero value
+ *            means T=0, where the T=1 stream remains blocked while the core
+ *            owns required DMA operations such as ATS.
  * @require_direct: device requires IOMMU_RESV_DIRECT regions
  * @shadow_on_flush: IOTLB flushes are used to sync shadow tables
  *

@@ -888,6 +891,7 @@ struct dev_iommu {
 	u32				max_pasids;
 	u32				attach_deferred:1;
 	u32				pci_32bit_workaround:1;
+	u32				tdisp_t1:1;
 	u32				require_direct:1;
 	u32				shadow_on_flush:1;
 };

63 lines not on this page · raw message

index ba98a0bb800a5..a7464e155e55a 100644
@@ -535,9 +535,17 @@ static int iommu_init_device(struct device *dev)
 	}
 	dev->iommu->iommu_dev = iommu_dev;
 
+	if (iommu_using_t0_stream(dev)) {
+		/*
+		 * After probe the iommu has to leave the T=1 stream in BLOCKING,
+		 * while the T=0 stream will go through the physical path.
+		 */
+		iommu_tdisp_enter_t0(dev);
+	}
+
 	ret = iommu_device_link(iommu_dev, dev);
 	if (ret)
-		goto err_release;
+		goto err_tdisp_t0;
 
 	group = ops->device_group(dev);
 	if (WARN_ON_ONCE(group == NULL))

@@ -555,7 +563,9 @@ static int iommu_init_device(struct device *dev)
 
 err_unlink:
 	iommu_device_unlink(iommu_dev, dev);
-err_release:
+err_tdisp_t0:
+	if (iommu_using_t0_stream(dev))
+		iommu_tdisp_exit_t0(dev);
 	if (ops->release_device)
 		ops->release_device(dev);
 err_module_put:

@@ -573,6 +583,9 @@ static void iommu_deinit_device(struct device *dev)
 
 	lockdep_assert_held(&group->mutex);
 
+	if (iommu_using_t0_stream(dev))
+		iommu_tdisp_exit_t0(dev);
+
 	iommu_device_unlink(dev->iommu->iommu_dev, dev);
 
 	/*

@@ -4087,6 +4100,15 @@ int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
 		}
 	}
 
+	/*
+	 * A T=0 device stays in the blocking domain while the core keeps ATS on
+	 * for it. So the attach above does not disable ATS. Disable it here, as
+	 * the note above recommends. pci_dev_reset_iommu_done() will enable it
+	 * again.
+	 */
+	if (iommu_using_t0_stream(&pdev->dev))
+		iommu_tdisp_exit_t0(&pdev->dev);
+
 	/*
 	 * Update gdev->blocked upon the domain change, as it is used to return
 	 * the correct domain in iommu_driver_get_domain_for_dev() that might be

@@ -4213,6 +4235,10 @@ void pci_dev_reset_iommu_done(struct pci_dev *pdev)
 					      group->blocking_domain));
 	}
 
+	/* Enable ATS again for a T=0 device after the reset */
+	if (iommu_using_t0_stream(&pdev->dev))
+		iommu_tdisp_enter_t0(&pdev->dev);
+
 	/*
 	 * Update gdev->blocked upon the domain change, as it is used to return
 	 * the correct domain in iommu_driver_get_domain_for_dev() that might be

index a7464e155e55a..992aeee8f9294 100644
@@ -1238,6 +1238,14 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain,
 	unsigned long pg_size;
 	int ret = 0;
 
+	/*
+	 * The direct-map reserved regions are still fine because the T=0 stream
+	 * always goes to an identity mapping. Skip them: recording the request
+	 * would only refuse the device its blocking domain.
+	 */
+	if (iommu_using_t0_stream(dev))
+		return 0;
+
 	pg_size = domain->pgsize_bitmap ? 1UL << __ffs(domain->pgsize_bitmap) : 0;
 
 	if (WARN_ON_ONCE(iommu_is_dma_domain(domain) && !pg_size))

@@ -1790,11 +1798,22 @@ __iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
 static struct iommu_domain *
 iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)
 {
-	const struct iommu_ops *ops = dev_iommu_ops(iommu_group_first_dev(group));
+	struct device *first = iommu_group_first_dev(group);
+	const struct iommu_ops *ops = dev_iommu_ops(first);
 	struct iommu_domain *dom;
 
 	lockdep_assert_held(&group->mutex);
 
+	/*
+	 * When in T=0 mode the T=1 vIOMMU must always be set to BLOCKED to
+	 * release control of ATS.
+	 */
+	if (iommu_using_t0_stream(first)) {
+		if (!ops->blocked_domain)
+			return ERR_PTR(-EINVAL);
+		return ops->blocked_domain;
+	}
+
 	/*
 	 * Allow legacy drivers to specify the domain that will be the default
 	 * domain. This should always be either an IDENTITY/BLOCKED/PLATFORM

@@ -2399,6 +2418,10 @@ static int __iommu_attach_group(struct iommu_domain *domain,
 	    !domain_iommu_ops_compatible(dev_iommu_ops(dev), domain))
 		return -EINVAL;
 
+	/* A domain cannot be attached while using the T=0 stream. */
+	if (iommu_using_t0_stream(dev))
+		return -EINVAL;
+
 	return __iommu_group_set_domain(group, domain);
 }
 

@@ -3325,6 +3348,12 @@ static ssize_t iommu_group_store_type(struct iommu_group *group,
 		goto out_unlock;
 	}
 
+	/* Default domain changes are not allowed while using the T=0 stream. */
+	if (iommu_using_t0_stream(iommu_group_first_dev(group))) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
 	ret = iommu_setup_default_domain(group, req_type);
 	if (ret)
 		goto out_unlock;

@@ -3690,6 +3719,12 @@ int iommu_attach_device_pasid(struct iommu_domain *domain,
 		goto out_unlock;
 	}
 
+	/* PASID cannot be used on the T=0 stream. */
+	if (iommu_using_t0_stream(dev)) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
 	for_each_group_device(group, device) {
 		/*
 		 * Skip PASID validation for devices without PASID support

@@ -3783,6 +3818,12 @@ int iommu_replace_device_pasid(struct iommu_domain *domain,
 		goto out_unlock;
 	}
 
+	/* PASID cannot be used on the T=0 stream. */
+	if (iommu_using_t0_stream(dev)) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
 	entry = iommu_make_pasid_array_entry(domain, handle);
 	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL,
 			  XA_ZERO_ENTRY, GFP_KERNEL);

@@ -4017,6 +4058,12 @@ int iommu_replace_group_handle(struct iommu_group *group,
 		return -EINVAL;
 
 	mutex_lock(&group->mutex);
+	/* Like attach, replacement is not allowed when using the T=0 stream. */
+	if (iommu_using_t0_stream(iommu_group_first_dev(group))) {
+		ret = -EINVAL;
+		goto err_unlock;
+	}
+
 	entry = iommu_make_pasid_array_entry(new_domain, handle);
 	ret = xa_reserve(&group->pasid_array, IOMMU_NO_PASID, GFP_KERNEL);
 	if (ret)

@@ -4294,8 +4341,9 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
 		return ret;
 
 	mutex_lock(&group->mutex);
-	/* An IDENTITY domain must pass through */
-	if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
+	/* An IDENTITY domain or a T=0 device must pass through */
+	if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY &&
+	    !iommu_using_t0_stream(dev)) {
 		switch (group->domain->cookie_type) {
 		case IOMMU_COOKIE_DMA_MSI:
 		case IOMMU_COOKIE_DMA_IOVA:

index 6d4520c473086..f3a9436db15e4 100644
@@ -11,9 +11,11 @@
 
 #include <linux/acpi.h>
 #include <linux/acpi_iort.h>
+#include <linux/arm-rsi-cmds.h>
 #include <linux/bitops.h>
 #include <linux/crash_dump.h>
 #include <linux/delay.h>
+#include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
 #include <linux/io-pgtable.h>

@@ -5611,8 +5613,11 @@ static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
 		/*
 		 * Tegra241 implementation stores its SMMU options and impl_dev
 		 * in DSDT. Thus, go through the ACPI tables unconditionally.
+		 *
+		 * Realm doesn't support Tegra CMDQV yet.
 		 */
-		acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
+		if (!is_realm_world())
+			acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
 		break;
 	}
 

@@ -5676,7 +5681,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev,
 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
 
 	if (of_device_is_compatible(dev->of_node, "nvidia,tegra264-smmu")) {
-		tegra_cmdqv_dt_probe(dev->of_node, smmu);
+		/* Realm doesn't support Tegra CMDQV yet */
+		if (!is_realm_world())
+			tegra_cmdqv_dt_probe(dev->of_node, smmu);
 		static_branch_enable(&arm_smmu_erratum_repeat_tlbi_cfgi_key);
 	}
 

@@ -5779,6 +5786,53 @@ static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
 	return ERR_PTR(ret);
 }
 
+static void arm_smmu_clear_realm_private(void *dev)
+{
+	dma_set_cc_private(dev, false);
+}
+
+static int arm_smmu_probe_realm_vsmmu(struct arm_smmu_device *smmu,
+				      const struct resource *res)
+{
+	struct device *dev = smmu->dev;
+	unsigned long rsi_ret;
+	phys_addr_t top;
+
+	/*
+	 * Currently the driver does not support a T=0 SMMU inside a realm. For
+	 * instance it does not make the page table allocations into shared
+	 * memory. If we are in a realm reject any SMMU that is not T=1.
+	 */
+	rsi_ret = rsi_vsmmu_get_info(res->start, &top);
+	if (rsi_ret != RSI_SUCCESS) {
+		dev_err(dev, "RSI_VSMMU_GET_INFO failed for %pr: %lu\n", res,
+			rsi_ret);
+		return -ENODEV;
+	}
+
+	if (top != res->end + 1) {
+		dev_err(dev, "VSMMU range %pr ends at %pa\n", res, &top);
+		return -EINVAL;
+	}
+
+	rsi_ret = rsi_arch_dev_activate(res->start, RSI_ARCH_DEV_SMMUV3);
+	if (rsi_ret != RSI_SUCCESS) {
+		dev_err(dev, "RSI_ARCH_DEV_ACTIVATE failed for %pr: %lu\n", res,
+			rsi_ret);
+		return -EIO;
+	}
+
+	/*
+	 * Once activated, the SMMU DMA follows DEV_FLAG_DMA_CC_PRIVATE so its
+	 * queues and tables are allocated from private memory, not the SWIOTLB
+	 * shared pool. It only translates for a device we've requested the RMM
+	 * to put into T=1.
+	 */
+	iommu_device_set_confidential(&smmu->iommu, dev);
+
+	return devm_add_action_or_reset(dev, arm_smmu_clear_realm_private, dev);
+}
+
 static int arm_smmu_device_probe(struct platform_device *pdev)
 {
 	int irq, ret;

@@ -5814,6 +5868,12 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 	}
 	ioaddr = res->start;
 
+	if (is_realm_world()) {
+		ret = arm_smmu_probe_realm_vsmmu(smmu, res);
+		if (ret)
+			return ret;
+	}
+
 	/*
 	 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
 	 * the PMCG registers which are reserved by the PMU driver.