arm64: IOMMU-backed DMA mapping

Taking inspiration from the existing arch/arm code, break out some generic functions to interface the DMA-API to the IOMMU-API. This will do the bulk of the heavy lifting for IOMMU-backed dma-mapping.

Signed-off-by Robin Murphy <[email protected]>

---
 drivers/iommu/Kconfig     |   7 +
 drivers/iommu/Makefile    |   1 +
 drivers/iommu/dma-iommu.c | 560 ++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/dma-iommu.h |  94 ++++++++
 4 files changed, 662 insertions(+)
 create mode 100644 drivers/iommu/dma-iommu.c
 create mode 100644 include/linux/dma-iommu.h

drivers/iommu/dma-iommu.c new file +22 -0

+static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
+		dma_addr_t dma_addr)
+{
+	struct scatterlist *s, *seg = sg;
+	unsigned long seg_mask = dma_get_seg_boundary(dev);
+	unsigned int max_len = dma_get_max_seg_size(dev);
+	unsigned int seg_len = 0, seg_dma = 0;
+	int i, count = 1;
+
+	for_each_sg(sg, s, nents, i) {
+		/* Un-swizzling the fields here, hence the naming mismatch */
+		unsigned int s_offset = sg_dma_address(s);
+		unsigned int s_length = sg_dma_len(s);
+		unsigned int s_dma_len = s->length;
+
+		s->offset = s_offset;
+		s->length = s_length;
+		sg_dma_address(s) = DMA_ERROR_CODE;
+		sg_dma_len(s) = 0;
+
+		if (seg_len && (seg_dma + seg_len == dma_addr + s_offset) &&
+		    (seg_len + s_dma_len <= max_len) &&

drivers/iommu/dma-iommu.c new file +40 -0

+
+			seg_len = s_offset;
+			seg_dma = dma_addr + s_offset;
+		}
+		seg_len += s_length;
+		dma_addr += s_dma_len;
+	}
+	return count;
+}
+
+static void __invalidate_sg(struct scatterlist *sg, int nents)
+{
+	struct scatterlist *s;
+	int i;
+
+	for_each_sg(sg, s, nents, i) {
+		if (sg_dma_address(s) != DMA_ERROR_CODE)
+			s->offset = sg_dma_address(s);
+		if (sg_dma_len(s))
+			s->length = sg_dma_len(s);
+		sg_dma_address(s) = DMA_ERROR_CODE;
+		sg_dma_len(s) = 0;
+	}
+}
+
+int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
+		int nents, int prot, bool coherent)
+{
+	struct iommu_dma_domain *dom = arch_get_dma_domain(dev);
+	struct iova_domain *iovad = dom->iovad;
+	struct iova *iova;
+	struct scatterlist *s;
+	dma_addr_t dma_addr;
+	size_t iova_len = 0;
+	int i;
+
+	/*
+	 * Work out how much IOVA space we need, and align the segments to
+	 * IOVA granules for the IOMMU driver to handle. With some clever
+	 * trickery we can modify the list in a reversible manner.

drivers/iommu/dma-iommu.c new file +25 -0

+
+		iova_len += s_length;
+	}
+
+	iova = __alloc_iova(dev, iova_len, coherent);
+	if (!iova)
+		goto out_restore_sg;
+
+	/*
+	 * We'll leave any physical concatenation to the IOMMU driver's
+	 * implementation - it knows better than we do.
+	 */
+	dma_addr = iova_dma_addr(iovad, iova);
+	if (iommu_map_sg(dom->domain, dma_addr, sg, nents, prot) < iova_len)
+		goto out_free_iova;
+
+	return __finalise_sg(dev, sg, nents, dev_dma_addr(dev, dma_addr));
+
+out_free_iova:
+	__free_iova(iovad, iova);
+out_restore_sg:
+	__invalidate_sg(sg, nents);
+	return 0;
+}
+

Hi Robin,
Thanks.

While we test venc in v4l2, we get a problem:
When we enter the funtion[0], it will be break unexpectedly in the
funcion[1] while the offset of sg table is not zero. It is ok if the offset is zero. Then I add more log in dma-iommu.c, please help check below.
All we tested it based on dma v2. and have not tested it on v3 yet.
The code of iommu-map-sg seems the same. if it's fixed in v3, I'm very sorry. The map_sg in mtk-iommu use default_iommu_map_sg.
Any question please tell me, Thanks very much.

[0]http://lxr.free-electrons.com/source/drivers/media/v4l2-core/videobuf2-dma-contig.c#L564 [1]http://lxr.free-electrons.com/source/drivers/media/v4l2-core/videobuf2-dma-contig.c#L70

Quoted 14 lines · Robin Murphy

Taking inspiration from the existing arch/arm code, break out some generic functions to interface the DMA-API to the IOMMU-API. This will do the bulk of the heavy lifting for IOMMU-backed dma-mapping.

Signed-off-by Robin Murphy <[email protected]>

---
 drivers/iommu/Kconfig     |   7 +
 drivers/iommu/Makefile    |   1 +
 drivers/iommu/dma-iommu.c | 560 ++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/dma-iommu.h |  94 ++++++++
 4 files changed, 662 insertions(+)
 create mode 100644 drivers/iommu/dma-iommu.c
 create mode 100644 include/linux/dma-iommu.h

[snip]

Quoted 22 lines

drivers/iommu/dma-iommu.c new file +22 -0

+static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
+		dma_addr_t dma_addr)
+{
+	struct scatterlist *s, *seg = sg;
+	unsigned long seg_mask = dma_get_seg_boundary(dev);
+	unsigned int max_len = dma_get_max_seg_size(dev);
+	unsigned int seg_len = 0, seg_dma = 0;
+	int i, count = 1;
+
+	for_each_sg(sg, s, nents, i) {
+		/* Un-swizzling the fields here, hence the naming mismatch */
+		unsigned int s_offset = sg_dma_address(s);
+		unsigned int s_length = sg_dma_len(s);
+		unsigned int s_dma_len = s->length;
+
+		s->offset = s_offset;
+		s->length = s_length;
+		sg_dma_address(s) = DMA_ERROR_CODE;
+		sg_dma_len(s) = 0;
+
+		if (seg_len && (seg_dma + seg_len == dma_addr + s_offset) &&
+		    (seg_len + s_dma_len <= max_len) &&

drivers/iommu/dma-iommu.c line 461

Quoted 10 lines

drivers/iommu/dma-iommu.c new file +10 -0

+		    ((seg_dma & seg_mask) <= seg_mask - (seg_len + s_length))
+		   ) {
+			sg_dma_len(seg) += s_dma_len;
+		} else {
+			if (seg_len) {
+				seg = sg_next(seg);
+				count++;
+			}
+			sg_dma_len(seg) = s_dma_len;
+			sg_dma_address(seg) = dma_addr + s_offset;

Here the value of sg_dma_address have added s_offset, but
sg_dma_len(seg) still is s_dma_len.
In the first loop, s_dma_len is from s->length which is alignd by
"s_length = iova_align(iovad, s_length + s_offset);" in the interface iommu_dma_map_sg.

Quoted 40 lines

drivers/iommu/dma-iommu.c new file +40 -0

+
+			seg_len = s_offset;
+			seg_dma = dma_addr + s_offset;
+		}
+		seg_len += s_length;
+		dma_addr += s_dma_len;
+	}
+	return count;
+}
+
+static void __invalidate_sg(struct scatterlist *sg, int nents)
+{
+	struct scatterlist *s;
+	int i;
+
+	for_each_sg(sg, s, nents, i) {
+		if (sg_dma_address(s) != DMA_ERROR_CODE)
+			s->offset = sg_dma_address(s);
+		if (sg_dma_len(s))
+			s->length = sg_dma_len(s);
+		sg_dma_address(s) = DMA_ERROR_CODE;
+		sg_dma_len(s) = 0;
+	}
+}
+
+int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
+		int nents, int prot, bool coherent)
+{
+	struct iommu_dma_domain *dom = arch_get_dma_domain(dev);
+	struct iova_domain *iovad = dom->iovad;
+	struct iova *iova;
+	struct scatterlist *s;
+	dma_addr_t dma_addr;
+	size_t iova_len = 0;
+	int i;
+
+	/*
+	 * Work out how much IOVA space we need, and align the segments to
+	 * IOVA granules for the IOMMU driver to handle. With some clever
+	 * trickery we can modify the list in a reversible manner.

drivers/iommu/dma-iommu.c line 511

Quoted 10 lines

drivers/iommu/dma-iommu.c new file +10 -0

+	 */
+	for_each_sg(sg, s, nents, i) {
+		size_t s_offset = iova_offset(iovad, s->offset);
+		size_t s_length = s->length;
+
+		sg_dma_address(s) = s->offset;
+		sg_dma_len(s) = s_length;
+		s->offset -= s_offset;
+		s_length = iova_align(iovad, s_length + s_offset);
+		s->length = s_length;

At the begging, s->length is the length of valid data. but it's aligned here.

Quoted 25 lines

drivers/iommu/dma-iommu.c new file +25 -0

+
+		iova_len += s_length;
+	}
+
+	iova = __alloc_iova(dev, iova_len, coherent);
+	if (!iova)
+		goto out_restore_sg;
+
+	/*
+	 * We'll leave any physical concatenation to the IOMMU driver's
+	 * implementation - it knows better than we do.
+	 */
+	dma_addr = iova_dma_addr(iovad, iova);
+	if (iommu_map_sg(dom->domain, dma_addr, sg, nents, prot) < iova_len)
+		goto out_free_iova;
+
+	return __finalise_sg(dev, sg, nents, dev_dma_addr(dev, dma_addr));
+
+out_free_iova:
+	__free_iova(iovad, iova);
+out_restore_sg:
+	__invalidate_sg(sg, nents);
+	return 0;
+}
+

[snip]

Hi Robin,
Thanks.

While we test venc in v4l2, we get a problem:

When we enter the funtion[0], it will be break unexpectedly in the
funcion[1] while the offset of sg table is not zero. It is ok if the offset is zero. Then I add more log in dma-iommu.c, please help check below.

All we tested it based on dma v2. and have not tested it on v3 yet.
The code of iommu-map-sg seems the same. if it's fixed in v3, I'm very sorry. The map_sg in mtk-iommu use default_iommu_map_sg.
Any question please tell me, Thanks very much.

[0]http://lxr.free-electrons.com/source/drivers/media/v4l2-core/videobuf2-dma-contig.c#L564 [1]http://lxr.free-electrons.com/source/drivers/media/v4l2-core/videobuf2-dma-contig.c#L70

Quoted 14 lines · Robin Murphy

Taking inspiration from the existing arch/arm code, break out some generic functions to interface the DMA-API to the IOMMU-API. This will do the bulk of the heavy lifting for IOMMU-backed dma-mapping.

Signed-off-by Robin Murphy <[email protected]>

---
  drivers/iommu/Kconfig     |   7 +
  drivers/iommu/Makefile    |   1 +
  drivers/iommu/dma-iommu.c | 560 ++++++++++++++++++++++++++++++++++++++++++++++
  include/linux/dma-iommu.h |  94 ++++++++
  4 files changed, 662 insertions(+)

create mode 100644 drivers/iommu/dma-iommu.c
create mode 100644 include/linux/dma-iommu.h

[snip]

Quoted 23 lines

drivers/iommu/dma-iommu.c new file +23 -0

+static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
+		dma_addr_t dma_addr)
+{
+	struct scatterlist *s, *seg = sg;
+	unsigned long seg_mask = dma_get_seg_boundary(dev);
+	unsigned int max_len = dma_get_max_seg_size(dev);
+	unsigned int seg_len = 0, seg_dma = 0;
+	int i, count = 1;
+
+	for_each_sg(sg, s, nents, i) {
+		/* Un-swizzling the fields here, hence the naming mismatch */
+		unsigned int s_offset = sg_dma_address(s);
+		unsigned int s_length = sg_dma_len(s);
+		unsigned int s_dma_len = s->length;
+
+		s->offset = s_offset;
+		s->length = s_length;
+		sg_dma_address(s) = DMA_ERROR_CODE;
+		sg_dma_len(s) = 0;
+
+		if (seg_len && (seg_dma + seg_len == dma_addr + s_offset) &&
+		    (seg_len + s_dma_len <= max_len) &&
+		    ((seg_dma & seg_mask) <= seg_mask - (seg_len + s_length))

drivers/iommu/dma-iommu.c line 460

Quoted 8 lines

drivers/iommu/dma-iommu.c new file +8 -0

+		   ) {
+			sg_dma_len(seg) += s_dma_len;
+		} else {
+			if (seg_len) {
+				seg = sg_next(seg);
+				count++;
+			}
+			sg_dma_len(seg) = s_dma_len;

Taking some inspiration from the arch/arm code, implement the arch-specific side of the DMA mapping ops using the new IOMMU-DMA layer.

Whilst proliferating per-device private IOMMU data via dev->archdata is less than ideal, it will do the job for now, especially since we can't easily handle the kind of problematic system topologies in the current IOMMU API anyway.

Signed-off-by Robin Murphy <[email protected]>

---
 arch/arm64/include/asm/device.h      |   3 +
 arch/arm64/include/asm/dma-mapping.h |  14 ++
 arch/arm64/mm/dma-mapping.c          | 342 +++++++++++++++++++++++++++++++++++
 include/linux/dma-iommu.h            |   4 +-
 4 files changed, 361 insertions(+), 2 deletions(-)

arch/arm64/mm/dma-mapping.c +104 -0

+struct iommu_dma_notifier_data {
+	struct list_head list;
+	struct device *dev;
+	struct iommu_dma_domain *dma_domain;
+};
+static LIST_HEAD(iommu_dma_masters);
+static DEFINE_MUTEX(iommu_dma_notifier_lock);
+
+static int __iommu_attach_notifier(struct notifier_block *nb,
+				   unsigned long action, void *data)
+{
+	struct iommu_dma_notifier_data *master, *tmp;
+
+	if (action != BUS_NOTIFY_ADD_DEVICE)
+		return 0;
+	/*
+	 * We expect the list to only contain the most recent addition,
+	 * which *should* be the same device as @data, so just process the
+	 * whole thing blindly. If any previous attachments did happen to
+	 * fail, they get a free retry since the domains are still live.
+	 */
+	mutex_lock(&iommu_dma_notifier_lock);
+	list_for_each_entry_safe(master, tmp, &iommu_dma_masters, list) {
+		if (iommu_dma_attach_device(master->dev, master->dma_domain)) {
+			pr_warn("Failed to attach device %s to IOMMU mapping; retaining platform DMA ops\n",
+				dev_name(master->dev));
+		} else {
+			master->dev->archdata.dma_ops = &iommu_dma_ops;
+			/* it's safe to drop the initial refcount now */
+			iommu_dma_release_domain(master->dma_domain);
+			list_del(&master->list);
+			kfree(master);
+		}
+	}
+	mutex_unlock(&iommu_dma_notifier_lock);
+	return 0;
+}
+
+static int register_iommu_dma_ops_notifier(struct bus_type *bus)
+{
+	int ret;
+	struct notifier_block *nb = kzalloc(sizeof(*nb), GFP_KERNEL);
+
+	/*
+	 * The device must be attached to a domain before its driver probe,
+	 * in case the driver allocates DMA buffers immediately. However, most
+	 * IOMMU drivers are currently configuring groups in their add_device
+	 * callback, so the attach should happen after that. Since the IOMMU
+	 * core uses a bus notifier for add_device, do the same but with a
+	 * stupidly low priority to ensure the appropriate ordering.
+	 *
+	 * This can hopefully all go away once we have default domains in the
+	 * IOMMU core.
+	 */
+	nb->notifier_call = __iommu_attach_notifier;
+	nb->priority = INT_MIN;
+
+	ret = bus_register_notifier(bus, nb);
+	if (ret) {
+		pr_warn("Failed to register DMA domain notifier; IOMMU DMA ops unavailable on bus '%s'\n",
+			bus->name);
+		kfree(nb);
+	}
+	return ret;
+}
+
+static int __init arm64_iommu_dma_init(void)
+{
+	int ret;
+
+	ret = iommu_dma_init();
+	if (!ret)
+		ret = register_iommu_dma_ops_notifier(&platform_bus_type);
+	if (!ret)
+		ret = register_iommu_dma_ops_notifier(&amba_bustype);
+	return ret;
+}
+arch_initcall(arm64_iommu_dma_init);
+
+static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+				  const struct iommu_ops *ops)
+{
+	struct iommu_dma_notifier_data *iommudata;
+
+	if (!ops)
+		return;
+
+	iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL);
+	if (!iommudata)
+		return;
+
+	iommudata->dev = dev;
+	iommudata->dma_domain = iommu_dma_create_domain(ops, dma_base, size);
+	if (!iommudata->dma_domain) {
+		pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
+				size, dev_name(dev));
+		kfree(iommudata);
+		return;
+	}
+	mutex_lock(&iommu_dma_notifier_lock);
+	list_add(&iommudata->list, &iommu_dma_masters);
+	mutex_unlock(&iommu_dma_notifier_lock);
+}
+

Hi Robin,
Sorry to disturb you. When prepare iommu based on this dma-v3. I
meet some problem about how to use the dma-iommu.
I list some sample code like below:
//========
int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)

{
         if (args->args_count != 2) {
                 return -EINVAL;
         }

         /* save the private data from args
          * the private data is larbid and portid in our platform */
         priv->larbid = args->args[0];
         priv->portid = args->args[1];

         /*list the private data */
         list_add_tail(&priv->client, &head->client);

         /* save the list to dev->archdata->iommu */
         ...
}
static int mtk_iommu_init_fn(struct device_node *np)
{
         of_iommu_set_ops(np, &mtk_iommu_ops);
         return 0;
}

IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn); //=======

Question :
1: if there are 2 client devices use iommu, it will be like this in
dtsi,

                 disp:testiommu at 1400c000 {
                         compatible = "mediatek,mt8173-test";
                         iommus = <&iommu M4U_LARB0_ID 0>,
                                  <&iommu M4U_LARB1_ID 3>;
                 };
                 disp1:testiommu at 1400D000 {
                         compatible = "mediatek,mt8173-disptest1";
                         iommus = <&iommu M4U_LARB0_ID 8>,
                                  <&iommu M4U_LARB1_ID 2>;
                 };
     Then both devices will enter arch_setup_dma_ops and their parameter

"struct iommu_ops *" is not zero, then it will create two dma-iommu-domain.
If we expect all the client device share a iommu domain, then how
should i do?

2. iommu_dma_attach_device will be called automatically in the notify "__iommu_attach_notifier". But it run before the probe of our iommu device.
Then we can't write the register to enable iommu in *_attach_device
because we have not parsed the dtsi at that time.

    ->I have tried to move parse dtsi into mtk_iommu_of_xlate, in order

to read/write register in attach_device.
But it will be warning like this:
(150604_09:43:41.094)WARNING: CPU: 0 PID: 1
at /proj/mtk40525/upstreamdev/chromebook_kernelonly/kernel/mediatek/drivers/base/dd.c:286 driver_probe_device+0x25c/0x29c()

3.In mtk_iommu_probe, we can't get "struct iommu_domain *" from the current iommu device, we should use it while devm_request_irq. I have to use the global variable here?!

4.If I don't implement the of_xlate and IOMMU_OF_DECLARE, I call iommu_dma_create_domain in the mtk_iommu_probe, but I can not get the "iommu_dma_ops" which is static in arch/arm64/mm/dma-mapping.c to assign to dev->archdata->dma_ops for the iommu client device.

If i miss something, please tell me and help give some suggestion. Thanks very much.

Quoted 16 lines · Robin Murphy

Taking some inspiration from the arch/arm code, implement the arch-specific side of the DMA mapping ops using the new IOMMU-DMA layer.

Whilst proliferating per-device private IOMMU data via dev->archdata is less than ideal, it will do the job for now, especially since we can't easily handle the kind of problematic system topologies in the current IOMMU API anyway.

Signed-off-by Robin Murphy <[email protected]>

---
  arch/arm64/include/asm/device.h      |   3 +
  arch/arm64/include/asm/dma-mapping.h |  14 ++
  arch/arm64/mm/dma-mapping.c          | 342 +++++++++++++++++++++++++++++++++++
  include/linux/dma-iommu.h            |   4 +-
  4 files changed, 361 insertions(+), 2 deletions(-)

[snip]

Quoted 104 lines

arch/arm64/mm/dma-mapping.c +104 -0

+struct iommu_dma_notifier_data {
+	struct list_head list;
+	struct device *dev;
+	struct iommu_dma_domain *dma_domain;
+};
+static LIST_HEAD(iommu_dma_masters);
+static DEFINE_MUTEX(iommu_dma_notifier_lock);
+
+static int __iommu_attach_notifier(struct notifier_block *nb,
+				   unsigned long action, void *data)
+{
+	struct iommu_dma_notifier_data *master, *tmp;
+
+	if (action != BUS_NOTIFY_ADD_DEVICE)
+		return 0;
+	/*
+	 * We expect the list to only contain the most recent addition,
+	 * which *should* be the same device as @data, so just process the
+	 * whole thing blindly. If any previous attachments did happen to
+	 * fail, they get a free retry since the domains are still live.
+	 */
+	mutex_lock(&iommu_dma_notifier_lock);
+	list_for_each_entry_safe(master, tmp, &iommu_dma_masters, list) {
+		if (iommu_dma_attach_device(master->dev, master->dma_domain)) {
+			pr_warn("Failed to attach device %s to IOMMU mapping; retaining platform DMA ops\n",
+				dev_name(master->dev));
+		} else {
+			master->dev->archdata.dma_ops = &iommu_dma_ops;
+			/* it's safe to drop the initial refcount now */
+			iommu_dma_release_domain(master->dma_domain);
+			list_del(&master->list);
+			kfree(master);
+		}
+	}
+	mutex_unlock(&iommu_dma_notifier_lock);
+	return 0;
+}
+
+static int register_iommu_dma_ops_notifier(struct bus_type *bus)
+{
+	int ret;
+	struct notifier_block *nb = kzalloc(sizeof(*nb), GFP_KERNEL);
+
+	/*
+	 * The device must be attached to a domain before its driver probe,
+	 * in case the driver allocates DMA buffers immediately. However, most
+	 * IOMMU drivers are currently configuring groups in their add_device
+	 * callback, so the attach should happen after that. Since the IOMMU
+	 * core uses a bus notifier for add_device, do the same but with a
+	 * stupidly low priority to ensure the appropriate ordering.
+	 *
+	 * This can hopefully all go away once we have default domains in the
+	 * IOMMU core.
+	 */
+	nb->notifier_call = __iommu_attach_notifier;
+	nb->priority = INT_MIN;
+
+	ret = bus_register_notifier(bus, nb);
+	if (ret) {
+		pr_warn("Failed to register DMA domain notifier; IOMMU DMA ops unavailable on bus '%s'\n",
+			bus->name);
+		kfree(nb);
+	}
+	return ret;
+}
+
+static int __init arm64_iommu_dma_init(void)
+{
+	int ret;
+
+	ret = iommu_dma_init();
+	if (!ret)
+		ret = register_iommu_dma_ops_notifier(&platform_bus_type);
+	if (!ret)
+		ret = register_iommu_dma_ops_notifier(&amba_bustype);
+	return ret;
+}
+arch_initcall(arm64_iommu_dma_init);
+
+static void __iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+				  const struct iommu_ops *ops)
+{
+	struct iommu_dma_notifier_data *iommudata;
+
+	if (!ops)
+		return;
+
+	iommudata = kzalloc(sizeof(*iommudata), GFP_KERNEL);
+	if (!iommudata)
+		return;
+
+	iommudata->dev = dev;
+	iommudata->dma_domain = iommu_dma_create_domain(ops, dma_base, size);
+	if (!iommudata->dma_domain) {
+		pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
+				size, dev_name(dev));
+		kfree(iommudata);
+		return;
+	}
+	mutex_lock(&iommu_dma_notifier_lock);
+	list_add(&iommudata->list, &iommu_dma_masters);
+	mutex_unlock(&iommu_dma_notifier_lock);
+}
+

In the meantime, Laurent's proposal for probe deferral[2] offers hope that the bus notifier dance may only need to be short-lived, and I'm hoping to spend some more time on DT-based IOMMU group handling for platform devices, which should hopefully tie in with default domains to make the add_device callback mostly redundant.