arm64: IOMMU-backed DMA mapping

Taking inspiration from the existing arch/arm code, break out some generic functions to interface the DMA-API to the IOMMU-API. This will do the bulk of the heavy lifting for IOMMU-backed dma-mapping.

Signed-off-by Robin Murphy <[email protected]>

---
 drivers/iommu/Kconfig     |   7 +
 drivers/iommu/Makefile    |   1 +
 drivers/iommu/dma-iommu.c | 536 ++++++++++++++++++++++++++++++++++++++++++++++
 include/linux/dma-iommu.h |  84 ++++++++
 include/linux/iommu.h     |   1 +
 5 files changed, 629 insertions(+)
 create mode 100644 drivers/iommu/dma-iommu.c
 create mode 100644 include/linux/dma-iommu.h

drivers/iommu/Kconfig +7 -0

index f1fb1d3..efb0e66 100644
@@ -48,6 +48,13 @@ config OF_IOMMU
        def_bool y
        depends on OF && IOMMU_API
 
+# IOMMU-agnostic DMA-mapping layer
+config IOMMU_DMA
+	bool
+	depends on NEED_SG_DMA_LENGTH
+	select IOMMU_API
+	select IOMMU_IOVA
+
 config FSL_PAMU
 	bool "Freescale IOMMU support"
 	depends on PPC32

drivers/iommu/Makefile +1 -0

index c6dcc51..f465cfb 100644
@@ -1,6 +1,7 @@
 obj-$(CONFIG_IOMMU_API) += iommu.o
 obj-$(CONFIG_IOMMU_API) += iommu-traces.o
 obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o
+obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o
 obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o
 obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
 obj-$(CONFIG_IOMMU_IOVA) += iova.o

drivers/iommu/Makefile +14 -0

+/**
+ * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space
+ * @dev: Device to allocate memory for. Must be a real device
+ *	 attached to an iommu_dma_domain
+ * @size: Size of buffer in bytes
+ * @gfp: Allocation flags
+ * @prot: IOMMU mapping flags
+ * @coherent: Which dma_mask to base IOVA allocation on
+ * @handle: Out argument for allocated DMA handle
+ * @flush_page: Arch callback to flush a single page from caches as
+ *		necessary. May be NULL for coherent allocations
+ *
+ * If @size is less than PAGE_SIZE, then a full CPU page will be allocated,
+ * but an IOMMU which supports smaller pages might not map the whole thing.

drivers/iommu/Makefile +49 -0

+	struct iova_domain *iovad = domain->dma_api_cookie;
+	struct iova *iova;
+	struct page **pages;
+	struct sg_table sgt;
+	struct sg_mapping_iter miter;
+	dma_addr_t dma_addr;
+	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+	*handle = DMA_ERROR_CODE;
+
+	pages = __iommu_dma_alloc_pages(count, gfp);
+	if (!pages)
+		return NULL;
+
+	iova = __alloc_iova(dev, size, coherent);
+	if (!iova)
+		goto out_free_pages;
+
+	size = iova_align(iovad, size);
+	if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, GFP_KERNEL))
+		goto out_free_iova;
+
+	dma_addr = iova_dma_addr(iovad, iova);
+	if (iommu_map_sg(domain, dma_addr, sgt.sgl, sgt.orig_nents, prot)
+			< size)
+		goto out_free_sg;
+
+	/* Using the non-flushing flag since we're doing our own */
+	sg_miter_start(&miter, sgt.sgl, sgt.orig_nents, SG_MITER_FROM_SG);
+	while (sg_miter_next(&miter)) {
+		memset(miter.addr, 0, PAGE_SIZE);
+		if (flush_page)
+			flush_page(miter.addr, page_to_phys(miter.page));
+	}
+	sg_miter_stop(&miter);
+	sg_free_table(&sgt);
+
+	*handle = dma_addr;
+	return pages;
+
+out_free_sg:
+	sg_free_table(&sgt);
+out_free_iova:
+	__free_iova(iovad, iova);
+out_free_pages:
+	__iommu_dma_free_pages(pages, count);
+	return NULL;
+}
+

enum iommu_cap {

drivers/iommu/dma-iommu.c new file +21 -0

+
+/**
+ * dma_direction_to_prot - Translate DMA API directions to IOMMU API page flags
+ * @dir: Direction of DMA transfer
+ * @coherent: Is the DMA master cache-coherent?
+ *
+ * Return: corresponding IOMMU API page protection flags
+ */
+int dma_direction_to_prot(enum dma_data_direction dir, bool coherent)
+{
+	int prot = coherent ? IOMMU_CACHE : 0;
+
+	switch (dir) {
+	case DMA_BIDIRECTIONAL:
+		return prot | IOMMU_READ | IOMMU_WRITE;
+	case DMA_TO_DEVICE:
+		return prot | IOMMU_READ;
+	case DMA_FROM_DEVICE:
+		return prot | IOMMU_WRITE;
+	default:
+		return 0;

drivers/iommu/dma-iommu.c new file +12 -0

+/**
+ * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space
+ * @dev: Device to allocate memory for. Must be a real device
+ *	 attached to an iommu_dma_domain
+ * @size: Size of buffer in bytes
+ * @gfp: Allocation flags
+ * @prot: IOMMU mapping flags
+ * @coherent: Which dma_mask to base IOVA allocation on
+ * @handle: Out argument for allocated DMA handle
+ * @flush_page: Arch callback to flush a single page from caches as
+ *		necessary. May be NULL for coherent allocations
+ *

drivers/iommu/dma-iommu.c new file +7 -0

+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = domain->dma_api_cookie;
+	struct iova *iova;
+	struct page **pages;
+	struct sg_table sgt;
+	struct sg_mapping_iter miter;

drivers/iommu/dma-iommu.c new file +28 -0

+/*
+ * The DMA API client is passing in a scatterlist which could describe
+ * any old buffer layout, but the IOMMU API requires everything to be
+ * aligned to IOMMU pages. Hence the need for this complicated bit of
+ * impedance-matching, to be able to hand off a suitably-aligned list,
+ * but still preserve the original offsets and sizes for the caller.
+ */
+int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
+		int nents, int prot, bool coherent)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = domain->dma_api_cookie;
+	struct iova *iova;
+	struct scatterlist *s;
+	dma_addr_t dma_addr;
+	size_t iova_len = 0;
+	int i;
+
+	/*
+	 * Work out how much IOVA space we need, and align the segments to
+	 * IOVA granules for the IOMMU driver to handle. With some clever
+	 * trickery we can modify the list in-place, but reversibly, by
+	 * hiding the original data in the as-yet-unused DMA fields.
+	 */
+	for_each_sg(sg, s, nents, i) {
+		size_t s_offset = iova_offset(iovad, s->offset);
+		size_t s_length = s->length;
+

drivers/iommu/dma-iommu.c line 125

Quoted 11 lines · Robin Murphy

drivers/iommu/dma-iommu.c new file +11 -0

+/*
+ * IOVAs are IOMMU _input_ addresses, so there still exists the possibility
+ * for static bus translation between device output and IOMMU input (yuck).
+ */
+static inline dma_addr_t dev_dma_addr(struct device *dev, dma_addr_t addr)
+{
+	dma_addr_t offset = (dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT;
+
+	BUG_ON(addr < offset);
+	return addr - offset;
+}

Are these just theoretical or you expect to see some at some point? I think the dma_limit in __alloc_iova() may also need to take the offset into account (or just ignore them altogether for now).

Quoted 21 lines

drivers/iommu/dma-iommu.c new file +21 -0

+
+/**
+ * dma_direction_to_prot - Translate DMA API directions to IOMMU API page flags
+ * @dir: Direction of DMA transfer
+ * @coherent: Is the DMA master cache-coherent?
+ *
+ * Return: corresponding IOMMU API page protection flags
+ */
+int dma_direction_to_prot(enum dma_data_direction dir, bool coherent)
+{
+	int prot = coherent ? IOMMU_CACHE : 0;
+
+	switch (dir) {
+	case DMA_BIDIRECTIONAL:
+		return prot | IOMMU_READ | IOMMU_WRITE;
+	case DMA_TO_DEVICE:
+		return prot | IOMMU_READ;
+	case DMA_FROM_DEVICE:
+		return prot | IOMMU_WRITE;
+	default:
+		return 0;

drivers/iommu/dma-iommu.c line 156

Quoted 10 lines

drivers/iommu/dma-iommu.c new file +10 -0

+	}
+}
+
+static struct iova *__alloc_iova(struct device *dev, size_t size, bool coherent)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = domain->dma_api_cookie;
+	unsigned long shift = iova_shift(iovad);
+	unsigned long length = iova_align(iovad, size) >> shift;
+	u64 dma_limit = coherent ? dev->coherent_dma_mask : dma_get_mask(dev);

"coherent" and "coherent_dma_mask" have related meanings here. As I can see in patch 3, the coherent information passed all the way to this function states whether the device is cache coherent or not (and whether cache maintenance is needed). The coherent_dma_mask refers to an allocation mask for the dma_alloc_coherent() API but that doesn't necessarily mean that the device is cache coherent. Similarly, dma_mask is used for streaming DMA.

You can rename it to coherent_api or simply pass a u64 dma_mask directly.

[...]

Quoted 17 lines

+/**
+ * iommu_dma_alloc - Allocate and map a buffer contiguous in IOVA space
+ * @dev: Device to allocate memory for. Must be a real device
+ * attached to an iommu_dma_domain
+ * @size: Size of buffer in bytes
+ * @gfp: Allocation flags
+ * @prot: IOMMU mapping flags
+ * @coherent: Which dma_mask to base IOVA allocation on
+ * @handle: Out argument for allocated DMA handle
+ * @flush_page: Arch callback to flush a single page from caches as
+ * necessary. May be NULL for coherent allocations
+ *
+ * If @size is less than PAGE_SIZE, then a full CPU page will be allocated,
+ * but an IOMMU which supports smaller pages might not map the whole thing.
+ * For now, the buffer is unconditionally zeroed for compatibility
+ *
+ * Return: Array of struct page pointers describing the buffer,

drivers/iommu/dma-iommu.c line 276

Quoted 5 lines

+ * or NULL on failure.
+ */

drivers/iommu/dma-iommu.c new file +3 -0

+struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp,
+		int prot, bool coherent, dma_addr_t *handle,
+		void (*flush_page)(const void *, phys_addr_t))

So for this function, coherent should always be true since this is only used with the coherent DMA API AFAICT.

Quoted 11 lines

drivers/iommu/dma-iommu.c new file +11 -0

+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = domain->dma_api_cookie;
+	struct iova *iova;
+	struct page **pages;
+	struct sg_table sgt;
+	struct sg_mapping_iter miter;
+	dma_addr_t dma_addr;
+	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+	*handle = DMA_ERROR_CODE;

drivers/iommu/dma-iommu.c line 293

Quoted 6 lines

drivers/iommu/dma-iommu.c new file +6 -0

+
+	pages = __iommu_dma_alloc_pages(count, gfp);
+	if (!pages)
+		return NULL;
+
+	iova = __alloc_iova(dev, size, coherent);

And here just __alloc_iova(dev, size, true);

[...]

drivers/iommu/dma-iommu.c line 362

Quoted 10 lines

drivers/iommu/dma-iommu.c new file +10 -0

+dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
+		unsigned long offset, size_t size, int prot, bool coherent)
+{
+	dma_addr_t dma_addr;
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = domain->dma_api_cookie;
+	phys_addr_t phys = page_to_phys(page) + offset;
+	size_t iova_off = iova_offset(iovad, phys);
+	size_t len = iova_align(iovad, size + iova_off);
+	struct iova *iova = __alloc_iova(dev, len, coherent);

Here __alloc_iova(dev, len, false);

[...]

Quoted 28 lines

drivers/iommu/dma-iommu.c new file +28 -0

+/*
+ * The DMA API client is passing in a scatterlist which could describe
+ * any old buffer layout, but the IOMMU API requires everything to be
+ * aligned to IOMMU pages. Hence the need for this complicated bit of
+ * impedance-matching, to be able to hand off a suitably-aligned list,
+ * but still preserve the original offsets and sizes for the caller.
+ */
+int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
+		int nents, int prot, bool coherent)
+{
+	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
+	struct iova_domain *iovad = domain->dma_api_cookie;
+	struct iova *iova;
+	struct scatterlist *s;
+	dma_addr_t dma_addr;
+	size_t iova_len = 0;
+	int i;
+
+	/*
+	 * Work out how much IOVA space we need, and align the segments to
+	 * IOVA granules for the IOMMU driver to handle. With some clever
+	 * trickery we can modify the list in-place, but reversibly, by
+	 * hiding the original data in the as-yet-unused DMA fields.
+	 */
+	for_each_sg(sg, s, nents, i) {
+		size_t s_offset = iova_offset(iovad, s->offset);
+		size_t s_length = s->length;
+

drivers/iommu/dma-iommu.c line 492

Quoted 10 lines

drivers/iommu/dma-iommu.c new file +10 -0

+		sg_dma_address(s) = s->offset;
+		sg_dma_len(s) = s_length;
+		s->offset -= s_offset;
+		s_length = iova_align(iovad, s_length + s_offset);
+		s->length = s_length;
+
+		iova_len += s_length;
+	}
+
+	iova = __alloc_iova(dev, iova_len, coherent);

__alloc_iova(dev, iova_len, false);

arch/arm64/mm/dma-mapping.c +22 -0

index d16a1ce..ccadfd4 100644
@@ -526,3 +526,426 @@ static int __init dma_debug_do_init(void)
 	return 0;
 }
 fs_initcall(dma_debug_do_init);
+
+
+#ifdef CONFIG_IOMMU_DMA
+#include <linux/dma-iommu.h>
+#include <linux/platform_device.h>
+#include <linux/amba/bus.h>
+
+/* Thankfully, all cache ops are by VA so we can ignore phys here */
+static void flush_page(const void *virt, phys_addr_t phys)
+{
+	__dma_flush_range(virt, virt + PAGE_SIZE);
+}
+
+static void *__iommu_alloc_attrs(struct device *dev, size_t size,
+				 dma_addr_t *handle, gfp_t gfp,
+				 struct dma_attrs *attrs)
+{
+	bool coherent = is_device_dma_coherent(dev);
+	int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent);
+	void *addr;
+
+	if (WARN(!dev, "cannot create IOMMU mapping for unknown device\n"))

arch/arm64/mm/dma-mapping.c +7 -0

+		if (!pages)
+			return NULL;
+
+		addr = dma_common_pages_remap(pages, size, VM_USERMAP, pgprot,
+					      __builtin_return_address(0));
+		if (!addr)
+			iommu_dma_free(dev, pages, size, handle);

arch/arm64/mm/dma-mapping.c +15 -0

+static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+			       dma_addr_t handle, struct dma_attrs *attrs)
+{
+	/*
+	 * @cpu_addr will be one of 3 things depending on how it was allocated:
+	 * - A remapped array of pages from iommu_dma_alloc(), for all
+	 *   non-atomic allocations.
+	 * - A non-cacheable alias from the atomic pool, for atomic
+	 *   allocations by non-coherent devices.
+	 * - A normal lowmem address, for atomic allocations by
+	 *   coherent devices.
+	 * Hence how dodgy the below logic looks...
+	 */
+	if (__free_from_pool(cpu_addr, size)) {
+		iommu_dma_unmap_page(dev, handle, size, 0, NULL);

Quoted 30 lines · Robin Murphy

arch/arm64/mm/dma-mapping.c +22 -0

index d16a1ce..ccadfd4 100644
@@ -526,3 +526,426 @@ static int __init dma_debug_do_init(void)
  	return 0;
  }
  fs_initcall(dma_debug_do_init);
+
+
+#ifdef CONFIG_IOMMU_DMA
+#include <linux/dma-iommu.h>
+#include <linux/platform_device.h>
+#include <linux/amba/bus.h>
+
+/* Thankfully, all cache ops are by VA so we can ignore phys here */
+static void flush_page(const void *virt, phys_addr_t phys)
+{
+	__dma_flush_range(virt, virt + PAGE_SIZE);
+}
+
+static void *__iommu_alloc_attrs(struct device *dev, size_t size,
+				 dma_addr_t *handle, gfp_t gfp,
+				 struct dma_attrs *attrs)
+{
+	bool coherent = is_device_dma_coherent(dev);
+	int ioprot = dma_direction_to_prot(DMA_BIDIRECTIONAL, coherent);
+	void *addr;
+
+	if (WARN(!dev, "cannot create IOMMU mapping for unknown device\n"))

arch/arm64/mm/dma-mapping.c static int __init dma_debug_do_init(void) · line 560

Quoted 10 lines

arch/arm64/mm/dma-mapping.c +10 -0

+		return NULL;
+
+	if (gfp & __GFP_WAIT) {
+		struct page **pages;
+		pgprot_t pgprot = coherent ? __pgprot(PROT_NORMAL) :
+					     __pgprot(PROT_NORMAL_NC);
+
+		pgprot = __get_dma_pgprot(attrs, pgprot, coherent);
+		pages = iommu_dma_alloc(dev, size, gfp, ioprot,	coherent,
+					handle, coherent ? NULL : flush_page);

As I replied already on the other patch, the "coherent" argument here should always be true.

BTW, why do we need to call flush_page via iommu_dma_alloc() and not flush the buffer directly in the arch __iommu_alloc_attrs()? We already have the pointer and size after remapping in the CPU address space), it would keep the iommu_dma_alloc() simpler.

Quoted 12 lines

arch/arm64/mm/dma-mapping.c +12 -0

+		if (!pages)
+			return NULL;
+
+		addr = dma_common_pages_remap(pages, size, VM_USERMAP, pgprot,
+					      __builtin_return_address(0));
+		if (!addr)
+			iommu_dma_free(dev, pages, size, handle);
+	} else {
+		struct page *page;
+		/*
+		 * In atomic context we can't remap anything, so we'll only
+		 * get the virtually contiguous buffer we need by way of a

arch/arm64/mm/dma-mapping.c static int __init dma_debug_do_init(void) · line 577

Quoted 5 lines

arch/arm64/mm/dma-mapping.c +5 -0

+		 * physically contiguous allocation.
+		 */
+		if (coherent) {
+			page = alloc_pages(gfp, get_order(size));
+			addr = page ? page_address(page) : NULL;

We could even use __get_free_pages(gfp & ~__GFP_HIGHMEM) since we don't have/need highmem on arm64.

arch/arm64/mm/dma-mapping.c static int __init dma_debug_do_init(void) · line 583

Quoted 6 lines

arch/arm64/mm/dma-mapping.c +6 -0

+		} else {
+			addr = __alloc_from_pool(size, &page, gfp);
+		}
+		if (addr) {
+			*handle = iommu_dma_map_page(dev, page, 0, size,
+						     ioprot, false);

Why coherent == false?

Quoted 6 lines

arch/arm64/mm/dma-mapping.c +6 -0

+			if (iommu_dma_mapping_error(dev, *handle)) {
+				if (coherent)
+					__free_pages(page, get_order(size));
+				else
+					__free_from_pool(addr, size);
+				addr = NULL;

arch/arm64/mm/dma-mapping.c static int __init dma_debug_do_init(void) · line 594

Quoted 5 lines

arch/arm64/mm/dma-mapping.c +5 -0

+			}
+		}
+	}
+	return addr;
+}

In the second case here (!__GFP_WAIT), do we do any cache maintenance? I can't see it and it's needed for the !coherent case.

Quoted 21 lines

arch/arm64/mm/dma-mapping.c +21 -0

+static void __iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
+			       dma_addr_t handle, struct dma_attrs *attrs)
+{
+	/*
+	 * @cpu_addr will be one of 3 things depending on how it was allocated:
+	 * - A remapped array of pages from iommu_dma_alloc(), for all
+	 *   non-atomic allocations.
+	 * - A non-cacheable alias from the atomic pool, for atomic
+	 *   allocations by non-coherent devices.
+	 * - A normal lowmem address, for atomic allocations by
+	 *   coherent devices.
+	 * Hence how dodgy the below logic looks...
+	 */
+	if (__free_from_pool(cpu_addr, size)) {
+		iommu_dma_unmap_page(dev, handle, size, 0, NULL);
+	} else if (is_vmalloc_addr(cpu_addr)){
+		struct vm_struct *area = find_vm_area(cpu_addr);
+
+		if (WARN_ON(!area || !area->pages))
+			return;
+		iommu_dma_free(dev, area->pages, size, &handle);

arch/arm64/mm/dma-mapping.c static int __init dma_debug_do_init(void) · line 620

Quoted 4 lines

arch/arm64/mm/dma-mapping.c +4 -0

+		dma_common_free_remap(cpu_addr, size, VM_USERMAP);
+	} else {
+		__free_pages(virt_to_page(cpu_addr), get_order(size));
+		iommu_dma_unmap_page(dev, handle, size, 0, NULL);

Just slightly paranoid but it's better to unmap the page from the iommu space before freeing (in case there is some rogue device still accessing it).

Quoted 15 lines · Catalin Marinas

arch/arm64/mm/dma-mapping.c static int __init dma_debug_do_init(void) · line 594

Quoted 11 lines · Robin Murphy

arch/arm64/mm/dma-mapping.c +11 -0

+			if (iommu_dma_mapping_error(dev, *handle)) {
+				if (coherent)
+					__free_pages(page, get_order(size));
+				else
+					__free_from_pool(addr, size);
+				addr = NULL;
+			}
+		}
+	}
+	return addr;
+}

In the second case here (!__GFP_WAIT), do we do any cache maintenance? I can't see it and it's needed for the !coherent case.

In the atomic non-coherent case, we're stealing from the atomic pool, so addr is already a non-cacheable alias (and alloc_from_pool does memset(0) through that). That shouldn't need anything extra, right?