dma-mapping: Use DMA_ATTR_CC_SHARED through direct, pool and swiotlb paths

Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in dma_direct_alloc() / dma_direct_alloc_pages().

This is needed for follow-up changes that simplify the handling of memory encryption/decryption based on the DMA attribute flags.

swiotlb backing pages are already mapped decrypted by swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so dma-direct should not call dma_set_decrypted() on allocation nor dma_set_encrypted() on free for swiotlb-backed memory.

Update alloc/free paths to detect swiotlb-backed pages and skip encrypt/decrypt transitions for those paths. Keep the existing highmem rejection in dma_direct_alloc_pages() for swiotlb allocations.

Only for "restricted-dma-pool", we currently set for_alloc = true, while rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is typically used together with "shared-dma-pool", where the shared region is accessed after remap/ioremap and the returned address is suitable for decrypted memory access. So existing code paths remain valid.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 kernel/dma/direct.c | 44 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 7 deletions(-)

kernel/dma/direct.c +11 -3

index ec887f443741..b958f150718a 100644
@@ -125,9 +125,6 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
 
 	WARN_ON_ONCE(!PAGE_ALIGNED(size));
 
-	if (is_swiotlb_for_alloc(dev))
-		return dma_direct_alloc_swiotlb(dev, size);
-
 	gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
 	page = dma_alloc_contiguous(dev, size, gfp);
 	if (page) {

@@ -204,6 +201,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
 	bool remap = false, set_uncached = false;
+	bool mark_mem_decrypt = true;
 	struct page *page;
 	void *ret;
 

@@ -250,11 +248,21 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	    dma_direct_use_pool(dev, gfp))
 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
 
+	if (is_swiotlb_for_alloc(dev)) {
+		page = dma_direct_alloc_swiotlb(dev, size);
+		if (page) {
+			mark_mem_decrypt = false;
+			goto setup_page;
+		}
+		return NULL;
+	}
+
 	/* we always manually zero the memory once we are done */
 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
 	if (!page)
 		return NULL;
 
+setup_page:
 	/*
 	 * dma_alloc_contiguous can return highmem pages depending on a
 	 * combination the cma= arguments and per-arch setup.  These need to be

kernel/dma/direct.c +6 -2

 			goto out_leak_pages;
 	}
 

@@ -298,7 +306,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	return ret;
 
 out_encrypt_pages:
-	if (dma_set_encrypted(dev, page_address(page), size))
+	if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
 		return NULL;
 out_free_pages:
 	__dma_direct_free_pages(dev, page, size);

@@ -310,6 +318,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 void dma_direct_free(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
 {
+	bool mark_mem_encrypted = true;
 	unsigned int page_order = get_order(size);
 
 	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&

@@ -338,12 +347,15 @@ void dma_direct_free(struct device *dev, size_t size,
 	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
 		return;
 
+	if (swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr)))
+		mark_mem_encrypted = false;
+
 	if (is_vmalloc_addr(cpu_addr)) {
 		vunmap(cpu_addr);
 	} else {
 		if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
 			arch_dma_clear_uncached(cpu_addr, size);
-		if (dma_set_encrypted(dev, cpu_addr, size))
+		if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
 			return;
 	}
 

kernel/dma/direct.c +13 -1

+			swiotlb_free(dev, page, size);
+			return NULL;
+		}
+		ret = page_address(page);
+		goto setup_page;
+	}
+
 	page = __dma_direct_alloc_pages(dev, size, gfp, false);
 	if (!page)
 		return NULL;

@@ -366,6 +391,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 	ret = page_address(page);
 	if (dma_set_decrypted(dev, ret, size))
 		goto out_leak_pages;
+setup_page:
 	memset(ret, 0, size);
 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
 	return page;

@@ -378,13 +404,17 @@ void dma_direct_free_pages(struct device *dev, size_t size,
 		enum dma_data_direction dir)
 {
 	void *vaddr = page_address(page);
+	bool mark_mem_encrypted = true;
 
 	/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
 	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
 	    dma_free_from_pool(dev, vaddr, size))
 		return;
 
-	if (dma_set_encrypted(dev, vaddr, size))
+	if (swiotlb_find_pool(dev, page_to_phys(page)))
+		mark_mem_encrypted = false;
+
+	if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size))
 		return;
 	__dma_direct_free_pages(dev, page, size);
 }

Signature · 1 line

2.43.0

Quoted 71 lines · Aneesh Kumar K.V (Arm)

Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in dma_direct_alloc() / dma_direct_alloc_pages().

This is needed for follow-up changes that simplify the handling of memory encryption/decryption based on the DMA attribute flags.

swiotlb backing pages are already mapped decrypted by swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so dma-direct should not call dma_set_decrypted() on allocation nor dma_set_encrypted() on free for swiotlb-backed memory.

Update alloc/free paths to detect swiotlb-backed pages and skip encrypt/decrypt transitions for those paths. Keep the existing highmem rejection in dma_direct_alloc_pages() for swiotlb allocations.

Only for "restricted-dma-pool", we currently set for_alloc = true, while rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is typically used together with "shared-dma-pool", where the shared region is accessed after remap/ioremap and the returned address is suitable for decrypted memory access. So existing code paths remain valid.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 kernel/dma/direct.c | 44 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 37 insertions(+), 7 deletions(-)

kernel/dma/direct.c +11 -3

index ec887f443741..b958f150718a 100644
@@ -125,9 +125,6 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
 
 	WARN_ON_ONCE(!PAGE_ALIGNED(size));
 
-	if (is_swiotlb_for_alloc(dev))
-		return dma_direct_alloc_swiotlb(dev, size);
-
 	gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
 	page = dma_alloc_contiguous(dev, size, gfp);
 	if (page) {

@@ -204,6 +201,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
 	bool remap = false, set_uncached = false;
+	bool mark_mem_decrypt = true;
 	struct page *page;
 	void *ret;
 

@@ -250,11 +248,21 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	    dma_direct_use_pool(dev, gfp))
 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
 
+	if (is_swiotlb_for_alloc(dev)) {
+		page = dma_direct_alloc_swiotlb(dev, size);
+		if (page) {
+			mark_mem_decrypt = false;
+			goto setup_page;
+		}
+		return NULL;
+	}
+
 	/* we always manually zero the memory once we are done */
 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
 	if (!page)
 		return NULL;
 
+setup_page:
 	/*
 	 * dma_alloc_contiguous can return highmem pages depending on a
 	 * combination the cma= arguments and per-arch setup.  These need to be

kernel/dma/direct.c void *dma_direct_alloc(struct device *dev, size_t size, · line 292

Quoted 5 lines

kernel/dma/direct.c +1 -1

@@ -281,7 +289,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 			goto out_free_pages;
 	} else {
 		ret = page_address(page);
-		if (dma_set_decrypted(dev, ret, size))
+		if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size))

I am ok with that approach, but Jason was mentioning we shouldn’t special case swiotlb and make the allocator return the memory state (similar to the dma_page [1]) . I am also OK if you want to merge that part of my series with is.

[1] https://lore.kernel.org/linux-iommu/[email protected]/

Quoted 42 lines

kernel/dma/direct.c +6 -2

 			goto out_leak_pages;
 	}
 

@@ -298,7 +306,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	return ret;
 
 out_encrypt_pages:
-	if (dma_set_encrypted(dev, page_address(page), size))
+	if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
 		return NULL;
 out_free_pages:
 	__dma_direct_free_pages(dev, page, size);

@@ -310,6 +318,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 void dma_direct_free(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
 {
+	bool mark_mem_encrypted = true;
 	unsigned int page_order = get_order(size);
 
 	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&

@@ -338,12 +347,15 @@ void dma_direct_free(struct device *dev, size_t size,
 	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
 		return;
 
+	if (swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr)))
+		mark_mem_encrypted = false;
+
 	if (is_vmalloc_addr(cpu_addr)) {
 		vunmap(cpu_addr);
 	} else {
 		if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
 			arch_dma_clear_uncached(cpu_addr, size);
-		if (dma_set_encrypted(dev, cpu_addr, size))
+		if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
 			return;
 	}
 

@@ -359,6 +371,19 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 	if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
 

kernel/dma/direct.c struct page *dma_direct_alloc_pages(struct device *dev, size_t size, · line 379

Quoted 6 lines

kernel/dma/direct.c +6 -0

+	if (is_swiotlb_for_alloc(dev)) {
+		page = dma_direct_alloc_swiotlb(dev, size);
+		if (!page)
+			return NULL;
+
+		if (PageHighMem(page)) {

My understanding is that rmem_swiotlb_device_init() asserts that there is no PageHighMem()? Also a similar check doesn’t exist in dma_direct_alloc().

Propagate force_dma_unencrypted() into DMA_ATTR_CC_SHARED in the dma-direct allocation path and use the attribute to drive the related decisions.

This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 kernel/dma/direct.c | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

kernel/dma/direct.c +31 -7

index b958f150718a..0c2e1f8436ce 100644
@@ -201,16 +201,31 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
 	bool remap = false, set_uncached = false;
-	bool mark_mem_decrypt = true;
+	bool mark_mem_decrypt = false;
 	struct page *page;
 	void *ret;
 
+	/*
+	 * DMA_ATTR_CC_SHARED is not a caller-visible dma_alloc_*()
+	 * attribute. The direct allocator uses it internally after it has
+	 * decided that the backing pages must be shared/decrypted, so the
+	 * rest of the allocation path can consistently select DMA addresses,
+	 * choose compatible pools and restore encryption on free.
+	 */
+	if (attrs & DMA_ATTR_CC_SHARED)
+		return NULL;
+
+	if (force_dma_unencrypted(dev)) {
+		attrs |= DMA_ATTR_CC_SHARED;
+		mark_mem_decrypt = true;
+	}
+
 	size = PAGE_ALIGN(size);
 	if (attrs & DMA_ATTR_NO_WARN)
 		gfp |= __GFP_NOWARN;
 
-	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
-	    !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
+	if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) ==
+	     DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
 		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
 
 	if (!dev_is_dma_coherent(dev)) {

@@ -244,7 +259,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	 * Remapping or decrypting memory may block, allocate the memory from
 	 * the atomic pools instead if we aren't allowed block.
 	 */
-	if ((remap || force_dma_unencrypted(dev)) &&
+	if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
 	    dma_direct_use_pool(dev, gfp))
 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
 

@@ -318,11 +333,20 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 void dma_direct_free(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
 {
-	bool mark_mem_encrypted = true;
+	bool mark_mem_encrypted = false;
 	unsigned int page_order = get_order(size);
 
-	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
-	    !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
+	/*
+	 * if the device had requested for an unencrypted buffer,
+	 * convert it to encrypted on free
+	 */
+	if (force_dma_unencrypted(dev)) {
+		attrs |= DMA_ATTR_CC_SHARED;
+		mark_mem_encrypted = true;
+	}
+
+	if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) ==
+	     DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
 		/* cpu_addr is a struct page cookie, not a kernel address */
 		dma_free_contiguous(dev, cpu_addr, size);
 		return;

@@ -365,10 +389,14 @@ void dma_direct_free(struct device *dev, size_t size,
 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
 {

kernel/dma/direct.c +0 -0

 	if (is_swiotlb_for_alloc(dev)) {

Signature · 1 line

2.43.0

Quoted 88 lines · Aneesh Kumar K.V (Arm)

Propagate force_dma_unencrypted() into DMA_ATTR_CC_SHARED in the dma-direct allocation path and use the attribute to drive the related decisions.

This updates dma_direct_alloc(), dma_direct_free(), and dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 kernel/dma/direct.c | 44 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 8 deletions(-)

kernel/dma/direct.c +31 -7

index b958f150718a..0c2e1f8436ce 100644
@@ -201,16 +201,31 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
 	bool remap = false, set_uncached = false;
-	bool mark_mem_decrypt = true;
+	bool mark_mem_decrypt = false;
 	struct page *page;
 	void *ret;
 
+	/*
+	 * DMA_ATTR_CC_SHARED is not a caller-visible dma_alloc_*()
+	 * attribute. The direct allocator uses it internally after it has
+	 * decided that the backing pages must be shared/decrypted, so the
+	 * rest of the allocation path can consistently select DMA addresses,
+	 * choose compatible pools and restore encryption on free.
+	 */
+	if (attrs & DMA_ATTR_CC_SHARED)
+		return NULL;
+
+	if (force_dma_unencrypted(dev)) {
+		attrs |= DMA_ATTR_CC_SHARED;
+		mark_mem_decrypt = true;
+	}
+
 	size = PAGE_ALIGN(size);
 	if (attrs & DMA_ATTR_NO_WARN)
 		gfp |= __GFP_NOWARN;
 
-	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
-	    !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
+	if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) ==
+	     DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
 		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
 
 	if (!dev_is_dma_coherent(dev)) {

@@ -244,7 +259,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	 * Remapping or decrypting memory may block, allocate the memory from
 	 * the atomic pools instead if we aren't allowed block.
 	 */
-	if ((remap || force_dma_unencrypted(dev)) &&
+	if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
 	    dma_direct_use_pool(dev, gfp))
 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
 

@@ -318,11 +333,20 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 void dma_direct_free(struct device *dev, size_t size,
 		void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
 {
-	bool mark_mem_encrypted = true;
+	bool mark_mem_encrypted = false;
 	unsigned int page_order = get_order(size);
 
-	if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
-	    !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
+	/*
+	 * if the device had requested for an unencrypted buffer,
+	 * convert it to encrypted on free
+	 */
+	if (force_dma_unencrypted(dev)) {
+		attrs |= DMA_ATTR_CC_SHARED;
+		mark_mem_encrypted = true;
+	}
+
+	if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) ==
+	     DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
 		/* cpu_addr is a struct page cookie, not a kernel address */
 		dma_free_contiguous(dev, cpu_addr, size);
 		return;

@@ -365,10 +389,14 @@ void dma_direct_free(struct device *dev, size_t size,
 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
 {

kernel/dma/direct.c void dma_direct_free(struct device *dev, size_t size, · line 400

Quoted 10 lines

kernel/dma/direct.c +5 -1

+	unsigned long attrs = 0;
 	struct page *page;
 	void *ret;
 
-	if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
+	if (force_dma_unencrypted(dev))
+		attrs |= DMA_ATTR_CC_SHARED;
+
+	if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);

What about dma_direct_free_pages()? Nothing inside uses attrs, but that’s quite similar to dma_direct_alloc_pages()

Also, at this point, shouldn’t this patch also remove force_dma_unencrypted() calls from dma_set_decrypted() and dma_set_encrypted()?

Teach the atomic DMA pool code to distinguish between encrypted and decrypted pools, and make pool allocation select the matching pool based on DMA attributes.

Introduce a dma_gen_pool wrapper that records whether a pool is decrypted, initialize that state when the atomic pools are created, and use it when expanding and resizing the pools. Update dma_alloc_from_pool() to take attrs and skip pools whose encrypted/decrypted state does not match DMA_ATTR_CC_SHARED. Update dma_free_from_pool() accordingly.

Also pass DMA_ATTR_CC_SHARED from the swiotlb atomic allocation path so decrypted swiotlb allocations are taken from the correct atomic pool.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 drivers/iommu/dma-iommu.c   |   2 +-
 include/linux/dma-map-ops.h |   2 +-
 kernel/dma/direct.c         |  11 ++-
 kernel/dma/pool.c           | 163 +++++++++++++++++++++++-------------
 kernel/dma/swiotlb.c        |   7 +-
 5 files changed, 122 insertions(+), 63 deletions(-)

drivers/iommu/dma-iommu.c +1 -1

index 54d96e847f16..c2595bee3d41 100644
@@ -1673,7 +1673,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
 	    !gfpflags_allow_blocking(gfp) && !coherent)
 		page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
-					       gfp, NULL);
+					   gfp, attrs, NULL);
 	else
 		cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
 	if (!cpu_addr)

include/linux/dma-map-ops.h +1 -1

index 6a1832a73cad..696b2c3a2305 100644
@@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
 void dma_common_free_remap(void *cpu_addr, size_t size);
 
 struct page *dma_alloc_from_pool(struct device *dev, size_t size,
-		void **cpu_addr, gfp_t flags,
+		void **cpu_addr, gfp_t flags, unsigned long attrs,
 		bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
 bool dma_free_from_pool(struct device *dev, void *start, size_t size);
 

kernel/dma/direct.c +7 -4

index 0c2e1f8436ce..dc2907439b3d 100644
@@ -162,7 +162,7 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
 }
 
 static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
-		dma_addr_t *dma_handle, gfp_t gfp)
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
 	struct page *page;
 	u64 phys_limit;

@@ -172,7 +172,8 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
 		return NULL;
 
 	gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
-	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
+	page = dma_alloc_from_pool(dev, size, &ret, gfp, attrs,
+				   dma_coherent_ok);
 	if (!page)
 		return NULL;
 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));

@@ -261,7 +262,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 	 */
 	if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
 	    dma_direct_use_pool(dev, gfp))
-		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+		return dma_direct_alloc_from_pool(dev, size, dma_handle,
+						  gfp, attrs);
 
 	if (is_swiotlb_for_alloc(dev)) {
 		page = dma_direct_alloc_swiotlb(dev, size);

@@ -397,7 +399,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 		attrs |= DMA_ATTR_CC_SHARED;
 
 	if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
-		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+		return dma_direct_alloc_from_pool(dev, size, dma_handle,
+						  gfp, attrs);
 
 	if (is_swiotlb_for_alloc(dev)) {
 		page = dma_direct_alloc_swiotlb(dev, size);

kernel/dma/pool.c +37 -25

index 2b2fbb709242..75f0eba48a23 100644
@@ -12,12 +12,18 @@
 #include <linux/set_memory.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
+#include <linux/cc_platform.h>
 
-static struct gen_pool *atomic_pool_dma __ro_after_init;
+struct dma_gen_pool {
+	bool unencrypted;
+	struct gen_pool *pool;
+};
+
+static struct dma_gen_pool atomic_pool_dma __ro_after_init;
 static unsigned long pool_size_dma;
-static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct dma_gen_pool atomic_pool_dma32 __ro_after_init;
 static unsigned long pool_size_dma32;
-static struct gen_pool *atomic_pool_kernel __ro_after_init;
+static struct dma_gen_pool atomic_pool_kernel __ro_after_init;
 static unsigned long pool_size_kernel;
 
 /* Size can be defined by the coherent_pool command line */

@@ -76,7 +82,7 @@ static bool cma_in_zone(gfp_t gfp)
 	return true;
 }
 
-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
 			      gfp_t gfp)
 {
 	unsigned int order;

@@ -113,12 +119,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
 	 * Memory in the atomic DMA pools must be unencrypted, the pools do not
 	 * shrink so no re-encryption occurs in dma_direct_free().
 	 */
-	ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+	if (dma_pool->unencrypted) {
+		ret = set_memory_decrypted((unsigned long)page_to_virt(page),
 				   1 << order);
-	if (ret)
-		goto remove_mapping;
-	ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
-				pool_size, NUMA_NO_NODE);
+		if (ret)
+			goto remove_mapping;
+	}
+
+	ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+				page_to_phys(page), pool_size, NUMA_NO_NODE);
 	if (ret)
 		goto encrypt_mapping;
 

@@ -126,11 +135,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
 	return 0;
 
 encrypt_mapping:
-	ret = set_memory_encrypted((unsigned long)page_to_virt(page),
-				   1 << order);
-	if (WARN_ON_ONCE(ret)) {
-		/* Decrypt succeeded but encrypt failed, purposely leak */
-		goto out;
+	if (dma_pool->unencrypted) {
+		int rc;
+
+		rc = set_memory_encrypted((unsigned long)page_to_virt(page),
+					  1 << order);
+		if (WARN_ON_ONCE(rc)) {
+			/* Decrypt succeeded but encrypt failed, purposely leak */
+			goto out;
+		}
 	}
 remove_mapping:
 #ifdef CONFIG_DMA_DIRECT_REMAP

@@ -142,46 +155,52 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
 	return ret;
 }
 
-static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp)
 {
-	if (pool && gen_pool_avail(pool) < atomic_pool_size)
-		atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+	if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size)
+		atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp);
 }
 
 static void atomic_pool_work_fn(struct work_struct *work)
 {
 	if (IS_ENABLED(CONFIG_ZONE_DMA))
-		atomic_pool_resize(atomic_pool_dma,
+		atomic_pool_resize(&atomic_pool_dma,
 				   GFP_KERNEL | GFP_DMA);
 	if (IS_ENABLED(CONFIG_ZONE_DMA32))
-		atomic_pool_resize(atomic_pool_dma32,
+		atomic_pool_resize(&atomic_pool_dma32,
 				   GFP_KERNEL | GFP_DMA32);
-	atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL);
+	atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL);
 }
 
-static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size,
-						      gfp_t gfp)
+static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_pool *dma_pool,
+		size_t pool_size, gfp_t gfp)
 {
-	struct gen_pool *pool;
 	int ret;
 
-	pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
-	if (!pool)
+	dma_pool->pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+	if (!dma_pool->pool)

kernel/dma/pool.c +50 -24

-	ret = atomic_pool_expand(pool, pool_size, gfp);
+	ret = atomic_pool_expand(dma_pool, pool_size, gfp);
 	if (ret) {
-		gen_pool_destroy(pool);
+		gen_pool_destroy(dma_pool->pool);
+		dma_pool->pool = NULL;
 		pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
 		       pool_size >> 10, &gfp);
 		return NULL;
 	}
 
 	pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
-		gen_pool_size(pool) >> 10, &gfp);
-	return pool;
+		gen_pool_size(dma_pool->pool) >> 10, &gfp);
+	return dma_pool;
 }
 
 #ifdef CONFIG_ZONE_DMA32

@@ -207,21 +226,22 @@ static int __init dma_atomic_pool_init(void)
 
 	/* All memory might be in the DMA zone(s) to begin with */
 	if (has_managed_zone(ZONE_NORMAL)) {
-		atomic_pool_kernel = __dma_atomic_pool_init(atomic_pool_size,
-						    GFP_KERNEL);
-		if (!atomic_pool_kernel)
+		__dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL);
+		if (!atomic_pool_kernel.pool)
 			ret = -ENOMEM;
 	}
+
 	if (has_managed_dma()) {
-		atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
-						GFP_KERNEL | GFP_DMA);
-		if (!atomic_pool_dma)
+		__dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size,
+				       GFP_KERNEL | GFP_DMA);
+		if (!atomic_pool_dma.pool)
 			ret = -ENOMEM;
 	}
+
 	if (has_managed_dma32) {
-		atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size,
-						GFP_KERNEL | GFP_DMA32);
-		if (!atomic_pool_dma32)
+		__dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size,
+				       GFP_KERNEL | GFP_DMA32);
+		if (!atomic_pool_dma32.pool)
 			ret = -ENOMEM;
 	}
 

@@ -230,19 +250,44 @@ static int __init dma_atomic_pool_init(void)
 }
 postcore_initcall(dma_atomic_pool_init);
 
-static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *first,
+		struct dma_gen_pool *second, struct dma_gen_pool *third)
+{
+	if (first->pool)
+		return first;
+	if (second && second->pool)
+		return second;
+	if (third && third->pool)
+		return third;
+	return NULL;
+}
+
+static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *prev,
+		gfp_t gfp)
 {
-	if (prev == NULL) {
+	if (!prev) {
 		if (gfp & GFP_DMA)
-			return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel;
+			return __dma_guess_pool(&atomic_pool_dma,
+						&atomic_pool_dma32,
+						&atomic_pool_kernel);
+
 		if (gfp & GFP_DMA32)
-			return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel;
-		return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma;
+			return __dma_guess_pool(&atomic_pool_dma32,
+						&atomic_pool_dma,
+						&atomic_pool_kernel);
+
+		return __dma_guess_pool(&atomic_pool_kernel,
+					&atomic_pool_dma32,
+					&atomic_pool_dma);
 	}
-	if (prev == atomic_pool_kernel)
-		return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
-	if (prev == atomic_pool_dma32)
-		return atomic_pool_dma;
+
+	if (prev == &atomic_pool_kernel)
+		return __dma_guess_pool(&atomic_pool_dma32,
+					&atomic_pool_dma, NULL);
+
+	if (prev == &atomic_pool_dma32)
+		return __dma_guess_pool(&atomic_pool_dma, NULL, NULL);
+
 	return NULL;
 }
 

@@ -272,16 +317,20 @@ static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
 }
 
 struct page *dma_alloc_from_pool(struct device *dev, size_t size,
-		void **cpu_addr, gfp_t gfp,
+		void **cpu_addr, gfp_t gfp, unsigned long attrs,
 		bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
 {
-	struct gen_pool *pool = NULL;

kernel/dma/pool.c +7 -5

 		pool_found = true;
-		page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+		page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
 					     phys_addr_ok);
 		if (page)
 			return page;

@@ -296,12 +345,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
 
 bool dma_free_from_pool(struct device *dev, void *start, size_t size)
 {
-	struct gen_pool *pool = NULL;
+	struct dma_gen_pool *dma_pool = NULL;
+
+	while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
 
-	while ((pool = dma_guess_pool(pool, 0))) {
-		if (!gen_pool_has_addr(pool, (unsigned long)start, size))
+		if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size))
 			continue;
-		gen_pool_free(pool, (unsigned long)start, size);
+
+		gen_pool_free(dma_pool->pool, (unsigned long)start, size);
 		return true;
 	}
 

kernel/dma/swiotlb.c +6 -1

index 1abd3e6146f4..ab4eccbaa076 100644
@@ -612,6 +612,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
 		u64 phys_limit, gfp_t gfp)
 {
 	struct page *page;
+	unsigned long attrs = 0;
 
 	/*
 	 * Allocate from the atomic pools if memory is encrypted and

@@ -623,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
 		if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
 			return NULL;
 
+		/* swiotlb considered decrypted by default */
+		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+			attrs = DMA_ATTR_CC_SHARED;
+
 		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
-					   dma_coherent_ok);
+					   attrs, dma_coherent_ok);
 	}
 
 	gfp &= ~GFP_ZONEMASK;

Signature · 1 line

2.43.0

Quoted 10 lines

struct page *dma_alloc_from_pool(struct device *dev, size_t size,

kernel/dma/pool.c +2 -2

-		void **cpu_addr, gfp_t gfp,
+		void **cpu_addr, gfp_t gfp, unsigned long attrs,
 		bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
 {
-	struct gen_pool *pool = NULL;
+	struct dma_gen_pool *dma_pool = NULL;
 	struct page *page;
 	bool pool_found = false;

kernel/dma/pool.c static struct page *__dma_alloc_from_pool(struct device *dev, size_t size, · line 331

Quoted 6 lines

kernel/dma/pool.c +5 -1

-	while ((pool = dma_guess_pool(pool, gfp))) {
+	while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+		if (dma_pool->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+			continue;
+

nit: If we fail to find a matching pool, a slightly misleading message is printed as pool_found = false

kernel/dma/pool.c struct page *dma_alloc_from_pool(struct device *dev, size_t size,

Quoted 8 lines

kernel/dma/pool.c +1 -1

 		pool_found = true;
-		page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+		page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
 					     phys_addr_ok);
 		if (page)
 			return page;

@@ -296,12 +345,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,

Mostafa Saleh <[email protected]> writes:

...

Quoted 10 lines

Quoted 10 lines

struct page *dma_alloc_from_pool(struct device *dev, size_t size,

kernel/dma/pool.c +2 -2

-            void **cpu_addr, gfp_t gfp,
+            void **cpu_addr, gfp_t gfp, unsigned long attrs,
             bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
 {
-    struct gen_pool *pool = NULL;
+    struct dma_gen_pool *dma_pool = NULL;
     struct page *page;
     bool pool_found = false;

Quoted 10 lines

kernel/dma/pool.c static struct page *__dma_alloc_from_pool(struct device *dev, size_t size, · line 331

Quoted 6 lines

kernel/dma/pool.c +5 -1

-    while ((pool = dma_guess_pool(pool, gfp))) {
+    while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+            if (dma_pool->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+                    continue;
+

nit: If we fail to find a matching pool, a slightly misleading message is printed as pool_found = false

The message printed is

        WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev));

That is correct, isn’t it? The kernel failed to find a pool with the correct encryption attribute. For example, the request was for an encrypted allocation from the pool, but no encrypted pool was available.

Quoted 8 lines

kernel/dma/pool.c struct page *dma_alloc_from_pool(struct device *dev, size_t size,

Quoted 8 lines

             pool_found = true;

kernel/dma/pool.c +1 -1

-            page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+            page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
                                          phys_addr_ok);
             if (page)
                     return page;

@@ -296,12 +345,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,

-aneesh

Teach the atomic DMA pool code to distinguish between encrypted and decrypted pools, and make pool allocation select the matching pool based on DMA attributes.

Introduce a dma_gen_pool wrapper that records whether a pool is decrypted, initialize that state when the atomic pools are created, and use it when expanding and resizing the pools. Update dma_alloc_from_pool() to take attrs and skip pools whose encrypted/decrypted state does not match DMA_ATTR_CC_SHARED. Update dma_free_from_pool() accordingly.

Also pass DMA_ATTR_CC_SHARED from the swiotlb atomic allocation path so decrypted swiotlb allocations are taken from the correct atomic pool.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
  drivers/iommu/dma-iommu.c   |   2 +-
  include/linux/dma-map-ops.h |   2 +-
  kernel/dma/direct.c         |  11 ++-
  kernel/dma/pool.c           | 163 +++++++++++++++++++++++-------------
  kernel/dma/swiotlb.c        |   7 +-
  5 files changed, 122 insertions(+), 63 deletions(-)

drivers/iommu/dma-iommu.c +1 -1

index 54d96e847f16..c2595bee3d41 100644
@@ -1673,7 +1673,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
  	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
  	    !gfpflags_allow_blocking(gfp) && !coherent)
  		page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
-					       gfp, NULL);
+					   gfp, attrs, NULL);
  	else
  		cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
  	if (!cpu_addr)

include/linux/dma-map-ops.h +1 -1

index 6a1832a73cad..696b2c3a2305 100644
@@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
  void dma_common_free_remap(void *cpu_addr, size_t size);
 
  struct page *dma_alloc_from_pool(struct device *dev, size_t size,
-		void **cpu_addr, gfp_t flags,
+		void **cpu_addr, gfp_t flags, unsigned long attrs,
  		bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
  bool dma_free_from_pool(struct device *dev, void *start, size_t size);
 

kernel/dma/direct.c +7 -4

index 0c2e1f8436ce..dc2907439b3d 100644
@@ -162,7 +162,7 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
  }
 
  static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
-		dma_addr_t *dma_handle, gfp_t gfp)
+		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
  {
  	struct page *page;
  	u64 phys_limit;

@@ -172,7 +172,8 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
  		return NULL;
 
  	gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
-	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
+	page = dma_alloc_from_pool(dev, size, &ret, gfp, attrs,
+				   dma_coherent_ok);
  	if (!page)
  		return NULL;
  	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));

@@ -261,7 +262,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
  	 */
  	if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
  	    dma_direct_use_pool(dev, gfp))
-		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+		return dma_direct_alloc_from_pool(dev, size, dma_handle,
+						  gfp, attrs);
 
  	if (is_swiotlb_for_alloc(dev)) {
  		page = dma_direct_alloc_swiotlb(dev, size);

@@ -397,7 +399,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
  		attrs |= DMA_ATTR_CC_SHARED;
 
  	if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
-		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+		return dma_direct_alloc_from_pool(dev, size, dma_handle,
+						  gfp, attrs);
 
  	if (is_swiotlb_for_alloc(dev)) {
  		page = dma_direct_alloc_swiotlb(dev, size);

kernel/dma/pool.c +101 -52

index 2b2fbb709242..75f0eba48a23 100644
@@ -12,12 +12,18 @@
  #include <linux/set_memory.h>
  #include <linux/slab.h>
  #include <linux/workqueue.h>
+#include <linux/cc_platform.h>
 
-static struct gen_pool *atomic_pool_dma __ro_after_init;
+struct dma_gen_pool {
+	bool unencrypted;
+	struct gen_pool *pool;
+};
+
+static struct dma_gen_pool atomic_pool_dma __ro_after_init;
  static unsigned long pool_size_dma;
-static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct dma_gen_pool atomic_pool_dma32 __ro_after_init;
  static unsigned long pool_size_dma32;
-static struct gen_pool *atomic_pool_kernel __ro_after_init;
+static struct dma_gen_pool atomic_pool_kernel __ro_after_init;
  static unsigned long pool_size_kernel;
 
  /* Size can be defined by the coherent_pool command line */

@@ -76,7 +82,7 @@ static bool cma_in_zone(gfp_t gfp)
  	return true;
  }
 
-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
  			      gfp_t gfp)
  {
  	unsigned int order;

@@ -113,12 +119,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
  	 * Memory in the atomic DMA pools must be unencrypted, the pools do not
  	 * shrink so no re-encryption occurs in dma_direct_free().
  	 */
-	ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+	if (dma_pool->unencrypted) {
+		ret = set_memory_decrypted((unsigned long)page_to_virt(page),
  				   1 << order);
-	if (ret)
-		goto remove_mapping;
-	ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
-				pool_size, NUMA_NO_NODE);
+		if (ret)
+			goto remove_mapping;
+	}
+
+	ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+				page_to_phys(page), pool_size, NUMA_NO_NODE);
  	if (ret)
  		goto encrypt_mapping;
 

@@ -126,11 +135,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
  	return 0;
 
  encrypt_mapping:
-	ret = set_memory_encrypted((unsigned long)page_to_virt(page),
-				   1 << order);
-	if (WARN_ON_ONCE(ret)) {
-		/* Decrypt succeeded but encrypt failed, purposely leak */
-		goto out;
+	if (dma_pool->unencrypted) {
+		int rc;
+
+		rc = set_memory_encrypted((unsigned long)page_to_virt(page),
+					  1 << order);
+		if (WARN_ON_ONCE(rc)) {
+			/* Decrypt succeeded but encrypt failed, purposely leak */
+			goto out;
+		}
  	}
  remove_mapping:
  #ifdef CONFIG_DMA_DIRECT_REMAP

@@ -142,46 +155,52 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
  	return ret;
  }
 
-static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp)
  {
-	if (pool && gen_pool_avail(pool) < atomic_pool_size)
-		atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+	if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size)
+		atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp);
  }
 
  static void atomic_pool_work_fn(struct work_struct *work)
  {
  	if (IS_ENABLED(CONFIG_ZONE_DMA))
-		atomic_pool_resize(atomic_pool_dma,
+		atomic_pool_resize(&atomic_pool_dma,
  				   GFP_KERNEL | GFP_DMA);
  	if (IS_ENABLED(CONFIG_ZONE_DMA32))
-		atomic_pool_resize(atomic_pool_dma32,
+		atomic_pool_resize(&atomic_pool_dma32,
  				   GFP_KERNEL | GFP_DMA32);
-	atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL);
+	atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL);
  }
 
-static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size,
-						      gfp_t gfp)
+static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_pool *dma_pool,
+		size_t pool_size, gfp_t gfp)
  {
-	struct gen_pool *pool;
  	int ret;
 
-	pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
-	if (!pool)
+	dma_pool->pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+	if (!dma_pool->pool)
  		return NULL;
 
-	gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);
+	gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL);
+
+	/* if platform is using memory encryption atomic pools are by default decrypted. */
+	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+		dma_pool->unencrypted = true;
+	else
+		dma_pool->unencrypted = false;
 
-	ret = atomic_pool_expand(pool, pool_size, gfp);
+	ret = atomic_pool_expand(dma_pool, pool_size, gfp);
  	if (ret) {
-		gen_pool_destroy(pool);
+		gen_pool_destroy(dma_pool->pool);
+		dma_pool->pool = NULL;
  		pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
  		       pool_size >> 10, &gfp);
  		return NULL;
  	}
 
  	pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
-		gen_pool_size(pool) >> 10, &gfp);
-	return pool;
+		gen_pool_size(dma_pool->pool) >> 10, &gfp);
+	return dma_pool;
  }
 
  #ifdef CONFIG_ZONE_DMA32

@@ -207,21 +226,22 @@ static int __init dma_atomic_pool_init(void)
 
  	/* All memory might be in the DMA zone(s) to begin with */
  	if (has_managed_zone(ZONE_NORMAL)) {
-		atomic_pool_kernel = __dma_atomic_pool_init(atomic_pool_size,
-						    GFP_KERNEL);
-		if (!atomic_pool_kernel)
+		__dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL);
+		if (!atomic_pool_kernel.pool)
  			ret = -ENOMEM;
  	}
+
  	if (has_managed_dma()) {
-		atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
-						GFP_KERNEL | GFP_DMA);
-		if (!atomic_pool_dma)
+		__dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size,
+				       GFP_KERNEL | GFP_DMA);
+		if (!atomic_pool_dma.pool)
  			ret = -ENOMEM;
  	}
+
  	if (has_managed_dma32) {
-		atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size,
-						GFP_KERNEL | GFP_DMA32);
-		if (!atomic_pool_dma32)
+		__dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size,
+				       GFP_KERNEL | GFP_DMA32);
+		if (!atomic_pool_dma32.pool)
  			ret = -ENOMEM;
  	}
 

@@ -230,19 +250,44 @@ static int __init dma_atomic_pool_init(void)
  }
  postcore_initcall(dma_atomic_pool_init);
 
-static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *first,
+		struct dma_gen_pool *second, struct dma_gen_pool *third)
+{
+	if (first->pool)
+		return first;
+	if (second && second->pool)
+		return second;
+	if (third && third->pool)
+		return third;
+	return NULL;
+}
+
+static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *prev,
+		gfp_t gfp)
  {
-	if (prev == NULL) {
+	if (!prev) {
  		if (gfp & GFP_DMA)
-			return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel;
+			return __dma_guess_pool(&atomic_pool_dma,
+						&atomic_pool_dma32,
+						&atomic_pool_kernel);
+
  		if (gfp & GFP_DMA32)
-			return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel;
-		return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma;
+			return __dma_guess_pool(&atomic_pool_dma32,
+						&atomic_pool_dma,
+						&atomic_pool_kernel);
+
+		return __dma_guess_pool(&atomic_pool_kernel,
+					&atomic_pool_dma32,
+					&atomic_pool_dma);
  	}
-	if (prev == atomic_pool_kernel)
-		return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
-	if (prev == atomic_pool_dma32)
-		return atomic_pool_dma;
+
+	if (prev == &atomic_pool_kernel)
+		return __dma_guess_pool(&atomic_pool_dma32,
+					&atomic_pool_dma, NULL);
+
+	if (prev == &atomic_pool_dma32)
+		return __dma_guess_pool(&atomic_pool_dma, NULL, NULL);
+
  	return NULL;
  }
 

@@ -272,16 +317,20 @@ static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
  }
 
  struct page *dma_alloc_from_pool(struct device *dev, size_t size,
-		void **cpu_addr, gfp_t gfp,
+		void **cpu_addr, gfp_t gfp, unsigned long attrs,
  		bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
  {
-	struct gen_pool *pool = NULL;
+	struct dma_gen_pool *dma_pool = NULL;
  	struct page *page;
  	bool pool_found = false;
 
-	while ((pool = dma_guess_pool(pool, gfp))) {
+	while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+		if (dma_pool->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+			continue;
+
  		pool_found = true;
-		page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+		page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
  					     phys_addr_ok);
  		if (page)
  			return page;

@@ -296,12 +345,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
 

kernel/dma/pool.c +2 -1

  			continue;
-		gen_pool_free(pool, (unsigned long)start, size);
+
+		gen_pool_free(dma_pool->pool, (unsigned long)start, size);
  		return true;
  	}
 

kernel/dma/swiotlb.c +6 -1

index 1abd3e6146f4..ab4eccbaa076 100644
@@ -612,6 +612,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
  		u64 phys_limit, gfp_t gfp)
  {
  	struct page *page;
+	unsigned long attrs = 0;
 
  	/*
  	 * Allocate from the atomic pools if memory is encrypted and

@@ -623,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
  		if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
  			return NULL;
 
+		/* swiotlb considered decrypted by default */
+		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+			attrs = DMA_ATTR_CC_SHARED;
+
  		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
-					   dma_coherent_ok);
+					   attrs, dma_coherent_ok);
  	}
 
  	gfp &= ~GFP_ZONEMASK;

Quoted 126 lines · Aneesh Kumar K.V (Arm)

Teach the atomic DMA pool code to distinguish between encrypted and decrypted pools, and make pool allocation select the matching pool based on DMA attributes.

Introduce a dma_gen_pool wrapper that records whether a pool is decrypted, initialize that state when the atomic pools are created, and use it when expanding and resizing the pools.  Update dma_alloc_from_pool() to take attrs and skip pools whose encrypted/decrypted state does not match DMA_ATTR_CC_SHARED.  Update dma_free_from_pool() accordingly.

Also pass DMA_ATTR_CC_SHARED from the swiotlb atomic allocation path so decrypted swiotlb allocations are taken from the correct atomic pool.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
  drivers/iommu/dma-iommu.c   |   2 +- include/linux/dma-map-ops.h |   2 +-
  kernel/dma/direct.c         |  11 ++-
  kernel/dma/pool.c           | 163 +++++++++++++++++++++++------------- kernel/dma/swiotlb.c        |   7 +-

  5 files changed, 122 insertions(+), 63 deletions(-)

drivers/iommu/dma-iommu.c +0 -0

index 54d96e847f16..c2595bee3d41 100644
@@ -1673,7 +1673,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,

      if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && !gfpflags_allow_blocking(gfp) && !coherent)
          page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, -                           gfp, NULL);
+                       gfp, attrs, NULL);
      else
          cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); if (!cpu_addr)

include/linux/dma-map-ops.h +1 -1

index 6a1832a73cad..696b2c3a2305 100644
@@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
   void dma_common_free_remap(void *cpu_addr, size_t size);
   struct page *dma_alloc_from_pool(struct device *dev, size_t size,
-        void **cpu_addr, gfp_t flags,
+        void **cpu_addr, gfp_t flags, unsigned long attrs,
           bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
   bool dma_free_from_pool(struct device *dev, void *start, size_t size);

kernel/dma/direct.c +3 -2

index 0c2e1f8436ce..dc2907439b3d 100644
@@ -162,7 +162,7 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
   }
   static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
-        dma_addr_t *dma_handle, gfp_t gfp)
+        dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
   {
       struct page *page;
       u64 phys_limit;

@@ -172,7 +172,8 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
           return NULL;
       gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
-    page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
+    page = dma_alloc_from_pool(dev, size, &ret, gfp, attrs,
+                   dma_coherent_ok);
       if (!page)
           return NULL;
       *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));

@@ -261,7 +262,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
        */

      if ((remap || (attrs & DMA_ATTR_CC_SHARED)) && dma_direct_use_pool(dev, gfp))
-        return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); +        return dma_direct_alloc_from_pool(dev, size, dma_handle, +                          gfp, attrs);
      if (is_swiotlb_for_alloc(dev)) {
          page = dma_direct_alloc_swiotlb(dev, size);

kernel/dma/direct.c +2 -1

@@ -397,7 +399,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
           attrs |= DMA_ATTR_CC_SHARED;
       if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
-        return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+        return dma_direct_alloc_from_pool(dev, size, dma_handle,
+                          gfp, attrs);
       if (is_swiotlb_for_alloc(dev)) {
           page = dma_direct_alloc_swiotlb(dev, size);

kernel/dma/pool.c +12 -7

index 2b2fbb709242..75f0eba48a23 100644
@@ -12,12 +12,18 @@
   #include <linux/set_memory.h>
   #include <linux/slab.h>
   #include <linux/workqueue.h>
+#include <linux/cc_platform.h>
-static struct gen_pool *atomic_pool_dma __ro_after_init;
+struct dma_gen_pool {
+    bool unencrypted;
+    struct gen_pool *pool;
+};
+
+static struct dma_gen_pool atomic_pool_dma __ro_after_init;
   static unsigned long pool_size_dma;
-static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct dma_gen_pool atomic_pool_dma32 __ro_after_init;
   static unsigned long pool_size_dma32;
-static struct gen_pool *atomic_pool_kernel __ro_after_init;
+static struct dma_gen_pool atomic_pool_kernel __ro_after_init;
   static unsigned long pool_size_kernel;
   /* Size can be defined by the coherent_pool command line */

@@ -76,7 +82,7 @@ static bool cma_in_zone(gfp_t gfp)
       return true;
   }
-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
                     gfp_t gfp)
   {
       unsigned int order;

@@ -113,12 +119,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
        * Memory in the atomic DMA pools must be unencrypted, the pools do not
        * shrink so no re-encryption occurs in dma_direct_free().
        */
-    ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+    if (dma_pool->unencrypted) {
+        ret = set_memory_decrypted((unsigned long)page_to_virt(page),
                      1 << order);
-    if (ret)
-        goto remove_mapping;

kernel/dma/pool.c static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, · line 130

Quoted 8 lines

kernel/dma/pool.c +6 -2

-    ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
-                pool_size, NUMA_NO_NODE);
+        if (ret)
+            goto remove_mapping;
+    }
+
+    ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+                page_to_phys(page), pool_size, NUMA_NO_NODE);

Quoted 193 lines

kernel/dma/pool.c +0 -0

       if (ret)
           goto encrypt_mapping;

@@ -126,11 +135,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
       return 0;

  encrypt_mapping:
-    ret = set_memory_encrypted((unsigned long)page_to_virt(page), -                   1 << order);
-    if (WARN_ON_ONCE(ret)) {
-        /* Decrypt succeeded but encrypt failed, purposely leak */ -        goto out;
+    if (dma_pool->unencrypted) {
+        int rc;
+
+        rc = set_memory_encrypted((unsigned long)page_to_virt(page), +                      1 << order);
+        if (WARN_ON_ONCE(rc)) {
+            /* Decrypt succeeded but encrypt failed, purposely leak */ +            goto out;
+        }
      }
  remove_mapping:
  #ifdef CONFIG_DMA_DIRECT_REMAP

kernel/dma/pool.c +75 -39

@@ -142,46 +155,52 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
       return ret;
   }
-static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp)
   {
-    if (pool && gen_pool_avail(pool) < atomic_pool_size)
-        atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+    if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size)
+        atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp);
   }
   static void atomic_pool_work_fn(struct work_struct *work)
   {
       if (IS_ENABLED(CONFIG_ZONE_DMA))
-        atomic_pool_resize(atomic_pool_dma,
+        atomic_pool_resize(&atomic_pool_dma,
                      GFP_KERNEL | GFP_DMA);
       if (IS_ENABLED(CONFIG_ZONE_DMA32))
-        atomic_pool_resize(atomic_pool_dma32,
+        atomic_pool_resize(&atomic_pool_dma32,
                      GFP_KERNEL | GFP_DMA32);
-    atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL);
+    atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL);
   }
-static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size,
-                              gfp_t gfp)
+static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_pool *dma_pool,
+        size_t pool_size, gfp_t gfp)
   {
-    struct gen_pool *pool;
       int ret;
-    pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
-    if (!pool)
+    dma_pool->pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+    if (!dma_pool->pool)
           return NULL;
-    gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);
+    gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL);
+
+    /* if platform is using memory encryption atomic pools are by default decrypted. */
+    if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+        dma_pool->unencrypted = true;
+    else
+        dma_pool->unencrypted = false;
-    ret = atomic_pool_expand(pool, pool_size, gfp);
+    ret = atomic_pool_expand(dma_pool, pool_size, gfp);
       if (ret) {
-        gen_pool_destroy(pool);
+        gen_pool_destroy(dma_pool->pool);
+        dma_pool->pool = NULL;
           pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
                  pool_size >> 10, &gfp);
           return NULL;
       }
       pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
-        gen_pool_size(pool) >> 10, &gfp);
-    return pool;
+        gen_pool_size(dma_pool->pool) >> 10, &gfp);
+    return dma_pool;
   }
   #ifdef CONFIG_ZONE_DMA32

@@ -207,21 +226,22 @@ static int __init dma_atomic_pool_init(void)
       /* All memory might be in the DMA zone(s) to begin with */
       if (has_managed_zone(ZONE_NORMAL)) {
-        atomic_pool_kernel = __dma_atomic_pool_init(atomic_pool_size,
-                            GFP_KERNEL);
-        if (!atomic_pool_kernel)
+        __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL);
+        if (!atomic_pool_kernel.pool)
               ret = -ENOMEM;
       }
+
       if (has_managed_dma()) {
-        atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
-                        GFP_KERNEL | GFP_DMA);
-        if (!atomic_pool_dma)
+        __dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size,
+                       GFP_KERNEL | GFP_DMA);
+        if (!atomic_pool_dma.pool)
               ret = -ENOMEM;
       }
+
       if (has_managed_dma32) {
-        atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size,
-                        GFP_KERNEL | GFP_DMA32);
-        if (!atomic_pool_dma32)
+        __dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size,
+                       GFP_KERNEL | GFP_DMA32);
+        if (!atomic_pool_dma32.pool)
               ret = -ENOMEM;
       }

@@ -230,19 +250,44 @@ static int __init dma_atomic_pool_init(void)
   }
   postcore_initcall(dma_atomic_pool_init);
-static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *first,
+        struct dma_gen_pool *second, struct dma_gen_pool *third)
+{
+    if (first->pool)
+        return first;
+    if (second && second->pool)
+        return second;
+    if (third && third->pool)
+        return third;
+    return NULL;
+}
+
+static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *prev,
+        gfp_t gfp)
   {
-    if (prev == NULL) {
+    if (!prev) {
           if (gfp & GFP_DMA)
-            return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel;
+            return __dma_guess_pool(&atomic_pool_dma,
+                        &atomic_pool_dma32,
+                        &atomic_pool_kernel);
+
           if (gfp & GFP_DMA32)
-            return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel;
-        return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma;
+            return __dma_guess_pool(&atomic_pool_dma32,
+                        &atomic_pool_dma,
+                        &atomic_pool_kernel);
+
+        return __dma_guess_pool(&atomic_pool_kernel,
+                    &atomic_pool_dma32,
+                    &atomic_pool_dma);
       }
-    if (prev == atomic_pool_kernel)
-        return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
-    if (prev == atomic_pool_dma32)
-        return atomic_pool_dma;
+
+    if (prev == &atomic_pool_kernel)
+        return __dma_guess_pool(&atomic_pool_dma32,
+                    &atomic_pool_dma, NULL);
+
+    if (prev == &atomic_pool_dma32)
+        return __dma_guess_pool(&atomic_pool_dma, NULL, NULL);
+
       return NULL;
   }

@@ -272,16 +317,20 @@ static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
   }
   struct page *dma_alloc_from_pool(struct device *dev, size_t size,
-        void **cpu_addr, gfp_t gfp,
+        void **cpu_addr, gfp_t gfp, unsigned long attrs,
           bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
   {
-    struct gen_pool *pool = NULL;
+    struct dma_gen_pool *dma_pool = NULL;
       struct page *page;
       bool pool_found = false;
-    while ((pool = dma_guess_pool(pool, gfp))) {
+    while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+        if (dma_pool->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+            continue;
+
           pool_found = true;
-        page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+        page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
                            phys_addr_ok);
           if (page)
               return page;

@@ -296,12 +345,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
   bool dma_free_from_pool(struct device *dev, void *start, size_t size)
   {
-    struct gen_pool *pool = NULL;
+    struct dma_gen_pool *dma_pool = NULL;

kernel/dma/pool.c struct page *dma_alloc_from_pool(struct device *dev, size_t size, · line 352

Quoted 5 lines

kernel/dma/pool.c +3 -2

+
+    while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
-    while ((pool = dma_guess_pool(pool, 0))) {
-        if (!gen_pool_has_addr(pool, (unsigned long)start, size))
+        if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size))

v3 of this just crashed here with dma_pool!=NULL but dma_pool->pool==NULL. continuing debugging... Thanks,

Quoted 31 lines

kernel/dma/pool.c +2 -1

               continue;
-        gen_pool_free(pool, (unsigned long)start, size);
+
+        gen_pool_free(dma_pool->pool, (unsigned long)start, size);
           return true;
       }

kernel/dma/swiotlb.c +6 -1

index 1abd3e6146f4..ab4eccbaa076 100644
@@ -612,6 +612,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
           u64 phys_limit, gfp_t gfp)
   {
       struct page *page;
+    unsigned long attrs = 0;
       /*
        * Allocate from the atomic pools if memory is encrypted and

@@ -623,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
           if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
               return NULL;
+        /* swiotlb considered decrypted by default */
+        if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+            attrs = DMA_ATTR_CC_SHARED;
+
           return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
-                       dma_coherent_ok);
+                       attrs, dma_coherent_ok);
       }
       gfp &= ~GFP_ZONEMASK;

Quoted 1 line · Alexey Kardashevskiy

On 12/5/26 19:03, Aneesh Kumar K.V (Arm) wrote:

Quoted 18 lines

Quoted 18 lines

kernel/dma/pool.c +1 -1

-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,

                    gfp_t gfp)
  {
      unsigned int order;

kernel/dma/pool.c +3 -5

@@ -113,12 +119,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
        * Memory in the atomic DMA pools must be unencrypted, the pools do not
        * shrink so no re-encryption occurs in dma_direct_free().
        */
-    ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+    if (dma_pool->unencrypted) {
+        ret = set_memory_decrypted((unsigned long)page_to_virt(page),
                      1 << order);
-    if (ret)
-        goto remove_mapping;
-    ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
-                pool_size, NUMA_NO_NODE);
+        if (ret)

Quoted 5 lines

kernel/dma/pool.c static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, · line 130

Quoted 5 lines

kernel/dma/pool.c +5 -0

+            goto remove_mapping;
+    }
+
+    ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+                page_to_phys(page), pool_size, NUMA_NO_NODE);

This clause could go to the else branch.

Quoted 25 lines

Quoted 25 lines

kernel/dma/pool.c +0 -0

       if (ret)
           goto encrypt_mapping;

@@ -126,11 +135,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
       return 0;

  encrypt_mapping:
-    ret = set_memory_encrypted((unsigned long)page_to_virt(page), -                   1 << order);
-    if (WARN_ON_ONCE(ret)) {
-        /* Decrypt succeeded but encrypt failed, purposely leak */ -        goto out;
+    if (dma_pool->unencrypted) {
+        int rc;
+
+        rc = set_memory_encrypted((unsigned long)page_to_virt(page), +                      1 << order);
+        if (WARN_ON_ONCE(rc)) {
+            /* Decrypt succeeded but encrypt failed, purposely leak */ +            goto out;
+        }
      }
  remove_mapping:
  #ifdef CONFIG_DMA_DIRECT_REMAP

kernel/dma/pool.c +0 -0

@@ -142,46 +155,52 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
       return ret;
   }

Quoted 11 lines

Quoted 9 lines

  bool dma_free_from_pool(struct device *dev, void *start, size_t size) {
-    struct gen_pool *pool = NULL;
+    struct dma_gen_pool *dma_pool = NULL;
+
+    while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
-    while ((pool = dma_guess_pool(pool, 0))) {
-        if (!gen_pool_has_addr(pool, (unsigned long)start, size)) +        if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size))

Quoted 1 line

v3 of this just crashed here with dma_pool!=NULL but dma_pool->pool==NULL. continuing debugging... Thanks,

dma_direct_free:
dma_free_from_pool (loop over pools) -> false

     [here was a crash which I fixed by "if (!dma_pool->pool) continue"]

swiotlb_find_pool (loop again) -> false
__dma_direct_free_pages
swiotlb_free
swiotlb_find_pool (loop again)
dma_free_contiguous => done.

so that works but kinda hard to follow and there is some room for optimization. I do not normally have swiottlb when I test this and there is too many of this swiotlb stuff on the real direct dma mapping path imho. Thanks,

Quoted 24 lines

Quoted 24 lines

              continue;
-        gen_pool_free(pool, (unsigned long)start, size); +
+        gen_pool_free(dma_pool->pool, (unsigned long)start, size); return true;
      }

kernel/dma/swiotlb.c +3 -0

index 1abd3e6146f4..ab4eccbaa076 100644
@@ -612,6 +612,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
           u64 phys_limit, gfp_t gfp)
   {
       struct page *page;
+    unsigned long attrs = 0;
       /*
        * Allocate from the atomic pools if memory is encrypted and

@@ -623,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
           if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
               return NULL;
+        /* swiotlb considered decrypted by default */
+        if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))

Quoted 8 lines

Quoted 7 lines

kernel/dma/swiotlb.c +3 -1

+            attrs = DMA_ATTR_CC_SHARED;
+
           return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
-                       dma_coherent_ok);
+                       attrs, dma_coherent_ok);
       }
       gfp &= ~GFP_ZONEMASK;

Signature · 1 line

Alexey

Teach swiotlb to distinguish between encrypted and decrypted bounce buffer pools, and make allocation and mapping paths select a pool whose state matches the requested DMA attributes.

Add a decrypted flag to io_tlb_mem, initialize it for the default and restricted pools, and propagate DMA_ATTR_CC_SHARED into swiotlb pool allocation. Reject swiotlb alloc/map requests when the selected pool does not match the required encrypted/decrypted state.

Also return DMA addresses with the matching phys_to_dma_{encrypted, unencrypted} helper so the DMA address encoding stays consistent with the chosen pool.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 include/linux/dma-direct.h |  10 ++++
 include/linux/swiotlb.h    |   8 ++-
 kernel/dma/direct.c        |  14 +++--
 kernel/dma/swiotlb.c       | 108 +++++++++++++++++++++++++++----------
 4 files changed, 107 insertions(+), 33 deletions(-)

include/linux/dma-direct.h +10 -0

index c249912456f9..94fad4e7c11e 100644
@@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map)
 #ifndef phys_to_dma_unencrypted
 #define phys_to_dma_unencrypted		phys_to_dma
 #endif
+
+#ifndef phys_to_dma_encrypted
+#define phys_to_dma_encrypted		phys_to_dma
+#endif
 #else
 static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
 {

@@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
 {
 	return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
 }
+
+static inline dma_addr_t phys_to_dma_encrypted(struct device *dev,
+		phys_addr_t paddr)
+{
+	return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
 /*
  * If memory encryption is supported, phys_to_dma will set the memory encryption
  * bit in the DMA address, and dma_to_phys will clear it.

include/linux/swiotlb.h +6 -2

index 3dae0f592063..b3fa3c6e0169 100644
@@ -81,6 +81,7 @@ struct io_tlb_pool {
 	struct list_head node;
 	struct rcu_head rcu;
 	bool transient;
+	bool unencrypted;
 #endif
 };
 

@@ -111,6 +112,7 @@ struct io_tlb_mem {
 	struct dentry *debugfs;
 	bool force_bounce;
 	bool for_alloc;
+	bool unencrypted;
 #ifdef CONFIG_SWIOTLB_DYNAMIC
 	bool can_grow;
 	u64 phys_limit;

@@ -282,7 +284,8 @@ static inline void swiotlb_sync_single_for_cpu(struct device *dev,
 extern void swiotlb_print_info(void);
 
 #ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size);
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+		unsigned long attrs);
 bool swiotlb_free(struct device *dev, struct page *page, size_t size);
 
 static inline bool is_swiotlb_for_alloc(struct device *dev)

@@ -290,7 +293,8 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
 	return dev->dma_io_tlb_mem->for_alloc;
 }
 #else
-static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
+static inline struct page *swiotlb_alloc(struct device *dev, size_t size,
+		unsigned long attrs)
 {
 	return NULL;
 }

kernel/dma/direct.c +10 -4

index dc2907439b3d..97ae4fa10521 100644
@@ -104,9 +104,10 @@ static void __dma_direct_free_pages(struct device *dev, struct page *page,
 	dma_free_contiguous(dev, page, size);
 }
 
-static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
+static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,
+		unsigned long attrs)
 {
-	struct page *page = swiotlb_alloc(dev, size);
+	struct page *page = swiotlb_alloc(dev, size, attrs);
 
 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
 		swiotlb_free(dev, page, size);

@@ -266,8 +267,12 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 						  gfp, attrs);
 
 	if (is_swiotlb_for_alloc(dev)) {
-		page = dma_direct_alloc_swiotlb(dev, size);
+		page = dma_direct_alloc_swiotlb(dev, size, attrs);
 		if (page) {
+			/*
+			 * swiotlb allocations comes from pool already marked
+			 * decrypted
+			 */
 			mark_mem_decrypt = false;
 			goto setup_page;
 		}

@@ -374,6 +379,7 @@ void dma_direct_free(struct device *dev, size_t size,
 		return;
 
 	if (swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr)))
+		/* Swiotlb doesn't need a page attribute update on free */
 		mark_mem_encrypted = false;
 
 	if (is_vmalloc_addr(cpu_addr)) {

@@ -403,7 +409,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 						  gfp, attrs);
 
 	if (is_swiotlb_for_alloc(dev)) {
-		page = dma_direct_alloc_swiotlb(dev, size);
+		page = dma_direct_alloc_swiotlb(dev, size, attrs);
 		if (!page)
 			return NULL;
 

kernel/dma/swiotlb.c +71 -25

index ab4eccbaa076..065663be282c 100644
@@ -259,10 +259,21 @@ void __init swiotlb_update_mem_attributes(void)
 	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 	unsigned long bytes;
 
+	/*
+	 * if platform support memory encryption, swiotlb buffers are
+	 * decrypted by default.
+	 */
+	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+		io_tlb_default_mem.unencrypted = true;
+	else
+		io_tlb_default_mem.unencrypted = false;
+
 	if (!mem->nslabs || mem->late_alloc)
 		return;
 	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
-	set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
+
+	if (io_tlb_default_mem.unencrypted)
+		set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
 }
 
 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,

@@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
 	if (!mem->slots)
 		goto error_slots;
 
-	set_memory_decrypted((unsigned long)vstart,
-			     (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+	if (io_tlb_default_mem.unencrypted)
+		set_memory_decrypted((unsigned long)vstart,
+				     (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+
 	swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
 				 nareas);
 	add_mem_pool(&io_tlb_default_mem, mem);

@@ -539,7 +552,9 @@ void __init swiotlb_exit(void)
 	tbl_size = PAGE_ALIGN(mem->end - mem->start);
 	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
 
-	set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+	if (io_tlb_default_mem.unencrypted)
+		set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+
 	if (mem->late_alloc) {
 		area_order = get_order(array_size(sizeof(*mem->areas),
 			mem->nareas));

@@ -563,6 +578,7 @@ void __init swiotlb_exit(void)
  * @gfp:	GFP flags for the allocation.
  * @bytes:	Size of the buffer.
  * @phys_limit:	Maximum allowed physical address of the buffer.
+ * @unencrypted: true to allocate unencrypted memory, false for encrypted memory
  *
  * Allocate pages from the buddy allocator. If successful, make the allocated
  * pages decrypted that they can be used for DMA.

@@ -570,7 +586,8 @@ void __init swiotlb_exit(void)
  * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
  * if the allocated physical address was above @phys_limit.
  */
-static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
+static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes,
+		u64 phys_limit, bool unencrypted)
 {
 	unsigned int order = get_order(bytes);
 	struct page *page;

@@ -588,13 +605,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
 	}
 
 	vaddr = phys_to_virt(paddr);
-	if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
+	if (unencrypted && set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
 		goto error;
 	return page;
 
 error:
 	/* Intentional leak if pages cannot be encrypted again. */
-	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+	if (unencrypted && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 		__free_pages(page, order);
 	return NULL;
 }

@@ -604,30 +621,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
  * @dev:	Device for which a memory pool is allocated.
  * @bytes:	Size of the buffer.
  * @phys_limit:	Maximum allowed physical address of the buffer.
+ * @attrs:	DMA attributes for the allocation.
  * @gfp:	GFP flags for the allocation.
  *
  * Return: Allocated pages, or %NULL on allocation failure.
  */
 static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
-		u64 phys_limit, gfp_t gfp)
+		u64 phys_limit, unsigned long attrs, gfp_t gfp)
 {
 	struct page *page;
-	unsigned long attrs = 0;
 
 	/*
 	 * Allocate from the atomic pools if memory is encrypted and
 	 * the allocation is atomic, because decrypting may block.
 	 */
-	if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
+	if (!gfpflags_allow_blocking(gfp) && (attrs & DMA_ATTR_CC_SHARED)) {
 		void *vaddr;
 
 		if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
 			return NULL;
 
-		/* swiotlb considered decrypted by default */
-		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
-			attrs = DMA_ATTR_CC_SHARED;
-
 		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
 					   attrs, dma_coherent_ok);
 	}

@@ -638,7 +651,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
 	else if (phys_limit <= DMA_BIT_MASK(32))
 		gfp |= __GFP_DMA32;
 
-	while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
+	while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit,
+					     !!(attrs & DMA_ATTR_CC_SHARED)))) {
 		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 		    phys_limit < DMA_BIT_MASK(64) &&
 		    !(gfp & (__GFP_DMA32 | __GFP_DMA)))

@@ -657,15 +671,18 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
  * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
  * @vaddr:	Virtual address of the buffer.
  * @bytes:	Size of the buffer.
+ * @unencrypted: true if @vaddr was allocated decrypted and must be
+ *	re-encrypted before being freed
  */
-static void swiotlb_free_tlb(void *vaddr, size_t bytes)
+static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool unencrypted)
 {
 	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
 	    dma_free_from_pool(NULL, vaddr, bytes))
 		return;
 
 	/* Intentional leak if pages cannot be encrypted again. */
-	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+	if (!unencrypted ||
+	    !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 		__free_pages(virt_to_page(vaddr), get_order(bytes));
 }
 

@@ -676,6 +693,7 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
  * @nslabs:	Desired (maximum) number of slabs.
  * @nareas:	Number of areas.
  * @phys_limit:	Maximum DMA buffer physical address.
+ * @attrs:	DMA attributes for the allocation.
  * @gfp:	GFP flags for the allocations.
  *
  * Allocate and initialize a new IO TLB memory pool. The actual number of

@@ -686,7 +704,8 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
  */
 static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 		unsigned long minslabs, unsigned long nslabs,
-		unsigned int nareas, u64 phys_limit, gfp_t gfp)
+		unsigned int nareas, u64 phys_limit, unsigned long attrs,
+		gfp_t gfp)
 {
 	struct io_tlb_pool *pool;
 	unsigned int slot_order;

@@ -704,9 +723,10 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 	if (!pool)
 		goto error;
 	pool->areas = (void *)pool + sizeof(*pool);
+	pool->unencrypted = !!(attrs & DMA_ATTR_CC_SHARED);
 
 	tlb_size = nslabs << IO_TLB_SHIFT;
-	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
+	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp))) {
 		if (nslabs <= minslabs)
 			goto error_tlb;
 		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);

@@ -724,7 +744,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 	return pool;
 
 error_slots:
-	swiotlb_free_tlb(page_address(tlb), tlb_size);
+	swiotlb_free_tlb(page_address(tlb), tlb_size,
+			 !!(attrs & DMA_ATTR_CC_SHARED));
 error_tlb:
 	kfree(pool);
 error:

@@ -742,7 +763,9 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
 	struct io_tlb_pool *pool;
 
 	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
-				  default_nareas, mem->phys_limit, GFP_KERNEL);
+				  default_nareas, mem->phys_limit,
+				  mem->unencrypted ? DMA_ATTR_CC_SHARED : 0,
+				  GFP_KERNEL);
 	if (!pool) {
 		pr_warn_ratelimited("Failed to allocate new pool");
 		return;

@@ -762,7 +785,7 @@ static void swiotlb_dyn_free(struct rcu_head *rcu)
 	size_t tlb_size = pool->end - pool->start;
 
 	free_pages((unsigned long)pool->slots, get_order(slots_size));
-	swiotlb_free_tlb(pool->vaddr, tlb_size);
+	swiotlb_free_tlb(pool->vaddr, tlb_size, pool->unencrypted);
 	kfree(pool);
 }
 

@@ -1232,6 +1255,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
 	nslabs = nr_slots(alloc_size);
 	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
 	pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
+				  mem->unencrypted ? DMA_ATTR_CC_SHARED : 0,
 				  GFP_NOWAIT);
 	if (!pool)
 		return -1;

@@ -1394,6 +1418,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 		enum dma_data_direction dir, unsigned long attrs)
 {
 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+	bool require_decrypted = false;
 	unsigned int offset;
 	struct io_tlb_pool *pool;
 	unsigned int i;

@@ -1411,6 +1436,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
 		pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
 
+	/*
+	 * if we are trying to swiotlb map a decrypted paddr or the paddr is encrypted
+	 * but the device is forcing decryption, use decrypted io_tlb_mem
+	 */
+	if ((attrs & DMA_ATTR_CC_SHARED) || force_dma_unencrypted(dev))
+		require_decrypted = true;
+
+	if (require_decrypted != mem->unencrypted)
+		return (phys_addr_t)DMA_MAPPING_ERROR;
+
 	/*
 	 * The default swiotlb memory pool is allocated with PAGE_SIZE
 	 * alignment. If a mapping is requested with larger alignment,

@@ -1608,8 +1643,14 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
 	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
 		return DMA_MAPPING_ERROR;
 
-	/* Ensure that the address returned is DMA'ble */
-	dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+	/*
+	 * Use the allocated io_tlb_mem encryption type to determine dma addr.
+	 */
+	if (dev->dma_io_tlb_mem->unencrypted)
+		dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+	else
+		dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
+
 	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
 		__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
 			attrs | DMA_ATTR_SKIP_CPU_SYNC,

@@ -1773,7 +1814,8 @@ static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
 
 #ifdef CONFIG_DMA_RESTRICTED_POOL
 
-struct page *swiotlb_alloc(struct device *dev, size_t size)
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+		unsigned long attrs)
 {
 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
 	struct io_tlb_pool *pool;

@@ -1784,6 +1826,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
 	if (!mem)
 		return NULL;
 
+	if (mem->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+		return NULL;
+
 	align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
 	index = swiotlb_find_slots(dev, 0, size, align, &pool);
 	if (index == -1)

@@ -1853,9 +1898,18 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
 			kfree(mem);
 			return -ENOMEM;
 		}
+		/*

kernel/dma/swiotlb.c +0 -2

-		set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
-				     rmem->size >> PAGE_SHIFT);
 		swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
 					 false, nareas);
 		mem->force_bounce = true;

Signature · 1 line

2.43.0

Quoted 6 lines

kernel/dma/swiotlb.c +6 -0

+		/*
+		 * if platform supports memory encryption,
+		 * restricted mem pool is decrypted by default
+		 */
+		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+			mem->unencrypted = true;

kernel/dma/swiotlb.c static int rmem_swiotlb_device_init(struct reserved_mem *rmem, · line 1911

Quoted 5 lines

kernel/dma/swiotlb.c +5 -0

+			set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+					     rmem->size >> PAGE_SHIFT);
+		} else {
+			mem->unencrypted = false;
+		}

This breaks pKVM as it doesn’t set CC_ATTR_MEM_ENCRYPT, so all virtio traffic now fails.

Also, by design, some drivers are clueless about bouncing, so

I believe that the pool should have a way to control it’s property (encrypted or decrypted) and that takes priority over whatever attributes comes from allocation.

And that brings us to the same point whether it’s better to return the memory along with it’s state or we pass the requested state. I think for other cases it’s fine for the device/DMA-API to dictate the attrs, but not in restricted-dma case, the firmware just knows better.

Quoted 4 lines

And that brings us to the same point whether it’s better to return the memory along with it’s state or we pass the requested state. I think for other cases it’s fine for the device/DMA-API to dictate the attrs, but not in restricted-dma case, the firmware just knows better.

The memory type must be returned back at some level so downstream things can do the right transformation of the phys_addr_t.

One of the aspirational CC things that should work is a T=1 device tries to DMA from a decrypted page, finds the address is above the dma limit of the device, so it bounces it with SWIOTLB to an encrypted low address page and then the DMA API internal flow switiches from working with decrypted to encrypted phys_addr_t.

If we can make that work then maybe the flows are designed correctly.

Quoted 6 lines · Mostafa Saleh

Quoted 6 lines

kernel/dma/swiotlb.c +6 -0

+		/*
+		 * if platform supports memory encryption,
+		 * restricted mem pool is decrypted by default
+		 */
+		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+			mem->unencrypted = true;

Quoted 8 lines

kernel/dma/swiotlb.c static int rmem_swiotlb_device_init(struct reserved_mem *rmem, · line 1911

Quoted 5 lines

kernel/dma/swiotlb.c +5 -0

+			set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+					     rmem->size >> PAGE_SHIFT);
+		} else {
+			mem->unencrypted = false;
+		}

This breaks pKVM as it doesn’t set CC_ATTR_MEM_ENCRYPT, so all virtio traffic now fails.

How will pKVM signal what kind of memory the DMA needs then?

Does it use set_memory_decrypted()? How can it use set_memory_decrypted() without offering CC_ATTR_MEM_ENCRYPT ?

Quoted 2 lines

Also, by design, some drivers are clueless about bouncing, so

Oh? What does this mean? We take quite a dim view of drivers mis-using the DMA API..

Quoted 4 lines

I believe that the pool should have a way to control it’s property (encrypted or decrypted) and that takes priority over whatever attributes comes from allocation.

We should get here because dma_capable() fails, and then swiotlb needs to return something that makes dma_capable() succeed. Yes, it should return details about the thing it decided, but it shouldn't have been pre-created with some idea how to make dma_capable() work.

If dma_capable() can fail, then swiotlb should know exactly what to do to fix it.

If pkvm wants to use the hacky scheme where you force a swiotlb pool configuration during arch init with force swiotlb that's a somewhat different flow and, sure the forced pool should force do whatever it is forced to.

But lets try to keep them seperated in the discussion..

Quoted 5 lines

And that brings us to the same point whether it’s better to return the memory along with it’s state or we pass the requested state. I think for other cases it’s fine for the device/DMA-API to dictate the attrs, but not in restricted-dma case, the firmware just knows better.

The memory type must be returned back at some level so downstream things can do the right transformation of the phys_addr_t.

One of the aspirational CC things that should work is a T=1 device tries to DMA from a decrypted page, finds the address is above the dma limit of the device, so it bounces it with SWIOTLB to an encrypted low address page and then the DMA API internal flow switiches from working with decrypted to encrypted phys_addr_t.

If we can make that work then maybe the flows are designed correctly.

Jason

Quoted 4 lines

How will pKVM signal what kind of memory the DMA needs then?

Does it use set_memory_decrypted()? How can it use set_memory_decrypted() without offering CC_ATTR_MEM_ENCRYPT ?

pKVM (hypervisor) doesn’t signal anything.
The VMM when running protected guests will use restricted dma-pools for emulated vritio devices in the guest, which gets decrypted by the guest kernel and hence shared with the host kernel, and then traffic is bounced via the pool.

It’s also worth noting that bouncing here isn't just about visibility. Because memory sharing operates at page granularity, bouncing sub-page allocations through the restricted pool prevents adjacent, sensitive guest data from being exposed to the untrusted host.

Quoted 8 lines

Quoted 3 lines

I believe that the pool should have a way to control it’s property (encrypted or decrypted) and that takes priority over whatever attributes comes from allocation.

We should get here because dma_capable() fails, and then swiotlb needs to return something that makes dma_capable() succeed. Yes, it should return details about the thing it decided, but it shouldn't have been pre-created with some idea how to make dma_capable() work.

That sounds neat, but at the end we have force_dma_unencrypted() in dma_capable() which is just hardcoded to true/false by the platform.

How is that different from having the state static by the pool?

Quoted 2 lines

If dma_capable() can fail, then swiotlb should know exactly what to do to fix it.

dma_capable() returns a bool, I don’t think it can know what exactly went wrong (based on address, size, attrs, dev...)

While we can debate the aesthetics of the setup , this is the exisitng behaviour for Linux, which existed for years and pKVM relies on and is used extensively.
And, this patch alters that long-standing logic and introduces a functional regression.

We can address this by either adjusting this patch or by changing pKVM guests to be more aligned with other CCA guests which is something I have been wondering about if it would help reduce bouncing.

Quoted 1 line

If we can make that work then maybe the flows are designed correctly.

Mmm, I am not sure I understand this one, shouldn’t the device also be notified about the switch in memory state, if it expects to read/write decrypted memory, how would that work if the kernel changes it to an encrypted one?

Quoted 10 lines

Quoted 4 lines

How will pKVM signal what kind of memory the DMA needs then?

Does it use set_memory_decrypted()? How can it use set_memory_decrypted() without offering CC_ATTR_MEM_ENCRYPT ?

pKVM (hypervisor) doesn’t signal anything.
The VMM when running protected guests will use restricted dma-pools for emulated vritio devices in the guest, which gets decrypted by the guest kernel and hence shared with the host kernel, and then traffic is bounced via the pool.

That really does sound like CC and set_memory_decrypted() to me..

Quoted 4 lines

It’s also worth noting that bouncing here isn't just about visibility. Because memory sharing operates at page granularity, bouncing sub-page allocations through the restricted pool prevents adjacent, sensitive guest data from being exposed to the untrusted host.

That's a somewhat different problem, we have the dev->trusted stuff that is supposed to deal with this kind of security. We need it for IOMMU based systems too, eg hot plug thunderbolt should have it.

Then CC issue is more that the DMA API can't decrypt random passed in memory because doing so often requires changing the PTEs pointing at the page so it would break everything if done transparently.

Quoted 11 lines

Quoted 8 lines

Quoted 3 lines

I believe that the pool should have a way to control it’s property (encrypted or decrypted) and that takes priority over whatever attributes comes from allocation.

We should get here because dma_capable() fails, and then swiotlb needs to return something that makes dma_capable() succeed. Yes, it should return details about the thing it decided, but it shouldn't have been pre-created with some idea how to make dma_capable() work.

That sounds neat, but at the end we have force_dma_unencrypted() in dma_capable() which is just hardcoded to true/false by the platform.

For now, the next step is it becomes per-device and dynamic during the device lifecycle.

Quoted 1 line

How is that different from having the state static by the pool?

statically attached pools to the device are not so flexible when devices have dynamically changing capabilities..

Quoted 6 lines

Quoted 3 lines

If dma_capable() can fail, then swiotlb should know exactly what to do to fix it.

dma_capable() returns a bool, I don’t think it can know what exactly went wrong (based on address, size, attrs, dev...)

Yes, but I think the design is swiotlb is supposed to re-inspect what is going on against the limits dma_capable checks and then select the correct remedy..

Quoted 6 lines

While we can debate the aesthetics of the setup , this is the exisitng behaviour for Linux, which existed for years and pKVM relies on and is used extensively.
And, this patch alters that long-standing logic and introduces a functional regression.

Yeah, Aneesh needs to do something here, I'm pointing out it is entirely seperate thing from the CC path we are working on which is decoupling CC from reylying on force swiotlb.

Quoted 5 lines

We can address this by either adjusting this patch or by changing pKVM guests to be more aligned with other CCA guests which is something I have been wondering about if it would help reduce bouncing.

Every time I look at pkvm I think it is just ARM CCA with a different design and no access to the unique HW features..

Quoted 6 lines

Quoted 1 line

If we can make that work then maybe the flows are designed correctly.

Mmm, I am not sure I understand this one, shouldn’t the device also be notified about the switch in memory state, if it expects to read/write decrypted memory, how would that work if the kernel changes it to an encrypted one?

Nothing on the device changes. In a CC world we put the device in a T=0 or T=1 state before the driver loads and the expectation from the DMA API is that the device will only use that T=x DMA type during operation.

A T=1 state device can access all of memory, private or shared. Any information the platform may need is encoded in the dma_addr_t or in the S1 IOPTEs.

So we never need to tell the device driver what kind of memory the DMA is targetting, and we NEVER expect a device in T=1 mode to have to issue a T=0 DMA to use the DMA API.

In a pkvm world it should be the same, the S2 table for the SMMU will control what the device can access, and if the SMMU points to a "private" or "shared" page is not something the device needs to know or care about.

Jason

Quoted 3 lines

That's a somewhat different problem, we have the dev->trusted stuff that is supposed to deal with this kind of security. We need it for IOMMU based systems too, eg hot plug thunderbolt should have it.

I see that it is used only for dma-iommu and for PCI devices. However, I think that should be a problem with other CCA solutions with emulated devices as they are untrusted. As I'd expect they would have virtio devices.

Quoted 6 lines

Quoted 5 lines

While we can debate the aesthetics of the setup , this is the exisitng behaviour for Linux, which existed for years and pKVM relies on and is used extensively.
And, this patch alters that long-standing logic and introduces a functional regression.

Quoted 3 lines

Yeah, Aneesh needs to do something here, I'm pointing out it is entirely seperate thing from the CC path we are working on which is decoupling CC from reylying on force swiotlb.

I am looking into converting pKVM to use the CC stuff, I replied with a patch to Aneesh in this thread. However, I need to do more testing and make sure there are not any unwanted consequences.

Quoted 4 lines

In a pkvm world it should be the same, the S2 table for the SMMU will control what the device can access, and if the SMMU points to a "private" or "shared" page is not something the device needs to know or care about.

I see that's because dma-iommu chooses the attrs for iommu_map().

In pKVM, dma_addr_t and IOPTE are the same for private and shared, so nothing differs in that case.

We don’t expect pass-through devices to interact with shared memory (T=0) at the moment.
However, I can see use cases for that, where the host and the guest collaborate with device passthrough and require zero copy.

One other interesting case for device-passthrough is non-coherent devices which then require private pools for bouncing.

Quoted 8 lines · Mostafa Saleh

Quoted 3 lines

That's a somewhat different problem, we have the dev->trusted stuff that is supposed to deal with this kind of security. We need it for IOMMU based systems too, eg hot plug thunderbolt should have it.

I see that it is used only for dma-iommu and for PCI devices. However, I think that should be a problem with other CCA solutions with emulated devices as they are untrusted. As I'd expect they would have virtio devices.

Yes, any security solution with an out of TCB device should be using either memory encryption so the kernel already bounces or this trusted stuff and a force strict dma-iommu so the dma layer is careful.

This is more policy from userspace what devices they want in or out of their TCB. Like you make accept the device into T=1 but then still want to keep it out of your TCB with the vIOMMU, I can see good arguments for something like that.

Quoted 6 lines

Quoted 6 lines

Quoted 5 lines

While we can debate the aesthetics of the setup , this is the exisitng behaviour for Linux, which existed for years and pKVM relies on and is used extensively.
And, this patch alters that long-standing logic and introduces a functional regression.

Quoted 7 lines

Quoted 3 lines

Yeah, Aneesh needs to do something here, I'm pointing out it is entirely seperate thing from the CC path we are working on which is decoupling CC from reylying on force swiotlb.

I am looking into converting pKVM to use the CC stuff, I replied with a patch to Aneesh in this thread. However, I need to do more testing and make sure there are not any unwanted consequences.

Yeah, it is a nice patch and I think it will help reduce the complexity if it aligns to CCA type stuff.

Quoted 6 lines

Quoted 4 lines

In a pkvm world it should be the same, the S2 table for the SMMU will control what the device can access, and if the SMMU points to a "private" or "shared" page is not something the device needs to know or care about.

I see that's because dma-iommu chooses the attrs for iommu_map().

Long term the DMA API path through the dma-iommu will pass the ATTR_CC_SHARED through to iommu_map so when the arch requires a different IOPTE it can construct it.

Quoted 2 lines

In pKVM, dma_addr_t and IOPTE are the same for private and shared, so nothing differs in that case.

Yes, so you don't have to worry.

Quoted 4 lines

We don’t expect pass-through devices to interact with shared memory (T=0) at the moment.
However, I can see use cases for that, where the host and the guest collaborate with device passthrough and require zero copy.

Once you add the CC patch it becomes immediately possible though because the user can allocate a CC shared DMA HEAP and feed that all over the place.

Quoted 2 lines

One other interesting case for device-passthrough is non-coherent devices which then require private pools for bouncing.

Why does shared/private matter for bouncing? Why do you need to bounce at all? Do cmo's not work in pkvm guests?

Jason

Quoted 5 lines

Quoted 2 lines

One other interesting case for device-passthrough is non-coherent devices which then require private pools for bouncing.

Why does shared/private matter for bouncing? Why do you need to bounce at all? Do cmo's not work in pkvm guests?

At the moment, in iommu_dma_map_phys(), if a non coherent device tries to map an unaligned address or size it will be bounced.

In pKVM, dma-iommu is used for assigned devices which operate on private memory, so bouncing that through the SWIOTLB would leak information from the guest as the SWIOTLB is decrypted.

Quoted 422 lines · Aneesh Kumar K.V (Arm)

Teach swiotlb to distinguish between encrypted and decrypted bounce buffer pools, and make allocation and mapping paths select a pool whose state matches the requested DMA attributes.

Add a decrypted flag to io_tlb_mem, initialize it for the default and restricted pools, and propagate DMA_ATTR_CC_SHARED into swiotlb pool allocation. Reject swiotlb alloc/map requests when the selected pool does not match the required encrypted/decrypted state.

Also return DMA addresses with the matching phys_to_dma_{encrypted, unencrypted} helper so the DMA address encoding stays consistent with the chosen pool.

Signed-off-by Aneesh Kumar K.V (Arm) <[email protected]>

---
 include/linux/dma-direct.h |  10 ++++
 include/linux/swiotlb.h    |   8 ++-
 kernel/dma/direct.c        |  14 +++--
 kernel/dma/swiotlb.c       | 108 +++++++++++++++++++++++++++----------
 4 files changed, 107 insertions(+), 33 deletions(-)

include/linux/dma-direct.h +10 -0

index c249912456f9..94fad4e7c11e 100644
@@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map)
 #ifndef phys_to_dma_unencrypted
 #define phys_to_dma_unencrypted		phys_to_dma
 #endif
+
+#ifndef phys_to_dma_encrypted
+#define phys_to_dma_encrypted		phys_to_dma
+#endif
 #else
 static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
 {

@@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
 {
 	return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
 }
+
+static inline dma_addr_t phys_to_dma_encrypted(struct device *dev,
+		phys_addr_t paddr)
+{
+	return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
 /*
  * If memory encryption is supported, phys_to_dma will set the memory encryption
  * bit in the DMA address, and dma_to_phys will clear it.

include/linux/swiotlb.h +6 -2

index 3dae0f592063..b3fa3c6e0169 100644
@@ -81,6 +81,7 @@ struct io_tlb_pool {
 	struct list_head node;
 	struct rcu_head rcu;
 	bool transient;
+	bool unencrypted;
 #endif
 };
 

@@ -111,6 +112,7 @@ struct io_tlb_mem {
 	struct dentry *debugfs;
 	bool force_bounce;
 	bool for_alloc;
+	bool unencrypted;
 #ifdef CONFIG_SWIOTLB_DYNAMIC
 	bool can_grow;
 	u64 phys_limit;

@@ -282,7 +284,8 @@ static inline void swiotlb_sync_single_for_cpu(struct device *dev,
 extern void swiotlb_print_info(void);
 
 #ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size);
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+		unsigned long attrs);
 bool swiotlb_free(struct device *dev, struct page *page, size_t size);
 
 static inline bool is_swiotlb_for_alloc(struct device *dev)

@@ -290,7 +293,8 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
 	return dev->dma_io_tlb_mem->for_alloc;
 }
 #else
-static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
+static inline struct page *swiotlb_alloc(struct device *dev, size_t size,
+		unsigned long attrs)
 {
 	return NULL;
 }

kernel/dma/direct.c +10 -4

index dc2907439b3d..97ae4fa10521 100644
@@ -104,9 +104,10 @@ static void __dma_direct_free_pages(struct device *dev, struct page *page,
 	dma_free_contiguous(dev, page, size);
 }
 
-static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
+static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,
+		unsigned long attrs)
 {
-	struct page *page = swiotlb_alloc(dev, size);
+	struct page *page = swiotlb_alloc(dev, size, attrs);
 
 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
 		swiotlb_free(dev, page, size);

@@ -266,8 +267,12 @@ void *dma_direct_alloc(struct device *dev, size_t size,
 						  gfp, attrs);
 
 	if (is_swiotlb_for_alloc(dev)) {
-		page = dma_direct_alloc_swiotlb(dev, size);
+		page = dma_direct_alloc_swiotlb(dev, size, attrs);
 		if (page) {
+			/*
+			 * swiotlb allocations comes from pool already marked
+			 * decrypted
+			 */
 			mark_mem_decrypt = false;
 			goto setup_page;
 		}

@@ -374,6 +379,7 @@ void dma_direct_free(struct device *dev, size_t size,
 		return;
 
 	if (swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr)))
+		/* Swiotlb doesn't need a page attribute update on free */
 		mark_mem_encrypted = false;
 
 	if (is_vmalloc_addr(cpu_addr)) {

@@ -403,7 +409,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
 						  gfp, attrs);
 
 	if (is_swiotlb_for_alloc(dev)) {
-		page = dma_direct_alloc_swiotlb(dev, size);
+		page = dma_direct_alloc_swiotlb(dev, size, attrs);
 		if (!page)
 			return NULL;
 

kernel/dma/swiotlb.c +71 -25

index ab4eccbaa076..065663be282c 100644
@@ -259,10 +259,21 @@ void __init swiotlb_update_mem_attributes(void)
 	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
 	unsigned long bytes;
 
+	/*
+	 * if platform support memory encryption, swiotlb buffers are
+	 * decrypted by default.
+	 */
+	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+		io_tlb_default_mem.unencrypted = true;
+	else
+		io_tlb_default_mem.unencrypted = false;
+
 	if (!mem->nslabs || mem->late_alloc)
 		return;
 	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
-	set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
+
+	if (io_tlb_default_mem.unencrypted)
+		set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
 }
 
 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,

@@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
 	if (!mem->slots)
 		goto error_slots;
 
-	set_memory_decrypted((unsigned long)vstart,
-			     (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+	if (io_tlb_default_mem.unencrypted)
+		set_memory_decrypted((unsigned long)vstart,
+				     (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+
 	swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
 				 nareas);
 	add_mem_pool(&io_tlb_default_mem, mem);

@@ -539,7 +552,9 @@ void __init swiotlb_exit(void)
 	tbl_size = PAGE_ALIGN(mem->end - mem->start);
 	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
 
-	set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+	if (io_tlb_default_mem.unencrypted)
+		set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+
 	if (mem->late_alloc) {
 		area_order = get_order(array_size(sizeof(*mem->areas),
 			mem->nareas));

@@ -563,6 +578,7 @@ void __init swiotlb_exit(void)
  * @gfp:	GFP flags for the allocation.
  * @bytes:	Size of the buffer.
  * @phys_limit:	Maximum allowed physical address of the buffer.
+ * @unencrypted: true to allocate unencrypted memory, false for encrypted memory
  *
  * Allocate pages from the buddy allocator. If successful, make the allocated
  * pages decrypted that they can be used for DMA.

@@ -570,7 +586,8 @@ void __init swiotlb_exit(void)
  * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
  * if the allocated physical address was above @phys_limit.
  */
-static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
+static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes,
+		u64 phys_limit, bool unencrypted)
 {
 	unsigned int order = get_order(bytes);
 	struct page *page;

@@ -588,13 +605,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
 	}
 
 	vaddr = phys_to_virt(paddr);
-	if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
+	if (unencrypted && set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
 		goto error;
 	return page;
 
 error:
 	/* Intentional leak if pages cannot be encrypted again. */
-	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+	if (unencrypted && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 		__free_pages(page, order);
 	return NULL;
 }

@@ -604,30 +621,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
  * @dev:	Device for which a memory pool is allocated.
  * @bytes:	Size of the buffer.
  * @phys_limit:	Maximum allowed physical address of the buffer.
+ * @attrs:	DMA attributes for the allocation.
  * @gfp:	GFP flags for the allocation.
  *
  * Return: Allocated pages, or %NULL on allocation failure.
  */
 static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
-		u64 phys_limit, gfp_t gfp)
+		u64 phys_limit, unsigned long attrs, gfp_t gfp)
 {
 	struct page *page;
-	unsigned long attrs = 0;
 
 	/*
 	 * Allocate from the atomic pools if memory is encrypted and
 	 * the allocation is atomic, because decrypting may block.
 	 */
-	if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
+	if (!gfpflags_allow_blocking(gfp) && (attrs & DMA_ATTR_CC_SHARED)) {
 		void *vaddr;
 
 		if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
 			return NULL;
 
-		/* swiotlb considered decrypted by default */
-		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
-			attrs = DMA_ATTR_CC_SHARED;
-
 		return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
 					   attrs, dma_coherent_ok);
 	}

@@ -638,7 +651,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
 	else if (phys_limit <= DMA_BIT_MASK(32))
 		gfp |= __GFP_DMA32;
 
-	while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
+	while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit,
+					     !!(attrs & DMA_ATTR_CC_SHARED)))) {
 		if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
 		    phys_limit < DMA_BIT_MASK(64) &&
 		    !(gfp & (__GFP_DMA32 | __GFP_DMA)))

@@ -657,15 +671,18 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
  * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
  * @vaddr:	Virtual address of the buffer.
  * @bytes:	Size of the buffer.
+ * @unencrypted: true if @vaddr was allocated decrypted and must be
+ *	re-encrypted before being freed
  */
-static void swiotlb_free_tlb(void *vaddr, size_t bytes)
+static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool unencrypted)
 {
 	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
 	    dma_free_from_pool(NULL, vaddr, bytes))
 		return;
 
 	/* Intentional leak if pages cannot be encrypted again. */
-	if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+	if (!unencrypted ||
+	    !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
 		__free_pages(virt_to_page(vaddr), get_order(bytes));
 }
 

@@ -676,6 +693,7 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
  * @nslabs:	Desired (maximum) number of slabs.
  * @nareas:	Number of areas.
  * @phys_limit:	Maximum DMA buffer physical address.
+ * @attrs:	DMA attributes for the allocation.
  * @gfp:	GFP flags for the allocations.
  *
  * Allocate and initialize a new IO TLB memory pool. The actual number of

@@ -686,7 +704,8 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
  */
 static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 		unsigned long minslabs, unsigned long nslabs,
-		unsigned int nareas, u64 phys_limit, gfp_t gfp)
+		unsigned int nareas, u64 phys_limit, unsigned long attrs,
+		gfp_t gfp)
 {
 	struct io_tlb_pool *pool;
 	unsigned int slot_order;

@@ -704,9 +723,10 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 	if (!pool)
 		goto error;
 	pool->areas = (void *)pool + sizeof(*pool);
+	pool->unencrypted = !!(attrs & DMA_ATTR_CC_SHARED);
 
 	tlb_size = nslabs << IO_TLB_SHIFT;
-	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
+	while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp))) {
 		if (nslabs <= minslabs)
 			goto error_tlb;
 		nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);

@@ -724,7 +744,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
 	return pool;
 
 error_slots:
-	swiotlb_free_tlb(page_address(tlb), tlb_size);
+	swiotlb_free_tlb(page_address(tlb), tlb_size,
+			 !!(attrs & DMA_ATTR_CC_SHARED));
 error_tlb:
 	kfree(pool);
 error:

@@ -742,7 +763,9 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
 	struct io_tlb_pool *pool;
 
 	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
-				  default_nareas, mem->phys_limit, GFP_KERNEL);
+				  default_nareas, mem->phys_limit,
+				  mem->unencrypted ? DMA_ATTR_CC_SHARED : 0,
+				  GFP_KERNEL);
 	if (!pool) {
 		pr_warn_ratelimited("Failed to allocate new pool");
 		return;

@@ -762,7 +785,7 @@ static void swiotlb_dyn_free(struct rcu_head *rcu)
 	size_t tlb_size = pool->end - pool->start;
 
 	free_pages((unsigned long)pool->slots, get_order(slots_size));
-	swiotlb_free_tlb(pool->vaddr, tlb_size);
+	swiotlb_free_tlb(pool->vaddr, tlb_size, pool->unencrypted);
 	kfree(pool);
 }
 

@@ -1232,6 +1255,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
 	nslabs = nr_slots(alloc_size);
 	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
 	pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
+				  mem->unencrypted ? DMA_ATTR_CC_SHARED : 0,
 				  GFP_NOWAIT);
 	if (!pool)
 		return -1;

@@ -1394,6 +1418,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 		enum dma_data_direction dir, unsigned long attrs)
 {
 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
+	bool require_decrypted = false;
 	unsigned int offset;
 	struct io_tlb_pool *pool;
 	unsigned int i;

@@ -1411,6 +1436,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
 	if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
 		pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
 
+	/*
+	 * if we are trying to swiotlb map a decrypted paddr or the paddr is encrypted
+	 * but the device is forcing decryption, use decrypted io_tlb_mem
+	 */
+	if ((attrs & DMA_ATTR_CC_SHARED) || force_dma_unencrypted(dev))
+		require_decrypted = true;
+
+	if (require_decrypted != mem->unencrypted)
+		return (phys_addr_t)DMA_MAPPING_ERROR;
+
 	/*
 	 * The default swiotlb memory pool is allocated with PAGE_SIZE
 	 * alignment. If a mapping is requested with larger alignment,

@@ -1608,8 +1643,14 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
 	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
 		return DMA_MAPPING_ERROR;
 
-	/* Ensure that the address returned is DMA'ble */
-	dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+	/*
+	 * Use the allocated io_tlb_mem encryption type to determine dma addr.
+	 */
+	if (dev->dma_io_tlb_mem->unencrypted)
+		dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+	else
+		dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
+
 	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
 		__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
 			attrs | DMA_ATTR_SKIP_CPU_SYNC,

@@ -1773,7 +1814,8 @@ static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
 
 #ifdef CONFIG_DMA_RESTRICTED_POOL
 
-struct page *swiotlb_alloc(struct device *dev, size_t size)
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+		unsigned long attrs)
 {
 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
 	struct io_tlb_pool *pool;

@@ -1784,6 +1826,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
 	if (!mem)
 		return NULL;
 
+	if (mem->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+		return NULL;
+
 	align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
 	index = swiotlb_find_slots(dev, 0, size, align, &pool);
 	if (index == -1)

@@ -1853,9 +1898,18 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
 			kfree(mem);
 			return -ENOMEM;
 		}
+		/*

kernel/dma/swiotlb.c static int rmem_swiotlb_device_init(struct reserved_mem *rmem, · line 1911

Quoted 10 lines

kernel/dma/swiotlb.c +10 -0

+		 * if platform supports memory encryption,
+		 * restricted mem pool is decrypted by default
+		 */
+		if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+			mem->unencrypted = true;
+			set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+					     rmem->size >> PAGE_SHIFT);
+		} else {
+			mem->unencrypted = false;
+		}

This breaks pKVM as it doesn’t set CC_ATTR_MEM_ENCRYPT, so all virtio traffic now fails.