Skip to contentSkip to conversation

iommu: Optimize IOMMU UnMap

PATCH Superseded Ashish Mhetre 2 participants 6 messages Aug 1, 2024 – Aug 5, 2024

Subsystems (3)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/2 iommu: Optimize IOMMU UnMap

The current __arm_lpae_unmap() function calls dma_sync() on individual PTEs after clearing them. Overall unmap performance can be improved by around 25% for large buffer sizes by combining the syncs for adjacent leaf entries.
Optimize the unmap time by clearing all the leaf entries and issuing a single dma_sync() for them.
Below is detailed analysis of average unmap latency(in us) with and without this optimization obtained by running dma_map_benchmark for different buffer sizes.

UnMap Latency(us)
Size Without With % gain with
optimiztion optimization optimization

4KB 3 3 0
8KB 4 3.8 5
16KB 6.1 5.4 11.48
32KB 10.2 8.5 16.67
64KB 18.5 14.9 19.46
128KB 35 27.5 21.43
256KB 67.5 52.2 22.67
512KB 127.9 97.2 24.00
1MB 248.6 187.4 24.62
2MB 65.5 65.5 0
4MB 119.2 119 0.17

---
Changes in V2:
- Updated the commit message to be imperative.
- Fixed ptep at incorrect index getting cleared for non-leaf entries.

---
 drivers/iommu/io-pgtable-arm.c | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)
drivers/iommu/io-pgtable-arm.c +21 -13
index f5d9fd1f45bf..32401948b980 100644
@@ -274,13 +274,15 @@ static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
 				   sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
 }
 
-static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)
+static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries)
 {
+	int i;
 
-	*ptep = 0;
+	for (i = 0; i < num_entries; i++)
+		ptep[i] = 0;
 
 	if (!cfg->coherent_walk)
-		__arm_lpae_sync_pte(ptep, 1, cfg);
+		__arm_lpae_sync_pte(ptep, num_entries, cfg);
 }
 
 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
@@ -635,9 +637,10 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 			       unsigned long iova, size_t size, size_t pgcount,
 			       int lvl, arm_lpae_iopte *ptep)
 {
+	bool gather_queued;
 	arm_lpae_iopte pte;
 	struct io_pgtable *iop = &data->iop;
-	int i = 0, num_entries, max_entries, unmap_idx_start;
+	int i = 0, j = 0, num_entries, max_entries, unmap_idx_start;
 
 	/* Something went horribly wrong and we ran out of page table */
 	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
@@ -652,28 +655,33 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 	/* If the size matches this level, we're in the right place */
 	if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
 		max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
+		gather_queued = iommu_iotlb_gather_queued(gather);
 		num_entries = min_t(int, pgcount, max_entries);
 
-		while (i < num_entries) {
-			pte = READ_ONCE(*ptep);
+		/* Find and handle non-leaf entries */
+		for (i = 0; i < num_entries; i++) {
+			pte = READ_ONCE(ptep[i]);
 			if (WARN_ON(!pte))
 				break;
 
-			__arm_lpae_clear_pte(ptep, &iop->cfg);
-
 			if (!iopte_leaf(pte, lvl, iop->fmt)) {
+				__arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);
+
 				/* Also flush any partial walks */
 				io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
 							  ARM_LPAE_GRANULE(data));
 				__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
-			} else if (!iommu_iotlb_gather_queued(gather)) {
-				io_pgtable_tlb_add_page(iop, gather, iova + i * size, size);
 			}
-
-			ptep++;
-			i++;
 		}
 
+		/* Clear the remaining entries */
+		if (i)
+			__arm_lpae_clear_pte(ptep, &iop->cfg, i);
+
+		if (!gather_queued)
+			for (j = 0; j < i; j++)
+				io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);
+
 		return i * size;
 	} else if (iopte_leaf(pte, lvl, iop->fmt)) {
 		/*
Signature · 1 line
2.25.1
#1 lore raw

PATCH v2 2/2 include: linux: Update gather only if it's not NULL

Gather can be NULL when unmap is called for freeing old table while mapping. If it's NULL then there is no need to add page for syncing the TLB.

---
 include/linux/iommu.h | 3 +++
 1 file changed, 3 insertions(+)
include/linux/iommu.h +3 -0
index 4d47f2c33311..2a28c1ef8517 100644
@@ -928,6 +928,9 @@ static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
 					       struct iommu_iotlb_gather *gather,
 					       unsigned long iova, size_t size)
 {
+	if (!gather)
+		return;
+
 	/*
 	 * If the new page is disjoint from the current range or is mapped at
 	 * a different granularity, then sync the TLB so that the gather
Signature · 1 line
2.25.1
reviewer
#2 lore raw

Re: [PATCH V2 2/2] include: linux: Update gather only if it's not NULL

Quoted 3 lines · Ashish Mhetre

Gather can be NULL when unmap is called for freeing old table while mapping. If it's NULL then there is no need to add page for syncing the TLB.

But that's only because __arm_lpae_unmap() is now choosing to over-invalidate the table entries for simplicity. I think it would make more sense to handle that at the callsite, e.g. "if (gather && !iommu_iotlb_gather_queued(gather))".

Also doesn't this mean that bisection is broken as-is since patch #1 on its own is going to blow up dereferencing NULL->pgsize when it gets here?

Thanks,
Robin.

Quoted 19 lines
---
  include/linux/iommu.h | 3 +++
  1 file changed, 3 insertions(+)
include/linux/iommu.h +3 -0
index 4d47f2c33311..2a28c1ef8517 100644
@@ -928,6 +928,9 @@ static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
  					       struct iommu_iotlb_gather *gather,
  					       unsigned long iova, size_t size)
  {
+	if (!gather)
+		return;
+
  	/*
  	 * If the new page is disjoint from the current range or is mapped at
  	 * a different granularity, then sync the TLB so that the gather
#3 lore raw

Re: [PATCH V2 2/2] include: linux: Update gather only if it's not NULL

Quoted 13 lines · Robin Murphy

External email: Use caution opening links or attachments

Quoted 3 lines · Ashish Mhetre

Gather can be NULL when unmap is called for freeing old table while mapping. If it's NULL then there is no need to add page for syncing the TLB.

But that's only because __arm_lpae_unmap() is now choosing to over-invalidate the table entries for simplicity. I think it would make more sense to handle that at the callsite, e.g. "if (gather && !iommu_iotlb_gather_queued(gather))".

Quoted 6 lines

Also doesn't this mean that bisection is broken as-is since patch #1 on its own is going to blow up dereferencing NULL->pgsize when it gets here?

Thanks,
Robin.

Yes, gather can be NULL because of patch #1 only. I will fix it at callsite only with a single patch rather than splitting.

Quoted 20 lines
Quoted 20 lines

---
  include/linux/iommu.h | 3 +++

  1 file changed, 3 insertions(+)
include/linux/iommu.h +3 -0
index 4d47f2c33311..2a28c1ef8517 100644
@@ -928,6 +928,9 @@ static inline void 
 iommu_iotlb_gather_add_page(struct iommu_domain *domain,
                                              struct iommu_iotlb_gather *gather,
                                              unsigned long iova, size_t size)
   {
+     if (!gather)
+             return;
+
       /*
        * If the new page is disjoint from the current range or is mapped at

       * a different granularity, then sync the TLB so that the gather

reviewer
#4 lore raw
In reply to Ashish Mhetre
Quoted 40 lines · Ashish Mhetre

The current __arm_lpae_unmap() function calls dma_sync() on individual PTEs after clearing them. Overall unmap performance can be improved by around 25% for large buffer sizes by combining the syncs for adjacent leaf entries.
Optimize the unmap time by clearing all the leaf entries and issuing a single dma_sync() for them.
Below is detailed analysis of average unmap latency(in us) with and without this optimization obtained by running dma_map_benchmark for different buffer sizes.

UnMap Latency(us)
Size Without With % gain with
optimiztion optimization optimization

4KB 3 3 0
8KB 4 3.8 5
16KB 6.1 5.4 11.48
32KB 10.2 8.5 16.67
64KB 18.5 14.9 19.46
128KB 35 27.5 21.43
256KB 67.5 52.2 22.67
512KB 127.9 97.2 24.00
1MB 248.6 187.4 24.62
2MB 65.5 65.5 0
4MB 119.2 119 0.17

---
Changes in V2:
- Updated the commit message to be imperative.
- Fixed ptep at incorrect index getting cleared for non-leaf entries.

---
  drivers/iommu/io-pgtable-arm.c | 34 +++++++++++++++++++++-------------
  1 file changed, 21 insertions(+), 13 deletions(-)
drivers/iommu/io-pgtable-arm.c +0 -0
index f5d9fd1f45bf..32401948b980 100644

drivers/iommu/io-pgtable-arm.c static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, · line 279

Quoted 7 lines
drivers/iommu/io-pgtable-arm.c +2 -1
@@ -274,13 +274,15 @@ static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
  				   sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
  }
 
-static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg)
+static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg, int num_entries)
  {
+	int i;

You can make this a nice tidy loop-local declaration now.

Quoted 11 lines
drivers/iommu/io-pgtable-arm.c +3 -2
-	*ptep = 0;
+	for (i = 0; i < num_entries; i++)
+		ptep[i] = 0;
 
  	if (!cfg->coherent_walk)
-		__arm_lpae_sync_pte(ptep, 1, cfg);
+		__arm_lpae_sync_pte(ptep, num_entries, cfg);
  }
 
  static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,

drivers/iommu/io-pgtable-arm.c static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, · line 643

Quoted 8 lines
drivers/iommu/io-pgtable-arm.c +2 -1
@@ -635,9 +637,10 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
  			       unsigned long iova, size_t size, size_t pgcount,
  			       int lvl, arm_lpae_iopte *ptep)
  {
+	bool gather_queued;
  	arm_lpae_iopte pte;
  	struct io_pgtable *iop = &data->iop;
-	int i = 0, num_entries, max_entries, unmap_idx_start;
+	int i = 0, j = 0, num_entries, max_entries, unmap_idx_start;

Similarly there's no need to initialise j here, but I'd make it loop-local anyway.

drivers/iommu/io-pgtable-arm.c static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, · line 658

Quoted 7 lines
drivers/iommu/io-pgtable-arm.c +1 -0
  	/* Something went horribly wrong and we ran out of page table */
  	if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
@@ -652,28 +655,33 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
  	/* If the size matches this level, we're in the right place */
  	if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
  		max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
+		gather_queued = iommu_iotlb_gather_queued(gather);

This is used exactly once, do we really need to introduce the variable?

Quoted 19 lines
drivers/iommu/io-pgtable-arm.c +5 -4
  		num_entries = min_t(int, pgcount, max_entries);
 
-		while (i < num_entries) {
-			pte = READ_ONCE(*ptep);
+		/* Find and handle non-leaf entries */
+		for (i = 0; i < num_entries; i++) {
+			pte = READ_ONCE(ptep[i]);
  			if (WARN_ON(!pte))
  				break;
 
-			__arm_lpae_clear_pte(ptep, &iop->cfg);
-
  			if (!iopte_leaf(pte, lvl, iop->fmt)) {
+				__arm_lpae_clear_pte(&ptep[i], &iop->cfg, 1);
+
  				/* Also flush any partial walks */
  				io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
  							  ARM_LPAE_GRANULE(data));
  				__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));

drivers/iommu/io-pgtable-arm.c static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, · line 678

Quoted 10 lines
drivers/iommu/io-pgtable-arm.c +2 -5
-			} else if (!iommu_iotlb_gather_queued(gather)) {
-				io_pgtable_tlb_add_page(iop, gather, iova + i * size, size);
  			}
-
-			ptep++;
-			i++;
  		}
 
+		/* Clear the remaining entries */
+		if (i)

It seems a little non-obvious to optimise for one specific corner of unexpected failures here. I'd hope a zero-sized dma_sync wouldn't blow up in that case, but if you want to be safe then I'd cover it by tweaking the condition in __arm_lpae_clear_pte() to "if (!cfg->coherent_walk && num_entries)".

Thanks,
Robin.

Quoted 9 lines
drivers/iommu/io-pgtable-arm.c +6 -0
+			__arm_lpae_clear_pte(ptep, &iop->cfg, i);
+
+		if (!gather_queued)
+			for (j = 0; j < i; j++)
+				io_pgtable_tlb_add_page(iop, gather, iova + j * size, size);
+
  		return i * size;
  	} else if (iopte_leaf(pte, lvl, iop->fmt)) {
  		/*
#5 lore raw
Quoted 48 lines · Robin Murphy

External email: Use caution opening links or attachments

Quoted 44 lines · Ashish Mhetre

The current __arm_lpae_unmap() function calls dma_sync() on individual PTEs after clearing them. Overall unmap performance can be improved by around 25% for large buffer sizes by combining the syncs for adjacent leaf entries.
Optimize the unmap time by clearing all the leaf entries and issuing a single dma_sync() for them.
Below is detailed analysis of average unmap latency(in us) with and without this optimization obtained by running dma_map_benchmark for different buffer sizes.

              UnMap Latency(us)
Size  Without         With            % gain with optimiztion     optimization    optimization

4KB   3               3               0
8KB   4               3.8             5
16KB  6.1             5.4             11.48
32KB  10.2            8.5             16.67
64KB  18.5            14.9            19.46
128KB 35              27.5            21.43
256KB 67.5            52.2            22.67
512KB 127.9           97.2            24.00
1MB   248.6           187.4           24.62
2MB   65.5            65.5            0
4MB   119.2           119             0.17

---
Changes in V2:
- Updated the commit message to be imperative.
- Fixed ptep at incorrect index getting cleared for non-leaf entries.
---
  drivers/iommu/io-pgtable-arm.c | 34 +++++++++++++++++++++-------------

  1 file changed, 21 insertions(+), 13 deletions(-)
drivers/iommu/io-pgtable-arm.c +0 -0
index f5d9fd1f45bf..32401948b980 100644
@@ -274,13 +274,15 @@ static void __arm_lpae_sync_pte(arm_lpae_iopte 
 *ptep, int num_entries,
                                  sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
   }
 
Quoted 9 lines

drivers/iommu/io-pgtable-arm.c static void __arm_lpae_sync_pte(arm_lpae_iopte · line 282

Quoted 6 lines
drivers/iommu/io-pgtable-arm.c +2 -1
-static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct 
 io_pgtable_cfg *cfg)
+static void __arm_lpae_clear_pte(arm_lpae_iopte *ptep, struct 
 io_pgtable_cfg *cfg, int num_entries)
   {
+     int i;

You can make this a nice tidy loop-local declaration now.

Sure, will update this in new version.

Quoted 9 lines
Quoted 9 lines
drivers/iommu/io-pgtable-arm.c +3 -2
-     *ptep = 0;
+     for (i = 0; i < num_entries; i++)
+             ptep[i] = 0;
 
       if (!cfg->coherent_walk)
-             __arm_lpae_sync_pte(ptep, 1, cfg);
+             __arm_lpae_sync_pte(ptep, num_entries, cfg);
   }
Quoted 15 lines

drivers/iommu/io-pgtable-arm.c static size_t __arm_lpae_unmap(struct · line 644

Quoted 11 lines

  static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,

drivers/iommu/io-pgtable-arm.c +2 -1
@@ -635,9 +637,10 @@ static size_t __arm_lpae_unmap(struct 
 arm_lpae_io_pgtable *data,
                              unsigned long iova, size_t size, size_t pgcount,
                              int lvl, arm_lpae_iopte *ptep)
   {
+     bool gather_queued;
       arm_lpae_iopte pte;
       struct io_pgtable *iop = &data->iop;
-     int i = 0, num_entries, max_entries, unmap_idx_start;
+     int i = 0, j = 0, num_entries, max_entries, unmap_idx_start;

Similarly there's no need to initialise j here, but I'd make it loop-local anyway.

Ack, I will make j loop-local.

Quoted 11 lines

drivers/iommu/io-pgtable-arm.c static size_t __arm_lpae_unmap(struct · line 659

Quoted 8 lines
drivers/iommu/io-pgtable-arm.c +1 -0
       /* Something went horribly wrong and we ran out of page table */
       if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS))
@@ -652,28 +655,33 @@ static size_t __arm_lpae_unmap(struct 
 arm_lpae_io_pgtable *data,
       /* If the size matches this level, we're in the right place */
       if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
               max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
+             gather_queued = iommu_iotlb_gather_queued(gather);

This is used exactly once, do we really need to introduce the variable?

Ack, I will remove the unnecessary variable.

Quoted 38 lines
Quoted 21 lines
drivers/iommu/io-pgtable-arm.c +5 -5
               num_entries = min_t(int, pgcount, max_entries);
 
-             while (i < num_entries) {
-                     pte = READ_ONCE(*ptep);
+             /* Find and handle non-leaf entries */
+             for (i = 0; i < num_entries; i++) {
+                     pte = READ_ONCE(ptep[i]);
                       if (WARN_ON(!pte))
                               break;
 
-                     __arm_lpae_clear_pte(ptep, &iop->cfg);
-
                       if (!iopte_leaf(pte, lvl, iop->fmt)) {
+                             __arm_lpae_clear_pte(&ptep[i], 
 &iop->cfg, 1);
+
                               /* Also flush any partial walks */
                               io_pgtable_tlb_flush_walk(iop, iova + i * size, size,
 ARM_LPAE_GRANULE(data));
                               __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
-                     } else if (!iommu_iotlb_gather_queued(gather)) {

drivers/iommu/io-pgtable-arm.c static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, · line 678

Quoted 10 lines
drivers/iommu/io-pgtable-arm.c +2 -4
-                             io_pgtable_tlb_add_page(iop, gather, 
 iova + i * size, size);
                       }
-
-                     ptep++;
-                     i++;
               }
 
+             /* Clear the remaining entries */
+             if (i)

It seems a little non-obvious to optimise for one specific corner of unexpected failures here. I'd hope a zero-sized dma_sync wouldn't blow up in that case, but if you want to be safe then I'd cover it by tweaking the condition in __arm_lpae_clear_pte() to "if (!cfg->coherent_walk && num_entries)".

Quoted 3 lines

Thanks,
Robin.

Yes, makes sense to cover this in __arm_lpae_clear_pte(). I will update this in V3.

Quoted 10 lines
Quoted 10 lines
drivers/iommu/io-pgtable-arm.c +5 -0
+ __arm_lpae_clear_pte(ptep, &iop->cfg, i);
+
+             if (!gather_queued)
+                     for (j = 0; j < i; j++)
+                             io_pgtable_tlb_add_page(iop, gather, 

iova + j * size, size);
+
              return i * size;
      } else if (iopte_leaf(pte, lvl, iop->fmt)) { /*