Skip to contentSkip to conversation

iommu: reduce spinlock contention on fast path

PATCH Superseded Cong Wang 4 participants 20 messages Nov 29, 2019 – Dec 18, 2019

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Cong Wang

author
#0 lore raw

v2 cover iommu: reduce spinlock contention on fast path

This patchset contains three small optimizations for the global spinlock contention in IOVA cache. Our memcache perf test shows this reduced its p999 latency down by 45% on AMD when IOMMU is enabled.

Cong Wang (3):
iommu: match the original algorithm
iommu: optimize iova_magazine_free_pfns()
iommu: avoid taking iova_rbtree_lock twice

---
 drivers/iommu/iova.c | 75 ++++++++++++++++++++++++++------------------
 1 file changed, 45 insertions(+), 30 deletions(-)
Signature · 6 lines
2.21.0

_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
author
#1 lore raw

PATCH v2 1/3 iommu: match the original algorithm

The IOVA cache algorithm implemented in IOMMU code does not exactly match the original algorithm described in the paper.

Particularly, it doesn't need to free the loaded empty magazine when trying to put it back to global depot. To make it work, we have to pre-allocate magazines in the depot and only recycle them when all of them are full.

Before this patch, rcache->depot[] contains either full or freed entries, after this patch, it contains either full or empty (but allocated) entries.

Cc Joerg Roedel <[email protected]>
---
 drivers/iommu/iova.c | 45 +++++++++++++++++++++++++++-----------------
 1 file changed, 28 insertions(+), 17 deletions(-)
drivers/iommu/iova.c +28 -17
index 41c605b0058f..cb473ddce4cf 100644
@@ -862,12 +862,16 @@ static void init_iova_rcaches(struct iova_domain *iovad)
 	struct iova_cpu_rcache *cpu_rcache;
 	struct iova_rcache *rcache;
 	unsigned int cpu;
-	int i;
+	int i, j;
 
 	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
 		rcache = &iovad->rcaches[i];
 		spin_lock_init(&rcache->lock);
 		rcache->depot_size = 0;
+		for (j = 0; j < MAX_GLOBAL_MAGS; ++j) {
+			rcache->depot[j] = iova_magazine_alloc(GFP_KERNEL);
+			WARN_ON(!rcache->depot[j]);
+		}
 		rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
 		if (WARN_ON(!rcache->cpu_rcaches))
 			continue;
@@ -900,24 +904,30 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
 
 	if (!iova_magazine_full(cpu_rcache->loaded)) {
 		can_insert = true;
-	} else if (!iova_magazine_full(cpu_rcache->prev)) {
+	} else if (iova_magazine_empty(cpu_rcache->prev)) {
 		swap(cpu_rcache->prev, cpu_rcache->loaded);
 		can_insert = true;
 	} else {
-		struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
+		spin_lock(&rcache->lock);
+		if (rcache->depot_size < MAX_GLOBAL_MAGS) {
+			swap(rcache->depot[rcache->depot_size], cpu_rcache->prev);
+			swap(cpu_rcache->prev, cpu_rcache->loaded);
+			rcache->depot_size++;
+			can_insert = true;
+		} else {
+			mag_to_free = cpu_rcache->loaded;
+		}
+		spin_unlock(&rcache->lock);
+
+		if (mag_to_free) {
+			struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
 
-		if (new_mag) {
-			spin_lock(&rcache->lock);
-			if (rcache->depot_size < MAX_GLOBAL_MAGS) {
-				rcache->depot[rcache->depot_size++] =
-						cpu_rcache->loaded;
+			if (new_mag) {
+				cpu_rcache->loaded = new_mag;
+				can_insert = true;
 			} else {
-				mag_to_free = cpu_rcache->loaded;
+				mag_to_free = NULL;
 			}
-			spin_unlock(&rcache->lock);
-
-			cpu_rcache->loaded = new_mag;
-			can_insert = true;
 		}
 	}
 
@@ -963,14 +973,15 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
 
 	if (!iova_magazine_empty(cpu_rcache->loaded)) {
 		has_pfn = true;
-	} else if (!iova_magazine_empty(cpu_rcache->prev)) {
+	} else if (iova_magazine_full(cpu_rcache->prev)) {
 		swap(cpu_rcache->prev, cpu_rcache->loaded);
 		has_pfn = true;
 	} else {
 		spin_lock(&rcache->lock);
 		if (rcache->depot_size > 0) {
-			iova_magazine_free(cpu_rcache->loaded);
-			cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
+			swap(rcache->depot[rcache->depot_size - 1], cpu_rcache->prev);
+			swap(cpu_rcache->prev, cpu_rcache->loaded);
+			rcache->depot_size--;
 			has_pfn = true;
 		}
 		spin_unlock(&rcache->lock);
@@ -1019,7 +1030,7 @@ static void free_iova_rcaches(struct iova_domain *iovad)
 			iova_magazine_free(cpu_rcache->prev);
 		}
 		free_percpu(rcache->cpu_rcaches);
-		for (j = 0; j < rcache->depot_size; ++j)
+		for (j = 0; j < MAX_GLOBAL_MAGS; ++j)
 			iova_magazine_free(rcache->depot[j]);
 	}
 }
Signature · 1 line
2.21.0
#2 lore raw

Re: [Patch v2 1/3] iommu: match the original algorithm

Quoted 3 lines · Cong Wang

The IOVA cache algorithm implemented in IOMMU code does not exactly match the original algorithm described in the paper.

which paper?

Quoted 8 lines

Particularly, it doesn't need to free the loaded empty magazine when trying to put it back to global depot. To make it work, we have to pre-allocate magazines in the depot and only recycle them when all of them are full.

Before this patch, rcache->depot[] contains either full or freed entries, after this patch, it contains either full or empty (but allocated) entries.

I *quickly* tested this patch and got a small performance gain.

Quoted 30 lines
Cc Joerg Roedel <[email protected]>
---
  drivers/iommu/iova.c | 45 +++++++++++++++++++++++++++-----------------
  1 file changed, 28 insertions(+), 17 deletions(-)
drivers/iommu/iova.c +5 -1
index 41c605b0058f..cb473ddce4cf 100644
@@ -862,12 +862,16 @@ static void init_iova_rcaches(struct iova_domain *iovad)
  	struct iova_cpu_rcache *cpu_rcache;
  	struct iova_rcache *rcache;
  	unsigned int cpu;
-	int i;
+	int i, j;
 
  	for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
  		rcache = &iovad->rcaches[i];
  		spin_lock_init(&rcache->lock);
  		rcache->depot_size = 0;
+		for (j = 0; j < MAX_GLOBAL_MAGS; ++j) {
+			rcache->depot[j] = iova_magazine_alloc(GFP_KERNEL);
+			WARN_ON(!rcache->depot[j]);
+		}
  		rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
  		if (WARN_ON(!rcache->cpu_rcaches))
  			continue;

drivers/iommu/iova.c static bool __iova_rcache_insert(struct iova_domain *iovad, · line 907

Quoted 5 lines
drivers/iommu/iova.c +1 -1
@@ -900,24 +904,30 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
 
  	if (!iova_magazine_full(cpu_rcache->loaded)) {
  		can_insert = true;
-	} else if (!iova_magazine_full(cpu_rcache->prev)) {
+	} else if (iova_magazine_empty(cpu_rcache->prev)) {

is this change strictly necessary?

Quoted 39 lines
drivers/iommu/iova.c +17 -11
  		swap(cpu_rcache->prev, cpu_rcache->loaded);
  		can_insert = true;
  	} else {
-		struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
+		spin_lock(&rcache->lock);
+		if (rcache->depot_size < MAX_GLOBAL_MAGS) {
+			swap(rcache->depot[rcache->depot_size], cpu_rcache->prev);
+			swap(cpu_rcache->prev, cpu_rcache->loaded);
+			rcache->depot_size++;
+			can_insert = true;
+		} else {
+			mag_to_free = cpu_rcache->loaded;
+		}
+		spin_unlock(&rcache->lock);
+
+		if (mag_to_free) {
+			struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
 
-		if (new_mag) {
-			spin_lock(&rcache->lock);
-			if (rcache->depot_size < MAX_GLOBAL_MAGS) {
-				rcache->depot[rcache->depot_size++] =
-						cpu_rcache->loaded;
+			if (new_mag) {
+				cpu_rcache->loaded = new_mag;
+				can_insert = true;
  			} else {
-				mag_to_free = cpu_rcache->loaded;
+				mag_to_free = NULL;
  			}
-			spin_unlock(&rcache->lock);
-
-			cpu_rcache->loaded = new_mag;
-			can_insert = true;
  		}
  	}
 
@@ -963,14 +973,15 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
 

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 972

Quoted 10 lines
drivers/iommu/iova.c +1 -2
  	if (!iova_magazine_empty(cpu_rcache->loaded)) {
  		has_pfn = true;
-	} else if (!iova_magazine_empty(cpu_rcache->prev)) {
+	} else if (iova_magazine_full(cpu_rcache->prev)) {
  		swap(cpu_rcache->prev, cpu_rcache->loaded);
  		has_pfn = true;
  	} else {
  		spin_lock(&rcache->lock);
  		if (rcache->depot_size > 0) {
-			iova_magazine_free(cpu_rcache->loaded);

it is good to remove this from under the lock, apart from this change

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 984

Quoted 4 lines
drivers/iommu/iova.c +3 -1
-			cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
+			swap(rcache->depot[rcache->depot_size - 1], cpu_rcache->prev);
+			swap(cpu_rcache->prev, cpu_rcache->loaded);
+			rcache->depot_size--;

I'm not sure how appropriate the name "depot_size" is any longer.

Quoted 13 lines
drivers/iommu/iova.c +1 -1
  			has_pfn = true;
  		}
  		spin_unlock(&rcache->lock);
@@ -1019,7 +1030,7 @@ static void free_iova_rcaches(struct iova_domain *iovad)
  			iova_magazine_free(cpu_rcache->prev);
  		}
  		free_percpu(rcache->cpu_rcaches);
-		for (j = 0; j < rcache->depot_size; ++j)
+		for (j = 0; j < MAX_GLOBAL_MAGS; ++j)
  			iova_magazine_free(rcache->depot[j]);
  	}
  }
author
#3 lore raw

Re: [Patch v2 1/3] iommu: match the original algorithm

Quoted 7 lines · John Garry
Quoted 3 lines · Cong Wang

The IOVA cache algorithm implemented in IOMMU code does not exactly match the original algorithm described in the paper.

which paper?

It's in drivers/iommu/iova.c, from line 769:

769 /*

 770  * Magazine caches for IOVA ranges.  For an introduction to magazines,
 771  * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab
 772  * Allocator to Many CPUs and Arbitrary Resources" by Bonwick and Adams.
 773  * For simplicity, we use a static magazine size and don't implement the
 774  * dynamic size tuning described in the paper.
 775  */
Quoted 11 lines
Quoted 9 lines

Particularly, it doesn't need to free the loaded empty magazine when trying to put it back to global depot. To make it work, we have to pre-allocate magazines in the depot and only recycle them when all of them are full.

Before this patch, rcache->depot[] contains either full or freed entries, after this patch, it contains either full or empty (but allocated) entries.

I *quickly* tested this patch and got a small performance gain.

Thanks for testing! It requires a different workload to see bigger gain, in our case, 24 memcache.parallel servers with 120 clients.

Quoted 32 lines
Quoted 32 lines
Cc Joerg Roedel <[email protected]>
---
  drivers/iommu/iova.c | 45 +++++++++++++++++++++++++++-----------------
  1 file changed, 28 insertions(+), 17 deletions(-)
drivers/iommu/iova.c +5 -1
index 41c605b0058f..cb473ddce4cf 100644
@@ -862,12 +862,16 @@ static void init_iova_rcaches(struct iova_domain *iovad)
      struct iova_cpu_rcache *cpu_rcache;
      struct iova_rcache *rcache;
      unsigned int cpu;
-     int i;
+     int i, j;
 
      for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
              rcache = &iovad->rcaches[i];
              spin_lock_init(&rcache->lock);
              rcache->depot_size = 0;
+             for (j = 0; j < MAX_GLOBAL_MAGS; ++j) {
+                     rcache->depot[j] = iova_magazine_alloc(GFP_KERNEL);
+                     WARN_ON(!rcache->depot[j]);
+             }
              rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
              if (WARN_ON(!rcache->cpu_rcaches))
                      continue;
@@ -900,24 +904,30 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
 
Quoted 6 lines

drivers/iommu/iova.c static bool __iova_rcache_insert(struct iova_domain *iovad, · line 907

Quoted 4 lines
drivers/iommu/iova.c +1 -1
      if (!iova_magazine_full(cpu_rcache->loaded)) {
              can_insert = true;
-     } else if (!iova_magazine_full(cpu_rcache->prev)) {
+     } else if (iova_magazine_empty(cpu_rcache->prev)) {

is this change strictly necessary?

Yes, because it is what described in the paper. But it should be functionally same because cpu_rcache->prev is either full or empty.

Quoted 53 lines
Quoted 40 lines
drivers/iommu/iova.c +17 -11
              swap(cpu_rcache->prev, cpu_rcache->loaded);
              can_insert = true;
      } else {
-             struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
+             spin_lock(&rcache->lock);
+             if (rcache->depot_size < MAX_GLOBAL_MAGS) {
+                     swap(rcache->depot[rcache->depot_size], cpu_rcache->prev);
+                     swap(cpu_rcache->prev, cpu_rcache->loaded);
+                     rcache->depot_size++;
+                     can_insert = true;
+             } else {
+                     mag_to_free = cpu_rcache->loaded;
+             }
+             spin_unlock(&rcache->lock);
+
+             if (mag_to_free) {
+                     struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
 
-             if (new_mag) {
-                     spin_lock(&rcache->lock);
-                     if (rcache->depot_size < MAX_GLOBAL_MAGS) {
-                             rcache->depot[rcache->depot_size++] =
-                                             cpu_rcache->loaded;
+                     if (new_mag) {
+                             cpu_rcache->loaded = new_mag;
+                             can_insert = true;
                      } else {
-                             mag_to_free = cpu_rcache->loaded;
+                             mag_to_free = NULL;
                      }
-                     spin_unlock(&rcache->lock);
-
-                     cpu_rcache->loaded = new_mag;
-                     can_insert = true;
              }
      }
 
@@ -963,14 +973,15 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
 

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 972

Quoted 10 lines
drivers/iommu/iova.c +1 -2
      if (!iova_magazine_empty(cpu_rcache->loaded)) {
              has_pfn = true;
-     } else if (!iova_magazine_empty(cpu_rcache->prev)) {
+     } else if (iova_magazine_full(cpu_rcache->prev)) {
              swap(cpu_rcache->prev, cpu_rcache->loaded);
              has_pfn = true;
      } else {
              spin_lock(&rcache->lock);
              if (rcache->depot_size > 0) {
-                     iova_magazine_free(cpu_rcache->loaded);

it is good to remove this from under the lock, apart from this change

Quoted 6 lines

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 984

Quoted 4 lines
drivers/iommu/iova.c +3 -1
-                     cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
+                     swap(rcache->depot[rcache->depot_size - 1], cpu_rcache->prev);
+                     swap(cpu_rcache->prev, cpu_rcache->loaded);
+                     rcache->depot_size--;

I'm not sure how appropriate the name "depot_size" is any longer.

I think it is still okay, because empty ones don't count. However if you have better names, I am open to your suggestion.

Thanks.

#4 lore raw

Re: [Patch v2 1/3] iommu: match the original algorithm

Quoted 26 lines · Cong Wang
Quoted 7 lines · John Garry
Quoted 3 lines · Cong Wang

The IOVA cache algorithm implemented in IOMMU code does not exactly match the original algorithm described in the paper.

which paper?

It's in drivers/iommu/iova.c, from line 769:

769 /*

  770  * Magazine caches for IOVA ranges.  For an introduction to magazines,
  771  * see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab
  772  * Allocator to Many CPUs and Arbitrary Resources" by Bonwick and Adams.
  773  * For simplicity, we use a static magazine size and don't implement the
  774  * dynamic size tuning described in the paper.
  775  */
Quoted 6 lines
Quoted 6 lines

Particularly, it doesn't need to free the loaded empty magazine when trying to put it back to global depot. To make it work, we have to pre-allocate magazines in the depot and only recycle them when all of them are full.

Quoted 9 lines
Quoted 5 lines
Quoted 3 lines

Before this patch, rcache->depot[] contains either full or freed entries, after this patch, it contains either full or empty (but allocated) entries.

I *quickly* tested this patch and got a small performance gain.

Thanks for testing! It requires a different workload to see bigger gain, in our case, 24 memcache.parallel servers with 120 clients.

So in fact I was getting a ~10% throughput boost for my storage test. Seems more than I would expect/hope for. I would need to test more.

Quoted 40 lines
Quoted 33 lines
Quoted 33 lines
Cc Joerg Roedel <[email protected]>
---
   drivers/iommu/iova.c | 45 +++++++++++++++++++++++++++-----------------
   1 file changed, 28 insertions(+), 17 deletions(-)
drivers/iommu/iova.c +5 -1
index 41c605b0058f..cb473ddce4cf 100644
@@ -862,12 +862,16 @@ static void init_iova_rcaches(struct iova_domain *iovad)
       struct iova_cpu_rcache *cpu_rcache;
       struct iova_rcache *rcache;
       unsigned int cpu;
-     int i;
+     int i, j;
 
       for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
               rcache = &iovad->rcaches[i];
               spin_lock_init(&rcache->lock);
               rcache->depot_size = 0;
+             for (j = 0; j < MAX_GLOBAL_MAGS; ++j) {
+                     rcache->depot[j] = iova_magazine_alloc(GFP_KERNEL);
+                     WARN_ON(!rcache->depot[j]);
+             }
               rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
               if (WARN_ON(!rcache->cpu_rcaches))
                       continue;
@@ -900,24 +904,30 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
 
Quoted 6 lines

drivers/iommu/iova.c static bool __iova_rcache_insert(struct iova_domain *iovad, · line 907

Quoted 4 lines
drivers/iommu/iova.c +1 -1
       if (!iova_magazine_full(cpu_rcache->loaded)) {
               can_insert = true;
-     } else if (!iova_magazine_full(cpu_rcache->prev)) {
+     } else if (iova_magazine_empty(cpu_rcache->prev)) {

is this change strictly necessary?

Quoted 2 lines

Yes, because it is what described in the paper. But it should be functionally same because cpu_rcache->prev is either full or empty.

That is was what I was getting at.

Quoted 8 lines
Quoted 8 lines

drivers/iommu/iova.c static bool __iova_rcache_insert(struct iova_domain *iovad, · line 907

Quoted 8 lines
drivers/iommu/iova.c +0 -1
               swap(cpu_rcache->prev, cpu_rcache->loaded);
               can_insert = true;
       } else {
-             struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);

Apart from this change, did anyone ever consider kmem cache for the magazines?

Quoted 48 lines
Quoted 48 lines
Quoted 35 lines
drivers/iommu/iova.c +17 -10
+             spin_lock(&rcache->lock);
+             if (rcache->depot_size < MAX_GLOBAL_MAGS) {
+                     swap(rcache->depot[rcache->depot_size], cpu_rcache->prev);
+                     swap(cpu_rcache->prev, cpu_rcache->loaded);
+                     rcache->depot_size++;
+                     can_insert = true;
+             } else {
+                     mag_to_free = cpu_rcache->loaded;
+             }
+             spin_unlock(&rcache->lock);
+
+             if (mag_to_free) {
+                     struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
 
-             if (new_mag) {
-                     spin_lock(&rcache->lock);
-                     if (rcache->depot_size < MAX_GLOBAL_MAGS) {
-                             rcache->depot[rcache->depot_size++] =
-                                             cpu_rcache->loaded;
+                     if (new_mag) {
+                             cpu_rcache->loaded = new_mag;
+                             can_insert = true;
                       } else {
-                             mag_to_free = cpu_rcache->loaded;
+                             mag_to_free = NULL;
                       }
-                     spin_unlock(&rcache->lock);
-
-                     cpu_rcache->loaded = new_mag;
-                     can_insert = true;
               }
       }
 
@@ -963,14 +973,15 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
 

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 972

Quoted 10 lines
drivers/iommu/iova.c +1 -2
       if (!iova_magazine_empty(cpu_rcache->loaded)) {
               has_pfn = true;
-     } else if (!iova_magazine_empty(cpu_rcache->prev)) {
+     } else if (iova_magazine_full(cpu_rcache->prev)) {
               swap(cpu_rcache->prev, cpu_rcache->loaded);
               has_pfn = true;
       } else {
               spin_lock(&rcache->lock);
               if (rcache->depot_size > 0) {
-                     iova_magazine_free(cpu_rcache->loaded);

it is good to remove this from under the lock, apart from this change

Quoted 3 lines
Quoted 3 lines

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 983

Quoted 3 lines
drivers/iommu/iova.c +2 -1
-                     cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
+                     swap(rcache->depot[rcache->depot_size - 1], cpu_rcache->prev);
+                     swap(cpu_rcache->prev, cpu_rcache->loaded);

I wonder if not using swap() at all is neater here.

Quoted 6 lines
Quoted 3 lines

drivers/iommu/iova.c static unsigned long __iova_rcache_get(struct iova_rcache *rcache, · line 984

Quoted 1 line
drivers/iommu/iova.c +1 -0
+                     rcache->depot_size--;

I'm not sure how appropriate the name "depot_size" is any longer.

I think it is still okay, because empty ones don't count. However if you have better names, I am open to your suggestion.

Yeah, probably.

thanks,
John

author
#5 lore raw

Re: [Patch v2 1/3] iommu: match the original algorithm

Quoted 2 lines · John Garry

Apart from this change, did anyone ever consider kmem cache for the magazines?

You can always make any changes you want after this patch, I can't do all optimizations in one single patch. :)

So, I will leave this to you.

Thanks.

#6 lore raw

Re: [Patch v2 1/3] iommu: match the original algorithm

In reply to Cong Wang

I think a subject line better describes what you change, no that it matches an original algorithm. The fact that the fix matches the original algorithm can go somewhere towards the commit log, preferably with a reference to the actual paper.

author
#7 lore raw

Re: [Patch v2 1/3] iommu: match the original algorithm

Quoted 5 lines · Christoph Hellwig

I think a subject line better describes what you change, no that it matches an original algorithm. The fact that the fix matches the original algorithm can go somewhere towards the commit log, preferably with a reference to the actual paper.

Ok, I will change subject to "iommu: avoid unnecessary magazine allocations".

And I will put the title of the paper in the commit log even though it is already in the code comment.

Thanks.

author
#8 lore raw

PATCH v2 2/3 iommu: optimize iova_magazine_free_pfns()

In reply to Cong Wang

If the maganize is empty, iova_magazine_free_pfns() should be a nop, however it misses the case of mag->size==0. So we should just call iova_magazine_empty().

This should reduce the contention on iovad->iova_rbtree_lock a little bit.

Cc Joerg Roedel <[email protected]>
---
 drivers/iommu/iova.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)
drivers/iommu/iova.c +11 -11
index cb473ddce4cf..184d4c0e20b5 100644
@@ -797,13 +797,23 @@ static void iova_magazine_free(struct iova_magazine *mag)
 	kfree(mag);
 }
 
+static bool iova_magazine_full(struct iova_magazine *mag)
+{
+	return (mag && mag->size == IOVA_MAG_SIZE);
+}
+
+static bool iova_magazine_empty(struct iova_magazine *mag)
+{
+	return (!mag || mag->size == 0);
+}
+
 static void
 iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
 {
 	unsigned long flags;
 	int i;
 
-	if (!mag)
+	if (iova_magazine_empty(mag))
 		return;
 
 	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
@@ -820,16 +830,6 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
 	mag->size = 0;
 }
 
-static bool iova_magazine_full(struct iova_magazine *mag)
-{
-	return (mag && mag->size == IOVA_MAG_SIZE);
-}
-
-static bool iova_magazine_empty(struct iova_magazine *mag)
-{
-	return (!mag || mag->size == 0);
-}
-
 static unsigned long iova_magazine_pop(struct iova_magazine *mag,
 				       unsigned long limit_pfn)
 {
Signature · 6 lines
2.21.0

_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
#9 lore raw

Re: [Patch v2 2/3] iommu: optimize iova_magazine_free_pfns()

Quoted 1 line · Cong Wang

If the maganize is empty, iova_magazine_free_pfns() should

magazine

Quoted 28 lines

be a nop, however it misses the case of mag->size==0. So we should just call iova_magazine_empty().

This should reduce the contention on iovad->iova_rbtree_lock a little bit.

Cc Joerg Roedel <[email protected]>
---
  drivers/iommu/iova.c | 22 +++++++++++-----------
  1 file changed, 11 insertions(+), 11 deletions(-)
drivers/iommu/iova.c +8 -0
index cb473ddce4cf..184d4c0e20b5 100644
@@ -797,13 +797,23 @@ static void iova_magazine_free(struct iova_magazine *mag)
  	kfree(mag);
  }
 
+static bool iova_magazine_full(struct iova_magazine *mag)
+{
+	return (mag && mag->size == IOVA_MAG_SIZE);
+}
+
+static bool iova_magazine_empty(struct iova_magazine *mag)
+{
+	return (!mag || mag->size == 0);

drivers/iommu/iova.c static void iova_magazine_free(struct iova_magazine *mag) · line 816

Quoted 10 lines
drivers/iommu/iova.c +3 -1
+}
+
  static void
  iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
  {
  	unsigned long flags;
  	int i;
 
-	if (!mag)
+	if (iova_magazine_empty(mag))

The only hot path we this call is __iova_rcache_insert()->iova_magazine_free_pfns(mag_to_free) and mag_to_free is full in this case, so I am sure how the additional check helps, right?

Thanks,
John

Quoted 21 lines
drivers/iommu/iova.c +0 -10
  		return;
 
  	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
@@ -820,16 +830,6 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
  	mag->size = 0;
  }
 
-static bool iova_magazine_full(struct iova_magazine *mag)
-{
-	return (mag && mag->size == IOVA_MAG_SIZE);
-}
-
-static bool iova_magazine_empty(struct iova_magazine *mag)
-{
-	return (!mag || mag->size == 0);
-}
-
  static unsigned long iova_magazine_pop(struct iova_magazine *mag,
  				       unsigned long limit_pfn)
  {
author
#10 lore raw

Re: [Patch v2 2/3] iommu: optimize iova_magazine_free_pfns()

Quoted 5 lines · John Garry
Quoted 1 line · Cong Wang

If the maganize is empty, iova_magazine_free_pfns() should

magazine

Good catch!

Quoted 37 lines
Quoted 37 lines

be a nop, however it misses the case of mag->size==0. So we should just call iova_magazine_empty().

This should reduce the contention on iovad->iova_rbtree_lock a little bit.

Cc Joerg Roedel <[email protected]>
---
  drivers/iommu/iova.c | 22 +++++++++++-----------
  1 file changed, 11 insertions(+), 11 deletions(-)
drivers/iommu/iova.c +10 -0
index cb473ddce4cf..184d4c0e20b5 100644
@@ -797,13 +797,23 @@ static void iova_magazine_free(struct iova_magazine *mag)
      kfree(mag);
  }
 
+static bool iova_magazine_full(struct iova_magazine *mag)
+{
+     return (mag && mag->size == IOVA_MAG_SIZE);
+}
+
+static bool iova_magazine_empty(struct iova_magazine *mag)
+{
+     return (!mag || mag->size == 0);
+}
+
  static void
  iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
  {
      unsigned long flags;
      int i;
 
Quoted 7 lines

drivers/iommu/iova.c static void iova_magazine_free(struct iova_magazine *mag) · line 816

Quoted 2 lines
drivers/iommu/iova.c +1 -1
-     if (!mag)
+     if (iova_magazine_empty(mag))

The only hot path we this call is __iova_rcache_insert()->iova_magazine_free_pfns(mag_to_free) and mag_to_free is full in this case, so I am sure how the additional check helps, right?

This is what I mean by "a little bit" in changelog, did you miss it or misunderstand it? :)

Thanks.

#11 lore raw

Re: [Patch v2 2/3] iommu: optimize iova_magazine_free_pfns()

Quoted 54 lines · Cong Wang
Quoted 5 lines · John Garry
Quoted 1 line · Cong Wang

If the maganize is empty, iova_magazine_free_pfns() should

magazine

Good catch!

Quoted 37 lines
Quoted 37 lines

be a nop, however it misses the case of mag->size==0. So we should just call iova_magazine_empty().

This should reduce the contention on iovad->iova_rbtree_lock a little bit.

Cc Joerg Roedel <[email protected]>
---
   drivers/iommu/iova.c | 22 +++++++++++-----------
   1 file changed, 11 insertions(+), 11 deletions(-)
drivers/iommu/iova.c +10 -0
index cb473ddce4cf..184d4c0e20b5 100644
@@ -797,13 +797,23 @@ static void iova_magazine_free(struct iova_magazine *mag)
       kfree(mag);
   }
 
+static bool iova_magazine_full(struct iova_magazine *mag)
+{
+     return (mag && mag->size == IOVA_MAG_SIZE);
+}
+
+static bool iova_magazine_empty(struct iova_magazine *mag)
+{
+     return (!mag || mag->size == 0);
+}
+
   static void
   iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
   {
       unsigned long flags;
       int i;
 
Quoted 7 lines

drivers/iommu/iova.c static void iova_magazine_free(struct iova_magazine *mag) · line 816

Quoted 2 lines
drivers/iommu/iova.c +1 -1
-     if (!mag)
+     if (iova_magazine_empty(mag))

The only hot path we this call is __iova_rcache_insert()->iova_magazine_free_pfns(mag_to_free) and mag_to_free is full in this case, so I am sure how the additional check helps, right?

Quoted 2 lines

This is what I mean by "a little bit" in changelog, did you miss it or misunderstand it? :)

I was concerned that in the fastpath we actually make things very marginally slower by adding a check which will fail.

Thanks,
John

Quoted 4 lines

Thanks.
.

author
#12 lore raw

Re: [Patch v2 2/3] iommu: optimize iova_magazine_free_pfns()

Quoted 56 lines · John Garry
Quoted 54 lines · Cong Wang
Quoted 5 lines · John Garry
Quoted 1 line · Cong Wang

If the maganize is empty, iova_magazine_free_pfns() should

magazine

Good catch!

Quoted 44 lines
Quoted 29 lines

be a nop, however it misses the case of mag->size==0. So we should just call iova_magazine_empty().

This should reduce the contention on iovad->iova_rbtree_lock a little bit.

Cc Joerg Roedel <[email protected]>
---
   drivers/iommu/iova.c | 22 +++++++++++-----------
   1 file changed, 11 insertions(+), 11 deletions(-)
drivers/iommu/iova.c +8 -0
index cb473ddce4cf..184d4c0e20b5 100644
@@ -797,13 +797,23 @@ static void iova_magazine_free(struct iova_magazine *mag)
       kfree(mag);
   }
 
+static bool iova_magazine_full(struct iova_magazine *mag)
+{
+     return (mag && mag->size == IOVA_MAG_SIZE);
+}
+
+static bool iova_magazine_empty(struct iova_magazine *mag)
+{
+     return (!mag || mag->size == 0);

drivers/iommu/iova.c static void iova_magazine_free(struct iova_magazine *mag) · line 816

Quoted 10 lines
drivers/iommu/iova.c +3 -1
+}
+
   static void
   iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
   {
       unsigned long flags;
       int i;
 
-     if (!mag)
+     if (iova_magazine_empty(mag))

The only hot path we this call is __iova_rcache_insert()->iova_magazine_free_pfns(mag_to_free) and mag_to_free is full in this case, so I am sure how the additional check helps, right?

Quoted 5 lines
Quoted 2 lines

This is what I mean by "a little bit" in changelog, did you miss it or misunderstand it? :)

I was concerned that in the fastpath we actually make things very marginally slower by adding a check which will fail.

The check is done without any locking, so it is cheap. And it is a common pattern that we do a check without lock and do a second same check with lock:

https://en.wikipedia.org/wiki/Double-checked_locking

Thanks.

#13 lore raw

Re: [Patch v2 2/3] iommu: optimize iova_magazine_free_pfns()

In reply to Cong Wang

drivers/iommu/iova.c static void iova_magazine_free(struct iova_magazine *mag)

Quoted 3 lines
drivers/iommu/iova.c +2 -0
+	return (mag && mag->size == IOVA_MAG_SIZE);
 
+	return (!mag || mag->size == 0);

No need for the braces in both cases.
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu

author
#14 lore raw

Re: [Patch v2 2/3] iommu: optimize iova_magazine_free_pfns()

Quoted 6 lines · Christoph Hellwig

drivers/iommu/iova.c static void iova_magazine_free(struct iova_magazine *mag)

Quoted 4 lines
drivers/iommu/iova.c +2 -0
+     return (mag && mag->size == IOVA_MAG_SIZE);
 
+     return (!mag || mag->size == 0);

No need for the braces in both cases.

The current code is already this, I don't want to mix coding style changes with a non-coding-style change. You can always remove them in a separated patch if you feel necessary.

Thanks.

author
#15 lore raw

PATCH v2 3/3 iommu: avoid taking iova_rbtree_lock twice

In reply to Cong Wang

Both find_iova() and __free_iova() take iova_rbtree_lock, there is no reason to take and release it twice inside free_iova().

Fold them into the critical section by calling the unlock versions instead.

Cc Joerg Roedel <[email protected]>
---
 drivers/iommu/iova.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
drivers/iommu/iova.c +6 -2
index 184d4c0e20b5..f46f8f794678 100644
@@ -390,10 +390,14 @@ EXPORT_SYMBOL_GPL(__free_iova);
 void
 free_iova(struct iova_domain *iovad, unsigned long pfn)
 {
-	struct iova *iova = find_iova(iovad, pfn);
+	unsigned long flags;
+	struct iova *iova;
 
+	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
+	iova = private_find_iova(iovad, pfn);
 	if (iova)
-		__free_iova(iovad, iova);
+		private_free_iova(iovad, iova);
+	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
 
 }
 EXPORT_SYMBOL_GPL(free_iova);
Signature · 1 line
2.21.0
#16 lore raw

Re: [Patch v2 3/3] iommu: avoid taking iova_rbtree_lock twice

Quoted 6 lines · Cong Wang

Both find_iova() and __free_iova() take iova_rbtree_lock, there is no reason to take and release it twice inside free_iova().

Fold them into the critical section by calling the unlock versions instead.

Since generally the iova would be non-NULL, this seems a reasonable change (which could be mentioned in the commit log)

John

Quoted 29 lines
Cc Joerg Roedel <[email protected]>
---
  drivers/iommu/iova.c | 8 ++++++--
  1 file changed, 6 insertions(+), 2 deletions(-)
drivers/iommu/iova.c +6 -2
index 184d4c0e20b5..f46f8f794678 100644
@@ -390,10 +390,14 @@ EXPORT_SYMBOL_GPL(__free_iova);
  void
  free_iova(struct iova_domain *iovad, unsigned long pfn)
  {
-	struct iova *iova = find_iova(iovad, pfn);
+	unsigned long flags;
+	struct iova *iova;
 
+	spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
+	iova = private_find_iova(iovad, pfn);
  	if (iova)
-		__free_iova(iovad, iova);
+		private_free_iova(iovad, iova);
+	spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
 
  }
  EXPORT_SYMBOL_GPL(free_iova);
author
#17 lore raw

Re: [Patch v2 3/3] iommu: avoid taking iova_rbtree_lock twice

Quoted 11 lines · John Garry
Quoted 6 lines · Cong Wang

Both find_iova() and __free_iova() take iova_rbtree_lock, there is no reason to take and release it twice inside free_iova().

Fold them into the critical section by calling the unlock versions instead.

Since generally the iova would be non-NULL, this seems a reasonable change (which could be mentioned in the commit log)

I think it is too obvious to mention it. There are many things we can mention but we should only mention what's necessary, right?

Thanks.
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu

maintainer
#18 lore raw
In reply to Cong Wang
Quoted 3 lines · Cong Wang

This patchset contains three small optimizations for the global spinlock contention in IOVA cache. Our memcache perf test shows this reduced its p999 latency down by 45% on AMD when IOMMU is enabled.

Sounds nice. Can you please re-send with Robin Murphy on Cc? Robin, can you have a look at these IOVA changes please?

Thanks,

Joerg

author
#19 lore raw
Quoted 8 lines · Joerg Roedel
Quoted 3 lines · Cong Wang

This patchset contains three small optimizations for the global spinlock contention in IOVA cache. Our memcache perf test shows this reduced its p999 latency down by 45% on AMD when IOMMU is enabled.

Sounds nice. Can you please re-send with Robin Murphy on Cc? Robin, can you have a look at these IOVA changes please?

I will resend V3 with Robin Cc'ed.

Thanks.