iommu/iova: Move CPU magazine init to first insert
Subsystems (2)
PATCH v3 1/1 iommu/iova: Move CPU magazine init to first insert
A large amount of memory may be allocated for these magazines on machines with a lot of IOMMU groups and CPU cores. Not all may be used as some devices may be unused or be bound to drivers that do not use the DMA-API. Furthermore, some drivers may not use all levels or CPUs.
Move the initialization of the loaded and prev magazines for each CPU on the first attempt to try to insert a freed IOVA to them.
---
v2: only rebase
v3: Combine the initilaization logic with new magazine logic in __iova_rcache_insert
---
Change-Id: I987b6dfb2b1125d8da8618fff540f315b99bad13
--- drivers/iommu/iova.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
drivers/iommu/iova.c +15 -14
@@ -621,6 +621,9 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)unsigned long flags;int i; +if (!mag) +return ; +spin_lock_irqsave (&iovad->iova_rbtree_lock, flags);for (i =0 ; i < mag->size; ++i) {
@@ -739,12 +742,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad) cpu_rcache =per_cpu_ptr (rcache->cpu_rcaches, cpu);spin_lock_init (&cpu_rcache->lock); - cpu_rcache->loaded =iova_magazine_alloc (GFP_KERNEL ); - cpu_rcache->prev =iova_magazine_alloc (GFP_KERNEL ); -if (!cpu_rcache->loaded || !cpu_rcache->prev) { - ret = -ENOMEM ; - goto out_err; - } } }
@@ -777,19 +774,23 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, cpu_rcache =raw_cpu_ptr (rcache->cpu_rcaches);spin_lock_irqsave (&cpu_rcache->lock, flags); -if (!iova_magazine_full (cpu_rcache->loaded)) { +if (cpu_rcache->loaded && !iova_magazine_full (cpu_rcache->loaded)) { can_insert = true; - }else if (!iova_magazine_full (cpu_rcache->prev)) { + }else if (cpu_rcache->prev && !iova_magazine_full (cpu_rcache->prev)) {swap (cpu_rcache->prev, cpu_rcache->loaded); can_insert = true; }else {struct iova_magazine *new_mag =iova_magazine_alloc (GFP_ATOMIC );if (new_mag) { -spin_lock (&rcache->lock); -iova_depot_push (rcache, cpu_rcache->loaded); -spin_unlock (&rcache->lock); -schedule_delayed_work (&rcache->work,IOVA_DEPOT_DELAY ); +if (cpu_rcache->loaded && !cpu_rcache->prev) { + cpu_rcache->prev = cpu_rcache->loaded; + }else if (cpu_rcache->loaded) { +spin_lock (&rcache->lock); +iova_depot_push (rcache, cpu_rcache->loaded); +spin_unlock (&rcache->lock); +schedule_delayed_work (&rcache->work,IOVA_DEPOT_DELAY ); + } cpu_rcache->loaded = new_mag; can_insert = true;
@@ -831,9 +832,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, cpu_rcache =raw_cpu_ptr (rcache->cpu_rcaches);spin_lock_irqsave (&cpu_rcache->lock, flags); -if (!iova_magazine_empty (cpu_rcache->loaded)) { +if (cpu_rcache->loaded && !iova_magazine_empty (cpu_rcache->loaded)) { has_pfn = true; - }else if (!iova_magazine_empty (cpu_rcache->prev)) { + }else if (cpu_rcache->prev && !iova_magazine_empty (cpu_rcache->prev)) {swap (cpu_rcache->prev, cpu_rcache->loaded); has_pfn = true; }else {
Signature · 1 line
2.55.0.229.g6434b31f56-goog
Quoted 5 lines · Logan Odell
A large amount of memory may be allocated for these magazines on machines with a lot of IOMMU groups and CPU cores. Not all may be used as some devices may be unused or be bound to drivers that do not use the DMA-API. Furthermore, some drivers may not use all levels or CPUs.
Quoted 8 lines
Move the initialization of the loaded and prev magazines for each CPU on the first attempt to try to insert a freed IOVA to them.
Signed-off-by Logan Odell <[email protected]>Signed-off-by Michal Clapinski <[email protected]>---
v2: only rebase
v3: Combine the initilaization logic with new magazine logic in__iova_rcache_insert
LGTM,
Although as a process nit, since you've now taken back custody of the patch to make this update, technically you should either add your sign-off a second time after Michal's, or perhaps more neatly, give Michal a Co-developed-by and swap the sign-offs[1].
Thanks,
Robin.
Quoted 76 lines
---
Change-Id: I987b6dfb2b1125d8da8618fff540f315b99bad13
--- drivers/iommu/iova.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)drivers/iommu/iova.c +15
-14@@ -621,6 +621,9 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)unsigned long flags;int i; +if (!mag) +return ; +spin_lock_irqsave (&iovad->iova_rbtree_lock, flags);for (i =0 ; i < mag->size; ++i) {@@ -739,12 +742,6 @@ int iova_domain_init_rcaches(struct iova_domain *iovad) cpu_rcache =per_cpu_ptr (rcache->cpu_rcaches, cpu);spin_lock_init (&cpu_rcache->lock); - cpu_rcache->loaded =iova_magazine_alloc (GFP_KERNEL ); - cpu_rcache->prev =iova_magazine_alloc (GFP_KERNEL ); -if (!cpu_rcache->loaded || !cpu_rcache->prev) { - ret = -ENOMEM ; - goto out_err; - } } }@@ -777,19 +774,23 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, cpu_rcache =raw_cpu_ptr (rcache->cpu_rcaches);spin_lock_irqsave (&cpu_rcache->lock, flags); -if (!iova_magazine_full (cpu_rcache->loaded)) { +if (cpu_rcache->loaded && !iova_magazine_full (cpu_rcache->loaded)) { can_insert = true; - }else if (!iova_magazine_full (cpu_rcache->prev)) { + }else if (cpu_rcache->prev && !iova_magazine_full (cpu_rcache->prev)) {swap (cpu_rcache->prev, cpu_rcache->loaded); can_insert = true; }else {struct iova_magazine *new_mag =iova_magazine_alloc (GFP_ATOMIC );if (new_mag) { -spin_lock (&rcache->lock); -iova_depot_push (rcache, cpu_rcache->loaded); -spin_unlock (&rcache->lock); -schedule_delayed_work (&rcache->work,IOVA_DEPOT_DELAY ); +if (cpu_rcache->loaded && !cpu_rcache->prev) { + cpu_rcache->prev = cpu_rcache->loaded; + }else if (cpu_rcache->loaded) { +spin_lock (&rcache->lock); +iova_depot_push (rcache, cpu_rcache->loaded); +spin_unlock (&rcache->lock); +schedule_delayed_work (&rcache->work,IOVA_DEPOT_DELAY ); + } cpu_rcache->loaded = new_mag; can_insert = true;@@ -831,9 +832,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, cpu_rcache =raw_cpu_ptr (rcache->cpu_rcaches);spin_lock_irqsave (&cpu_rcache->lock, flags); -if (!iova_magazine_empty (cpu_rcache->loaded)) { +if (cpu_rcache->loaded && !iova_magazine_empty (cpu_rcache->loaded)) { has_pfn = true; - }else if (!iova_magazine_empty (cpu_rcache->prev)) { + }else if (cpu_rcache->prev && !iova_magazine_empty (cpu_rcache->prev)) {swap (cpu_rcache->prev, cpu_rcache->loaded); has_pfn = true; }else {
Quoted 2 lines · Logan Odell
drivers/iommu/iova.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
Applied, thanks.