iommu/iova: Move CPU magazine init to first insert
Subsystems (2)
PATCH v2 1/1 iommu/iova: Move CPU magazine init to first insert
From: Logan Odell <[email protected]>
A large amount of memory may be allocated for these magazines on machines with a lot of IOMMU groups and CPU cores. Not all may be used as some devices may be unused or be bound to drivers that do not use the DMA-API. Furthermore, some drivers may not use all levels or CPUs.
Move the initialization of the loaded and prev magazines for each CPU on the first attempt to try to insert a freed IOVA to them.
Signed-off-by Logan Odell <[email protected]>
Signed-off-by Michal Clapinski <[email protected]>
---
v2: only rebase
--- drivers/iommu/iova.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-)
drivers/iommu/iova.c +18 -9
@@ -621,6 +621,9 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)unsigned long flags;int i; +if (!mag) +return ; +spin_lock_irqsave (&iovad->iova_rbtree_lock, flags);for (i =0 ; i < mag->size; ++i) {
@@ -737,14 +740,7 @@ int iova_domain_init_rcaches(struct iova_domain *iovad) }for_each_possible_cpu (cpu) { cpu_rcache =per_cpu_ptr (rcache->cpu_rcaches, cpu); -spin_lock_init (&cpu_rcache->lock); - cpu_rcache->loaded =iova_magazine_alloc (GFP_KERNEL ); - cpu_rcache->prev =iova_magazine_alloc (GFP_KERNEL ); -if (!cpu_rcache->loaded || !cpu_rcache->prev) { - ret = -ENOMEM ; - goto out_err; - } } }
@@ -777,8 +773,20 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, cpu_rcache =raw_cpu_ptr (rcache->cpu_rcaches);spin_lock_irqsave (&cpu_rcache->lock, flags); +if (!cpu_rcache->loaded) { + cpu_rcache->loaded =iova_magazine_alloc (GFP_ATOMIC | __GFP_NOWARN); +if (!cpu_rcache->loaded) + goto unlock; + } +if (!iova_magazine_full (cpu_rcache->loaded)) { can_insert = true; + }else if (!cpu_rcache->prev) { + cpu_rcache->prev =iova_magazine_alloc (GFP_ATOMIC | __GFP_NOWARN); +if (!cpu_rcache->prev) + goto unlock; +swap (cpu_rcache->prev, cpu_rcache->loaded); + can_insert = true; }else if (!iova_magazine_full (cpu_rcache->prev)) {swap (cpu_rcache->prev, cpu_rcache->loaded); can_insert = true;
@@ -799,6 +807,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,if (can_insert)iova_magazine_push (cpu_rcache->loaded, iova_pfn); +unlock:spin_unlock_irqrestore (&cpu_rcache->lock, flags);return can_insert;
@@ -831,9 +840,9 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, cpu_rcache =raw_cpu_ptr (rcache->cpu_rcaches);spin_lock_irqsave (&cpu_rcache->lock, flags); -if (!iova_magazine_empty (cpu_rcache->loaded)) { +if (cpu_rcache->loaded && !iova_magazine_empty (cpu_rcache->loaded)) { has_pfn = true; - }else if (!iova_magazine_empty (cpu_rcache->prev)) { + }else if (cpu_rcache->prev && !iova_magazine_empty (cpu_rcache->prev)) {swap (cpu_rcache->prev, cpu_rcache->loaded); has_pfn = true; }else {
Signature · 1 line
2.55.0.795.g602f6c329a-goog