iommu: refactor flush queues into iommu-dma
Subsystems (5)
Robin Murphy
maintainerv3 cover iommu: refactor flush queues into iommu-dma
v1: https://lore.kernel.org/linux-iommu/[email protected]/
v2: https://lore.kernel.org/linux-iommu/[email protected]/
Hi all,
Just another quick update addressing the trivial nits and picking up the review tags so far. The previous Tegra DRM fixes for the initial kbuild issues have been picked up, so I've posted a final fix separately[1] for the others subsequently reported on v2. I've confirmed that arm64 allmodconfig builds cleanly with that, and nothing else jumped out from a manual audit of iova.h includers. At worst we could hold off applying the last patch (or the last two, logically), or temporarily reinstate the dma-iommu.h include, if we're worried about issues in linux-next until the DRM tree has caught up.
Thanks
Robin.
Matthew Wilcox (Oracle) (2):
iommu/amd: Use put_pages_list
iommu/vt-d: Use put_pages_list
Robin Murphy (6):
iommu/iova: Squash entry_dtor abstraction iommu/iova: Squash flush_cb abstraction iommu/amd: Simplify pagetable freeing iommu/iova: Consolidate flush queue code iommu/iova: Move flush queue code to iommu-dma
iommu: Move flush queue data into iommu_dma_cookie
Xiongfeng Wang (1):
iommu/iova: Fix race between FQ timeout and teardown
drivers/iommu/amd/io_pgtable.c | 120 ++++++--------- drivers/iommu/dma-iommu.c | 266 +++++++++++++++++++++++++++------ drivers/iommu/intel/iommu.c | 89 ++++------- drivers/iommu/iova.c | 200 ------------------------- include/linux/iommu.h | 3 +- include/linux/iova.h | 69 +-------- 6 files changed, 297 insertions(+), 450 deletions(-)
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 1/9 iommu/iova: Fix race between FQ timeout and teardown
From: Xiongfeng Wang <[email protected]>
It turns out to be possible for hotplugging out a device to reach the stage of tearing down the device's group and default domain before the domain's flush queue has drained naturally. At this point, it is then possible for the timeout to expire just before the del_timer() call in free_iova_flush_queue(), such that we then proceed to free the FQ resources while fq_flush_timeout() is still accessing them on another CPU. Crashes due to this have been observed in the wild while removing NVMe devices.
Close the race window by using del_timer_sync() to safely wait for any active timeout handler to finish before we start to free things. We already avoid any locking in free_iova_flush_queue() since the FQ is supposed to be inactive anyway, so the potential deadlock scenario does not apply.
--- drivers/iommu/iova.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
drivers/iommu/iova.c +1 -2
@@ -83,8 +83,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad)if (!has_iova_flush_queue (iovad))return ; -if (timer_pending (&iovad->fq_timer)) -del_timer (&iovad->fq_timer); +del_timer_sync (&iovad->fq_timer);fq_destroy_all_entries (iovad);
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 2/9 iommu/iova: Squash entry_dtor abstraction
In reply to Robin MurphyAll flush queues are driven by iommu-dma now, so there is no need to abstract entry_dtor or its data any more. Squash the now-canonical implementation directly into the IOVA code to get it out of the way.
--- drivers/iommu/dma-iommu.c | 17 ++--------------- drivers/iommu/iova.c | 28 +++++++++++++++------------- include/linux/iova.h | 26 +++----------------------- 3 files changed, 20 insertions(+), 51 deletions(-)
drivers/iommu/dma-iommu.c +2 -15
@@ -64,18 +64,6 @@ static int __init iommu_dma_forcedac_setup(char *str) }early_param ("iommu.forcedac" , iommu_dma_forcedac_setup); -static void iommu_dma_entry_dtor (unsigned long data) -{ -struct page *freelist = (struct page *)data; - -while (freelist) { -unsigned long p = (unsigned long )page_address (freelist); - - freelist = freelist->freelist; -free_page (p); - } -} -static inline size_t cookie_msi_granule (struct iommu_dma_cookie *cookie) {if (cookie->type ==IOMMU_DMA_IOVA_COOKIE )
@@ -324,8 +312,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain)if (cookie->fq_domain)return 0 ; - ret =init_iova_flush_queue (&cookie->iovad, iommu_dma_flush_iotlb_all, - iommu_dma_entry_dtor); + ret =init_iova_flush_queue (&cookie->iovad, iommu_dma_flush_iotlb_all);if (ret) {pr_warn ("iova flush queue initialization failed\n" );return ret;
@@ -479,7 +466,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,else if (gather && gather->queued)queue_iova (iovad,iova_pfn (iovad, iova), size >>iova_shift (iovad), - (unsigned long )gather->freelist); + gather->freelist);else free_iova_fast (iovad,iova_pfn (iovad, iova), size >>iova_shift (iovad));
drivers/iommu/iova.c +15 -13
@@ -91,11 +91,9 @@ static void free_iova_flush_queue(struct iova_domain *iovad) iovad->fq =NULL ; iovad->flush_cb =NULL ; - iovad->entry_dtor =NULL ; } -int init_iova_flush_queue (struct iova_domain *iovad, -iova_flush_cb flush_cb,iova_entry_dtor entry_dtor) +int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb) {struct iova_fq __percpu *queue;int cpu;
@@ -108,7 +106,6 @@ int init_iova_flush_queue(struct iova_domain *iovad,return -ENOMEM ; iovad->flush_cb = flush_cb; - iovad->entry_dtor = entry_dtor;for_each_possible_cpu (cpu) {struct iova_fq *fq;
@@ -538,6 +535,16 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) }EXPORT_SYMBOL_GPL (free_iova_fast); +static void fq_entry_dtor (struct page *freelist) +{ +while (freelist) { +unsigned long p = (unsigned long )page_address (freelist); + + freelist = freelist->freelist; +free_page (p); + } +} +#define fq_ring_for_each (i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
@@ -570,9 +577,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)if (fq->entries[idx].counter >= counter)break ; -if (iovad->entry_dtor) - iovad->entry_dtor (fq->entries[idx].data); - +fq_entry_dtor (fq->entries[idx].freelist);free_iova_fast (iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages);
@@ -597,15 +602,12 @@ static void fq_destroy_all_entries(struct iova_domain *iovad)* bother to free iovas, just call the entry_dtor on all remaining * entries. */ -if (!iovad->entry_dtor) -return ; -for_each_possible_cpu (cpu) {struct iova_fq *fq =per_cpu_ptr (iovad->fq, cpu);int idx;fq_ring_for_each (idx, fq) - iovad->entry_dtor (fq->entries[idx].data); +fq_entry_dtor (fq->entries[idx].freelist); } }
@@ -630,7 +632,7 @@ static void fq_flush_timeout(struct timer_list *t)void queue_iova (struct iova_domain *iovad,unsigned long pfn,unsigned long pages, -unsigned long data) +struct page *freelist) {struct iova_fq *fq;unsigned long flags;
@@ -664,7 +666,7 @@ void queue_iova(struct iova_domain *iovad, fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].data = data; + fq->entries[idx].freelist = freelist; fq->entries[idx].counter =atomic64_read (&iovad->fq_flush_start_cnt);spin_unlock_irqrestore (&fq->lock, flags);
include/linux/iova.h +3 -23
@@ -40,9 +40,6 @@ struct iova_domain;/* Call-Back from IOVA code into IOMMU drivers */ typedef void (*iova_flush_cb )(struct iova_domain *domain); -/* Destructor for per-entry data */ -typedef void (*iova_entry_dtor )(unsigned long data); -/* Number of entries per Flush Queue */ #define IOVA_FQ_SIZE 256
@@ -53,7 +50,7 @@ typedef void (* iova_entry_dtor)(unsigned long data);struct iova_fq_entry {unsigned long iova_pfn;unsigned long pages; -unsigned long data; +struct page *freelist;u64 counter;/* Flush counter when this entrie was added */ };
@@ -88,9 +85,6 @@ struct iova_domain {iova_flush_cb flush_cb;/* Call-Back function to flush IOMMU TLBs */ -iova_entry_dtor entry_dtor;/* IOMMU driver specific destructor for -iova entry */ -struct timer_list fq_timer;/* Timer to regularily empty the flush-queues */ atomic_t fq_timer_on;/* 1 when timer is active, 0
@@ -146,15 +140,14 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size);void queue_iova (struct iova_domain *iovad,unsigned long pfn,unsigned long pages, -unsigned long data); +struct page *freelist);unsigned long alloc_iova_fast (struct iova_domain *iovad,unsigned long size,unsigned long limit_pfn,bool flush_rcache);struct iova *reserve_iova (struct iova_domain *iovad,unsigned long pfn_lo,unsigned long pfn_hi);void init_iova_domain (struct iova_domain *iovad,unsigned long granule,unsigned long start_pfn); -int init_iova_flush_queue (struct iova_domain *iovad, -iova_flush_cb flush_cb,iova_entry_dtor entry_dtor); +int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb);struct iova *find_iova (struct iova_domain *iovad,unsigned long pfn);void put_iova_domain (struct iova_domain *iovad);#else
@@ -189,12 +182,6 @@ static inline void free_iova_fast(struct iova_domain *iovad, { } -static inline void queue_iova (struct iova_domain *iovad, -unsigned long pfn,unsigned long pages, -unsigned long data) -{ -} - static inline unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn,
@@ -216,13 +203,6 @@ static inline void init_iova_domain(struct iova_domain *iovad, { } -static inline int init_iova_flush_queue (struct iova_domain *iovad, -iova_flush_cb flush_cb, -iova_entry_dtor entry_dtor) -{ -return -ENODEV ; -} -static inline struct iova *find_iova (struct iova_domain *iovad,unsigned long pfn) {
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 3/9 iommu/iova: Squash flush_cb abstraction
In reply to Robin MurphyOnce again, with iommu-dma now being the only flush queue user, we no longer need the extra level of indirection through flush_cb. Squash that and let the flush queue code call the domain method directly. This does mean temporarily having to carry an additional copy of the IOMMU domain pointer around instead, but only until a later patch untangles it again.
--- drivers/iommu/dma-iommu.c | 13 +------------ drivers/iommu/iova.c | 11 +++++------ include/linux/iova.h | 11 +++-------- 3 files changed, 9 insertions(+), 26 deletions(-)
drivers/iommu/dma-iommu.c +1 -12
@@ -282,17 +282,6 @@ static int iova_reserve_iommu_regions(struct device *dev,return ret; } -static void iommu_dma_flush_iotlb_all (struct iova_domain *iovad) -{ -struct iommu_dma_cookie *cookie; -struct iommu_domain *domain; - - cookie = container_of(iovad, struct iommu_dma_cookie, iovad); - domain = cookie->fq_domain; - - domain->ops->flush_iotlb_all (domain); -} -static bool dev_is_untrusted (struct device *dev) {return dev_is_pci (dev) &&to_pci_dev (dev)->untrusted;
@@ -312,7 +301,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain)if (cookie->fq_domain)return 0 ; - ret =init_iova_flush_queue (&cookie->iovad, iommu_dma_flush_iotlb_all); + ret =init_iova_flush_queue (&cookie->iovad, domain);if (ret) {pr_warn ("iova flush queue initialization failed\n" );return ret;
drivers/iommu/iova.c +5 -6
@@ -63,7 +63,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn =1UL << (32 -iova_shift (iovad)); iovad->max32_alloc_size = iovad->dma_32bit_pfn; - iovad->flush_cb =NULL ; + iovad->fq_domain =NULL ; iovad->fq =NULL ; iovad->anchor.pfn_lo = iovad->anchor.pfn_hi =IOVA_ANCHOR ;rb_link_node (&iovad->anchor.node,NULL , &iovad->rbroot.rb_node);
@@ -90,10 +90,10 @@ static void free_iova_flush_queue(struct iova_domain *iovad)free_percpu (iovad->fq); iovad->fq =NULL ; - iovad->flush_cb =NULL ; + iovad->fq_domain =NULL ; } -int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb) +int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain) {struct iova_fq __percpu *queue;int cpu;
@@ -105,8 +105,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb)if (!queue)return -ENOMEM ; - iovad->flush_cb = flush_cb; -for_each_possible_cpu (cpu) {struct iova_fq *fq;
@@ -117,6 +115,7 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb)spin_lock_init (&fq->lock); } + iovad->fq_domain = fq_domain; iovad->fq = queue;timer_setup (&iovad->fq_timer, fq_flush_timeout,0 );
@@ -589,7 +588,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)static void iova_domain_flush (struct iova_domain *iovad) {atomic64_inc (&iovad->fq_flush_start_cnt); - iovad->flush_cb (iovad); + iovad->fq_domain->ops->flush_iotlb_all (iovad->fq_domain);atomic64_inc (&iovad->fq_flush_finish_cnt); }
include/linux/iova.h +3 -8
@@ -14,6 +14,7 @@#include <linux/rbtree.h> #include <linux/atomic.h> #include <linux/dma-mapping.h> +#include <linux/iommu.h> /* iova structure */ struct iova {
@@ -35,11 +36,6 @@ struct iova_rcache {struct iova_cpu_rcache __percpu *cpu_rcaches; }; -struct iova_domain ; - -/* Call-Back from IOVA code into IOMMU drivers */ -typedef void (*iova_flush_cb )(struct iova_domain *domain); -/* Number of entries per Flush Queue */ #define IOVA_FQ_SIZE 256
@@ -82,8 +78,7 @@ struct iova_domain {struct iova anchor;/* rbtree lookup anchor */ struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE ];/* IOVA range caches */ -iova_flush_cb flush_cb;/* Call-Back function to flush IOMMU -TLBs */ +struct iommu_domain *fq_domain;struct timer_list fq_timer;/* Timer to regularily empty the flush-queues */
@@ -147,7 +142,7 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi);void init_iova_domain (struct iova_domain *iovad,unsigned long granule,unsigned long start_pfn); -int init_iova_flush_queue (struct iova_domain *iovad,iova_flush_cb flush_cb); +int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain);struct iova *find_iova (struct iova_domain *iovad,unsigned long pfn);void put_iova_domain (struct iova_domain *iovad);#else
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 4/9 iommu/amd: Simplify pagetable freeing
In reply to Robin MurphyFor reasons unclear, pagetable freeing is an effectively recursive method implemented via an elaborate system of templated functions that turns out to account for 25% of the object file size. Implementing it using regular straightforward recursion makes the code simpler, and seems like a good thing to do before we work on it further. As part of that, also fix the types to avoid all the needless casting back and forth which just gets in the way.
--- drivers/iommu/amd/io_pgtable.c | 82 ++++++++++++++-------------------- 1 file changed, 34 insertions(+), 48 deletions(-)
drivers/iommu/amd/io_pgtable.c +34 -48
@@ -84,49 +84,45 @@ static void free_page_list(struct page *freelist) } } -static struct page *free_pt_page (unsigned long pt,struct page *freelist) +static struct page *free_pt_page (u64 *pt,struct page *freelist) { -struct page *p =virt_to_page ((void *)pt); +struct page *p =virt_to_page (pt); p->freelist = freelist;return p; } -#define DEFINE_FREE_PT_FN (LVL ,FN ) \ -static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \ -{ \ - unsigned long p; \ - u64 *pt; \ - int i; \ - \ - pt = (u64 *)__pt; \ - \ - for (i = 0; i < 512; ++i) { \ -/* PTE present? */ \ -if (!IOMMU_PTE_PRESENT (pt[i])) \ -continue ; \ - \ -/* Large PTE? */ \ -if (PM_PTE_LEVEL (pt[i]) ==0 || \ -PM_PTE_LEVEL (pt[i]) ==7 ) \ -continue ; \ - \ - p = (unsigned long )IOMMU_PTE_PAGE (pt[i]); \ - freelist =FN (p, freelist); \ - } \ - \ -return free_pt_page ((unsigned long )pt, freelist); \ +static struct page *free_pt_lvl (u64 *pt,struct page *freelist,int lvl) +{ +u64 *p; +int i; + +for (i =0 ; i <512 ; ++i) { +/* PTE present? */ +if (!IOMMU_PTE_PRESENT (pt[i])) +continue ; + +/* Large PTE? */ +if (PM_PTE_LEVEL (pt[i]) ==0 || +PM_PTE_LEVEL (pt[i]) ==7 ) +continue ; + +/* +* Free the next level. No need to look at l1 tables here since +* they can only contain leaf PTEs; just free them directly. +*/ + p =IOMMU_PTE_PAGE (pt[i]); +if (lvl >2 ) + freelist =free_pt_lvl (p, freelist, lvl -1 ); +else + freelist =free_pt_page (p, freelist); + } + +return free_pt_page (pt, freelist); } -DEFINE_FREE_PT_FN(l2, free_pt_page) -DEFINE_FREE_PT_FN(l3, free_pt_l2) -DEFINE_FREE_PT_FN(l4, free_pt_l3) -DEFINE_FREE_PT_FN(l5, free_pt_l4) -DEFINE_FREE_PT_FN(l6, free_pt_l5) - -static struct page *free_sub_pt (unsigned long root,int mode, -struct page *freelist) +static struct page *free_sub_pt (u64 *root,int mode,struct page *freelist) {switch (mode) {case PAGE_MODE_NONE :
@@ -136,19 +132,11 @@ static struct page *free_sub_pt(unsigned long root, int mode, freelist =free_pt_page (root, freelist);break ;case PAGE_MODE_2_LEVEL : - freelist =free_pt_l2 (root, freelist); -break ;case PAGE_MODE_3_LEVEL : - freelist =free_pt_l3 (root, freelist); -break ;case PAGE_MODE_4_LEVEL : - freelist =free_pt_l4 (root, freelist); -break ;case PAGE_MODE_5_LEVEL : - freelist =free_pt_l5 (root, freelist); -break ;case PAGE_MODE_6_LEVEL : - freelist =free_pt_l6 (root, freelist); +free_pt_lvl (root, freelist, mode);break ;default :BUG ();
@@ -364,7 +352,7 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable,static struct page *free_clear_pte (u64 *pte,u64 pteval,struct page *freelist) { -unsigned long pt; +u64 *pt;int mode;while (cmpxchg64 (pte, pteval,0 ) != pteval) {
@@ -375,7 +363,7 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist)if (!IOMMU_PTE_PRESENT (pteval))return freelist; - pt = (unsigned long )IOMMU_PTE_PAGE (pteval); + pt =IOMMU_PTE_PAGE (pteval); mode =IOMMU_PTE_MODE (pteval);return free_sub_pt (pt, mode, freelist);
@@ -512,7 +500,6 @@ static void v1_free_pgtable(struct io_pgtable *iop) struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop);struct protection_domain *dom;struct page *freelist =NULL ; -unsigned long root;if (pgtable->mode ==PAGE_MODE_NONE )return ;
@@ -529,8 +516,7 @@ static void v1_free_pgtable(struct io_pgtable *iop)BUG_ON (pgtable->mode <PAGE_MODE_NONE || pgtable->mode >PAGE_MODE_6_LEVEL ); - root = (unsigned long )pgtable->root; - freelist =free_sub_pt (root, pgtable->mode, freelist); + freelist =free_sub_pt (pgtable->root, pgtable->mode, freelist);free_page_list (freelist); }
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 5/9 iommu/amd: Use put_pages_list
In reply to Robin MurphyFrom: "Matthew Wilcox (Oracle)" <[email protected]>
page->freelist is for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of a list_head and for the pages to be chained together through page->lru. Switch the AMD IOMMU code over to using free_pages_list().
--- drivers/iommu/amd/io_pgtable.c | 50 ++++++++++++---------------------- 1 file changed, 18 insertions(+), 32 deletions(-)
drivers/iommu/amd/io_pgtable.c +18 -32
@@ -74,26 +74,14 @@ static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,* ****************************************************************************/ -static void free_page_list (struct page *freelist) -{ -while (freelist !=NULL ) { -unsigned long p = (unsigned long )page_address (freelist); - - freelist = freelist->freelist; -free_page (p); - } -} - -static struct page *free_pt_page (u64 *pt,struct page *freelist) +static void free_pt_page (u64 *pt,struct list_head *freelist) {struct page *p =virt_to_page (pt); - p->freelist = freelist; - -return p; +list_add_tail (&p->lru, freelist); } -static struct page *free_pt_lvl (u64 *pt,struct page *freelist,int lvl) +static void free_pt_lvl (u64 *pt,struct list_head *freelist,int lvl) {u64 *p;int i;
@@ -114,22 +102,22 @@ static struct page *free_pt_lvl(u64 *pt, struct page *freelist, int lvl)*/ p =IOMMU_PTE_PAGE (pt[i]);if (lvl >2 ) - freelist =free_pt_lvl (p, freelist, lvl -1 ); +free_pt_lvl (p, freelist, lvl -1 );else - freelist =free_pt_page (p, freelist); +free_pt_page (p, freelist); } -return free_pt_page (pt, freelist); +free_pt_page (pt, freelist); } -static struct page *free_sub_pt (u64 *root,int mode,struct page *freelist) +static void free_sub_pt (u64 *root,int mode,struct list_head *freelist) {switch (mode) {case PAGE_MODE_NONE :case PAGE_MODE_7_LEVEL :break ;case PAGE_MODE_1_LEVEL : - freelist =free_pt_page (root, freelist); +free_pt_page (root, freelist);break ;case PAGE_MODE_2_LEVEL :case PAGE_MODE_3_LEVEL :
@@ -141,8 +129,6 @@ static struct page *free_sub_pt(u64 *root, int mode, struct page *freelist)default :BUG (); } - -return freelist; } void amd_iommu_domain_set_pgtable(struct protection_domain *domain,
@@ -350,7 +336,7 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable,return pte; } -static struct page *free_clear_pte (u64 *pte,u64 pteval,struct page *freelist) +static void free_clear_pte (u64 *pte,u64 pteval,struct list_head *freelist) {u64 *pt;int mode;
@@ -361,12 +347,12 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) }if (!IOMMU_PTE_PRESENT (pteval)) -return freelist; +return ; pt =IOMMU_PTE_PAGE (pteval); mode =IOMMU_PTE_MODE (pteval); -return free_sub_pt (pt, mode, freelist); +free_sub_pt (pt, mode, freelist); }/*
@@ -380,7 +366,7 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct protection_domain *dom = io_pgtable_ops_to_domain(ops); -struct page *freelist =NULL ; +LIST_HEAD (freelist);bool updated = false;u64 __pte, *pte;int ret, i, count;
@@ -400,9 +386,9 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, goto out;for (i =0 ; i < count; ++i) - freelist =free_clear_pte (&pte[i], pte[i], freelist); +free_clear_pte (&pte[i], pte[i], &freelist); -if (freelist !=NULL ) +if (!list_empty (&freelist)) updated = true;if (count >1 ) {
@@ -437,7 +423,7 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, }/* Everything flushed out, free pages now */ -free_page_list (freelist); +put_pages_list (&freelist);return ret; }
@@ -499,7 +485,7 @@ static void v1_free_pgtable(struct io_pgtable *iop) { struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop);struct protection_domain *dom; -struct page *freelist =NULL ; +LIST_HEAD (freelist);if (pgtable->mode ==PAGE_MODE_NONE )return ;
@@ -516,9 +502,9 @@ static void v1_free_pgtable(struct io_pgtable *iop)BUG_ON (pgtable->mode <PAGE_MODE_NONE || pgtable->mode >PAGE_MODE_6_LEVEL ); - freelist =free_sub_pt (pgtable->root, pgtable->mode, freelist); +free_sub_pt (pgtable->root, pgtable->mode, &freelist); -free_page_list (freelist); +put_pages_list (&freelist); }static struct io_pgtable *v1_alloc_pgtable (struct io_pgtable_cfg *cfg,void *cookie)
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 6/9 iommu/vt-d: Use put_pages_list
In reply to Robin MurphyFrom: "Matthew Wilcox (Oracle)" <[email protected]>
page->freelist is for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of a list_head and for the pages to be chained together through page->lru. Switch the Intel IOMMU and IOVA code over to using free_pages_list().
--- drivers/iommu/dma-iommu.c | 2 +- drivers/iommu/intel/iommu.c | 89 +++++++++++++------------------------ drivers/iommu/iova.c | 26 ++++------- include/linux/iommu.h | 3 +- include/linux/iova.h | 4 +- 5 files changed, 45 insertions(+), 79 deletions(-)
drivers/iommu/dma-iommu.c +1 -1
@@ -455,7 +455,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,else if (gather && gather->queued)queue_iova (iovad,iova_pfn (iovad, iova), size >>iova_shift (iovad), - gather->freelist); + &gather->freelist);else free_iova_fast (iovad,iova_pfn (iovad, iova), size >>iova_shift (iovad));
drivers/iommu/intel/iommu.c +31 -58
177 lines not on this page · raw message
drivers/iommu/iova.c +9 -17
@@ -96,7 +96,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad)int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain) {struct iova_fq __percpu *queue; -int cpu; +int i, cpu;atomic64_set (&iovad->fq_flush_start_cnt,0 );atomic64_set (&iovad->fq_flush_finish_cnt,0 );
@@ -113,6 +113,9 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_dom fq->tail =0 ;spin_lock_init (&fq->lock); + +for (i =0 ; i <IOVA_FQ_SIZE ; i++) +INIT_LIST_HEAD (&fq->entries[i].freelist); } iovad->fq_domain = fq_domain;
@@ -534,16 +537,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) }EXPORT_SYMBOL_GPL (free_iova_fast); -static void fq_entry_dtor (struct page *freelist) -{ -while (freelist) { -unsigned long p = (unsigned long )page_address (freelist); - - freelist = freelist->freelist; -free_page (p); - } -} -#define fq_ring_for_each (i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
@@ -576,7 +569,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)if (fq->entries[idx].counter >= counter)break ; -fq_entry_dtor (fq->entries[idx].freelist); +put_pages_list (&fq->entries[idx].freelist);free_iova_fast (iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages);
@@ -598,15 +591,14 @@ static void fq_destroy_all_entries(struct iova_domain *iovad)/* * This code runs when the iova_domain is being detroyed, so don't -* bother to free iovas, just call the entry_dtor on all remaining -* entries. +* bother to free iovas, just free any remaining pagetable pages. */ for_each_possible_cpu (cpu) {struct iova_fq *fq =per_cpu_ptr (iovad->fq, cpu);int idx;fq_ring_for_each (idx, fq) -fq_entry_dtor (fq->entries[idx].freelist); +put_pages_list (&fq->entries[idx].freelist); } }
@@ -631,7 +623,7 @@ static void fq_flush_timeout(struct timer_list *t)void queue_iova (struct iova_domain *iovad,unsigned long pfn,unsigned long pages, -struct page *freelist) +struct list_head *freelist) {struct iova_fq *fq;unsigned long flags;
@@ -665,8 +657,8 @@ void queue_iova(struct iova_domain *iovad, fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].freelist = freelist; fq->entries[idx].counter =atomic64_read (&iovad->fq_flush_start_cnt); +list_splice (freelist, &fq->entries[idx].freelist);spin_unlock_irqrestore (&fq->lock, flags);
include/linux/iommu.h +2 -1
@@ -186,7 +186,7 @@ struct iommu_iotlb_gather {unsigned long start;unsigned long end;size_t pgsize; -struct page *freelist; +struct list_head freelist;bool queued; };
@@ -399,6 +399,7 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) { *gather = (struct iommu_iotlb_gather ) { .start =ULONG_MAX , + .freelist =LIST_HEAD_INIT (gather->freelist), }; }
include/linux/iova.h +2 -2
@@ -46,7 +46,7 @@ struct iova_rcache {struct iova_fq_entry {unsigned long iova_pfn;unsigned long pages; -struct page *freelist; +struct list_head freelist;u64 counter;/* Flush counter when this entrie was added */ };
@@ -135,7 +135,7 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size);void queue_iova (struct iova_domain *iovad,unsigned long pfn,unsigned long pages, -struct page *freelist); +struct list_head *freelist);unsigned long alloc_iova_fast (struct iova_domain *iovad,unsigned long size,unsigned long limit_pfn,bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH v3 6/9] iommu/vt-d: Use put_pages_list
Hi Robin,
Quoted 10 lines · Robin Murphy
From: "Matthew Wilcox (Oracle)"<[email protected]>
page->freelistis for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of alist_headand for the pages to be chained together throughpage->lru. Switch the Intel IOMMU and IOVA code over to usingfree_pages_list().Signed-off-by Matthew Wilcox (Oracle)<[email protected]>[rm: split from original patch, cosmetic tweaks, fix fq entries]Signed-off-by Robin Murphy<[email protected]>
I had boot tests on my development machines with both strict and lazy flushing modes. I didn't see any regression, the disk i/o and network i/o worked good for me.
For changes in iommu/vt-d:
Best regards,
baolu
Robin Murphy
maintainerPATCH v3 7/9 iommu/iova: Consolidate flush queue code
In reply to Robin MurphySquash and simplify some of the freeing code, and move the init and free routines down into the rest of the flush queue code to obviate the forward declarations.
--- drivers/iommu/iova.c | 131 +++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 73 deletions(-)
drivers/iommu/iova.c +58 -73
@@ -24,8 +24,6 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,static void init_iova_rcaches (struct iova_domain *iovad);static void free_cpu_cached_iovas (unsigned int cpu,struct iova_domain *iovad);static void free_iova_rcaches (struct iova_domain *iovad); -static void fq_destroy_all_entries (struct iova_domain *iovad); -static void fq_flush_timeout (struct timer_list *t);static int iova_cpuhp_dead (unsigned int cpu,struct hlist_node *node) {
@@ -73,60 +71,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, }EXPORT_SYMBOL_GPL (init_iova_domain); -static bool has_iova_flush_queue (struct iova_domain *iovad) -{ -return !!iovad->fq; -} - -static void free_iova_flush_queue (struct iova_domain *iovad) -{ -if (!has_iova_flush_queue (iovad)) -return ; - -del_timer_sync (&iovad->fq_timer); - -fq_destroy_all_entries (iovad); - -free_percpu (iovad->fq); - - iovad->fq =NULL ; - iovad->fq_domain =NULL ; -} - -int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain) -{ -struct iova_fq __percpu *queue; -int i, cpu; - -atomic64_set (&iovad->fq_flush_start_cnt,0 ); -atomic64_set (&iovad->fq_flush_finish_cnt,0 ); - - queue = alloc_percpu(struct iova_fq); -if (!queue) -return -ENOMEM ; - -for_each_possible_cpu (cpu) { -struct iova_fq *fq; - - fq =per_cpu_ptr (queue, cpu); - fq->head =0 ; - fq->tail =0 ; - -spin_lock_init (&fq->lock); - -for (i =0 ; i <IOVA_FQ_SIZE ; i++) -INIT_LIST_HEAD (&fq->entries[i].freelist); - } - - iovad->fq_domain = fq_domain; - iovad->fq = queue; - -timer_setup (&iovad->fq_timer, fq_flush_timeout,0 ); -atomic_set (&iovad->fq_timer_on,0 ); - -return 0 ; -} -static struct rb_node *__get_cached_rbnode (struct iova_domain *iovad,unsigned long limit_pfn) {
@@ -585,23 +529,6 @@ static void iova_domain_flush(struct iova_domain *iovad)atomic64_inc (&iovad->fq_flush_finish_cnt); } -static void fq_destroy_all_entries (struct iova_domain *iovad) -{ -int cpu; - -/* -* This code runs when the iova_domain is being detroyed, so don't -* bother to free iovas, just free any remaining pagetable pages. -*/ -for_each_possible_cpu (cpu) { -struct iova_fq *fq =per_cpu_ptr (iovad->fq, cpu); -int idx; - -fq_ring_for_each (idx, fq) -put_pages_list (&fq->entries[idx].freelist); - } -} -static void fq_flush_timeout (struct timer_list *t) {struct iova_domain *iovad =from_timer (iovad, t, fq_timer);
@@ -669,6 +596,64 @@ void queue_iova(struct iova_domain *iovad, jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); } +static void free_iova_flush_queue (struct iova_domain *iovad) +{ +int cpu, idx; + +if (!iovad->fq) +return ; + +del_timer_sync (&iovad->fq_timer); +/* +* This code runs when the iova_domain is being detroyed, so don't +* bother to free iovas, just free any remaining pagetable pages. +*/ +for_each_possible_cpu (cpu) { +struct iova_fq *fq =per_cpu_ptr (iovad->fq, cpu); + +fq_ring_for_each (idx, fq) +put_pages_list (&fq->entries[idx].freelist); + } + +free_percpu (iovad->fq); + + iovad->fq =NULL ; + iovad->fq_domain =NULL ; +} + +int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain) +{ +struct iova_fq __percpu *queue; +int i, cpu; + +atomic64_set (&iovad->fq_flush_start_cnt,0 ); +atomic64_set (&iovad->fq_flush_finish_cnt,0 ); + + queue = alloc_percpu(struct iova_fq); +if (!queue) +return -ENOMEM ; + +for_each_possible_cpu (cpu) { +struct iova_fq *fq =per_cpu_ptr (queue, cpu); + + fq->head =0 ; + fq->tail =0 ; + +spin_lock_init (&fq->lock); + +for (i =0 ; i <IOVA_FQ_SIZE ; i++) +INIT_LIST_HEAD (&fq->entries[i].freelist); + } + + iovad->fq_domain = fq_domain; + iovad->fq = queue; + +timer_setup (&iovad->fq_timer, fq_flush_timeout,0 ); +atomic_set (&iovad->fq_timer_on,0 ); + +return 0 ; +} +/** * put_iova_domain - destroys the iova domain * @iovad: - iova domain in question.
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 8/9 iommu/iova: Move flush queue code to iommu-dma
In reply to Robin MurphyFlush queues are specific to DMA ops, which are now handled exclusively by iommu-dma. As such, now that the historical artefacts from being shared directly with drivers have been cleaned up, move the flush queue code into iommu-dma itself to get it out of the way of other IOVA users.
This is pure code movement with no functional change; refactoring to clean up the headers and definitions will follow.
--- drivers/iommu/dma-iommu.c | 177 +++++++++++++++++++++++++++++++++++++- drivers/iommu/iova.c | 175 ------------------------------------- 2 files changed, 176 insertions(+), 176 deletions(-)
drivers/iommu/dma-iommu.c +176 -1
@@ -64,6 +64,179 @@ static int __init iommu_dma_forcedac_setup(char *str) }early_param ("iommu.forcedac" , iommu_dma_forcedac_setup); +#define fq_ring_for_each (i, fq) \ + for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) + +static inline bool fq_full (struct iova_fq *fq) +{ +assert_spin_locked (&fq->lock); +return (((fq->tail +1 ) %IOVA_FQ_SIZE ) == fq->head); +} + +static inline unsigned fq_ring_add (struct iova_fq *fq) +{ +unsigned idx = fq->tail; + +assert_spin_locked (&fq->lock); + + fq->tail = (idx +1 ) %IOVA_FQ_SIZE ; + +return idx; +} + +static void fq_ring_free (struct iova_domain *iovad,struct iova_fq *fq) +{ +u64 counter =atomic64_read (&iovad->fq_flush_finish_cnt); +unsigned idx; + +assert_spin_locked (&fq->lock); + +fq_ring_for_each (idx, fq) { + +if (fq->entries[idx].counter >= counter) +break ; + +put_pages_list (&fq->entries[idx].freelist); +free_iova_fast (iovad, + fq->entries[idx].iova_pfn, + fq->entries[idx].pages); + + fq->head = (fq->head +1 ) %IOVA_FQ_SIZE ; + } +} + +static void iova_domain_flush (struct iova_domain *iovad) +{ +atomic64_inc (&iovad->fq_flush_start_cnt); + iovad->fq_domain->ops->flush_iotlb_all (iovad->fq_domain); +atomic64_inc (&iovad->fq_flush_finish_cnt); +} + +static void fq_flush_timeout (struct timer_list *t) +{ +struct iova_domain *iovad =from_timer (iovad, t, fq_timer); +int cpu; + +atomic_set (&iovad->fq_timer_on,0 ); +iova_domain_flush (iovad); + +for_each_possible_cpu (cpu) { +unsigned long flags; +struct iova_fq *fq; + + fq =per_cpu_ptr (iovad->fq, cpu); +spin_lock_irqsave (&fq->lock, flags); +fq_ring_free (iovad, fq); +spin_unlock_irqrestore (&fq->lock, flags); + } +} + +void queue_iova (struct iova_domain *iovad, +unsigned long pfn,unsigned long pages, +struct list_head *freelist) +{ +struct iova_fq *fq; +unsigned long flags; +unsigned idx; + +/* +* Order against the IOMMU driver's pagetable update from unmapping +* @pte, to guarantee that iova_domain_flush() observes that if called +* from a different CPU before we release the lock below. Full barrier +* so it also pairs with iommu_dma_init_fq() to avoid seeing partially +* written fq state here. +*/ +smp_mb (); + + fq =raw_cpu_ptr (iovad->fq); +spin_lock_irqsave (&fq->lock, flags); + +/* +* First remove all entries from the flush queue that have already been +* flushed out on another CPU. This makes the fq_full() check below less +* likely to be true. +*/ +fq_ring_free (iovad, fq); + +if (fq_full (fq)) { +iova_domain_flush (iovad); +fq_ring_free (iovad, fq); + } + + idx =fq_ring_add (fq); + + fq->entries[idx].iova_pfn = pfn; + fq->entries[idx].pages = pages; + fq->entries[idx].counter =atomic64_read (&iovad->fq_flush_start_cnt); +list_splice (freelist, &fq->entries[idx].freelist); + +spin_unlock_irqrestore (&fq->lock, flags); + +/* Avoid false sharing as much as possible. */ +if (!atomic_read (&iovad->fq_timer_on) && + !atomic_xchg (&iovad->fq_timer_on,1 )) +mod_timer (&iovad->fq_timer, + jiffies +msecs_to_jiffies (IOVA_FQ_TIMEOUT )); +} + +static void free_iova_flush_queue (struct iova_domain *iovad) +{ +int cpu, idx; + +if (!iovad->fq) +return ; + +del_timer_sync (&iovad->fq_timer); +/* +* This code runs when the iova_domain is being detroyed, so don't +* bother to free iovas, just free any remaining pagetable pages. +*/ +for_each_possible_cpu (cpu) { +struct iova_fq *fq =per_cpu_ptr (iovad->fq, cpu); + +fq_ring_for_each (idx, fq) +put_pages_list (&fq->entries[idx].freelist); + } + +free_percpu (iovad->fq); + + iovad->fq =NULL ; + iovad->fq_domain =NULL ; +} + +int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain) +{ +struct iova_fq __percpu *queue; +int i, cpu; + +atomic64_set (&iovad->fq_flush_start_cnt,0 ); +atomic64_set (&iovad->fq_flush_finish_cnt,0 ); + + queue = alloc_percpu(struct iova_fq); +if (!queue) +return -ENOMEM ; + +for_each_possible_cpu (cpu) { +struct iova_fq *fq =per_cpu_ptr (queue, cpu); + + fq->head =0 ; + fq->tail =0 ; + +spin_lock_init (&fq->lock); + +for (i =0 ; i <IOVA_FQ_SIZE ; i++) +INIT_LIST_HEAD (&fq->entries[i].freelist); + } + + iovad->fq_domain = fq_domain; + iovad->fq = queue; + +timer_setup (&iovad->fq_timer, fq_flush_timeout,0 ); +atomic_set (&iovad->fq_timer_on,0 ); + +return 0 ; +} +static inline size_t cookie_msi_granule (struct iommu_dma_cookie *cookie) {if (cookie->type ==IOMMU_DMA_IOVA_COOKIE )
@@ -144,8 +317,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)if (!cookie)return ; -if (cookie->type ==IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) +if (cookie->type ==IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { +free_iova_flush_queue (&cookie->iovad);put_iova_domain (&cookie->iovad); + }list_for_each_entry_safe (msi, tmp, &cookie->msi_page_list, list) {list_del (&msi->list);
drivers/iommu/iova.c +0 -175
187 lines not on this page · raw message
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Robin Murphy
maintainerPATCH v3 9/9 iommu: Move flush queue data into iommu_dma_cookie
In reply to Robin MurphyComplete the move into iommu-dma by refactoring the flush queues themselves to belong to the DMA cookie rather than the IOVA domain.
The refactoring may as well extend to some minor cosmetic aspects too, to help us stay one step ahead of the style police.
--- drivers/iommu/dma-iommu.c | 171 +++++++++++++++++++++----------------- drivers/iommu/iova.c | 2 - include/linux/iova.h | 44 +--------- 3 files changed, 95 insertions(+), 122 deletions(-)
drivers/iommu/dma-iommu.c +94 -77
@@ -9,9 +9,12 @@*/ #include <linux/acpi_iort.h> +#include <linux/atomic.h> +#include <linux/crash_dump.h> #include <linux/device.h> -#include <linux/dma-map-ops.h> +#include <linux/dma-direct.h> #include <linux/dma-iommu.h> +#include <linux/dma-map-ops.h> #include <linux/gfp.h> #include <linux/huge_mm.h> #include <linux/iommu.h>
@@ -20,11 +23,10 @@#include <linux/mm.h> #include <linux/mutex.h> #include <linux/pci.h> -#include <linux/swiotlb.h> #include <linux/scatterlist.h> +#include <linux/spinlock.h> +#include <linux/swiotlb.h> #include <linux/vmalloc.h> -#include <linux/crash_dump.h> -#include <linux/dma-direct.h> struct iommu_dma_msi_page {struct list_head list;
@@ -41,7 +43,19 @@ struct iommu_dma_cookie {enum iommu_dma_cookie_type type;union {/* Full allocator for IOMMU_DMA_IOVA_COOKIE */ -struct iova_domain iovad; +struct { +struct iova_domain iovad; + +struct iova_fq __percpu *fq;/* Flush queue */ +/* Number of TLB flushes that have been started */ +atomic64_t fq_flush_start_cnt; +/* Number of TLB flushes that have been finished */ +atomic64_t fq_flush_finish_cnt; +/* Timer to regularily empty the flush queues */ +struct timer_list fq_timer; +/* 1 when timer is active, 0 when not */ +atomic_t fq_timer_on; + };/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ dma_addr_t msi_iova; };
@@ -64,6 +78,27 @@ static int __init iommu_dma_forcedac_setup(char *str) }early_param ("iommu.forcedac" , iommu_dma_forcedac_setup); +/* Number of entries per flush queue */ +#define IOVA_FQ_SIZE 256 + +/* Timeout (in ms) after which entries are flushed from the queue */ +#define IOVA_FQ_TIMEOUT 10 + +/* Flush queue entry for deferred flushing */ +struct iova_fq_entry { +unsigned long iova_pfn; +unsigned long pages; +struct list_head freelist; +u64 counter;/* Flush counter when this entry was added */ +}; + +/* Per-CPU flush queue structure */ +struct iova_fq { +struct iova_fq_entry entries[IOVA_FQ_SIZE ]; +unsigned int head, tail; +spinlock_t lock; +}; +#define fq_ring_for_each (i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
@@ -73,9 +108,9 @@ static inline bool fq_full(struct iova_fq *fq)return (((fq->tail +1 ) %IOVA_FQ_SIZE ) == fq->head); } -static inline unsigned fq_ring_add (struct iova_fq *fq) +static inline unsigned int fq_ring_add (struct iova_fq *fq) { -unsigned idx = fq->tail; +unsigned int idx = fq->tail;assert_spin_locked (&fq->lock);
@@ -84,10 +119,10 @@ static inline unsigned fq_ring_add(struct iova_fq *fq)return idx; } -static void fq_ring_free (struct iova_domain *iovad,struct iova_fq *fq) +static void fq_ring_free (struct iommu_dma_cookie *cookie,struct iova_fq *fq) { -u64 counter =atomic64_read (&iovad->fq_flush_finish_cnt); -unsigned idx; +u64 counter =atomic64_read (&cookie->fq_flush_finish_cnt); +unsigned int idx;assert_spin_locked (&fq->lock);
@@ -97,7 +132,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)break ;put_pages_list (&fq->entries[idx].freelist); -free_iova_fast (iovad, +free_iova_fast (&cookie->iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages);
@@ -105,50 +140,50 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) } } -static void iova_domain_flush (struct iova_domain *iovad) +static void fq_flush_iotlb (struct iommu_dma_cookie *cookie) { -atomic64_inc (&iovad->fq_flush_start_cnt); - iovad->fq_domain->ops->flush_iotlb_all (iovad->fq_domain); -atomic64_inc (&iovad->fq_flush_finish_cnt); +atomic64_inc (&cookie->fq_flush_start_cnt); + cookie->fq_domain->ops->flush_iotlb_all (cookie->fq_domain); +atomic64_inc (&cookie->fq_flush_finish_cnt); }static void fq_flush_timeout (struct timer_list *t) { -struct iova_domain *iovad =from_timer (iovad, t, fq_timer); +struct iommu_dma_cookie *cookie =from_timer (cookie, t, fq_timer);int cpu; -atomic_set (&iovad->fq_timer_on,0 ); -iova_domain_flush (iovad); +atomic_set (&cookie->fq_timer_on,0 ); +fq_flush_iotlb (cookie);for_each_possible_cpu (cpu) {unsigned long flags;struct iova_fq *fq; - fq =per_cpu_ptr (iovad->fq, cpu); + fq =per_cpu_ptr (cookie->fq, cpu);spin_lock_irqsave (&fq->lock, flags); -fq_ring_free (iovad, fq); +fq_ring_free (cookie, fq);spin_unlock_irqrestore (&fq->lock, flags); } } -void queue_iova (struct iova_domain *iovad, +static void queue_iova (struct iommu_dma_cookie *cookie,unsigned long pfn,unsigned long pages,struct list_head *freelist) {struct iova_fq *fq;unsigned long flags; -unsigned idx; +unsigned int idx;/* * Order against the IOMMU driver's pagetable update from unmapping -* @pte, to guarantee that iova_domain_flush() observes that if called +* @pte, to guarantee that fq_flush_iotlb() observes that if called * from a different CPU before we release the lock below. Full barrier * so it also pairs with iommu_dma_init_fq() to avoid seeing partially * written fq state here. */ smp_mb (); - fq =raw_cpu_ptr (iovad->fq); + fq =raw_cpu_ptr (cookie->fq);spin_lock_irqsave (&fq->lock, flags);/*
@@ -156,65 +191,66 @@ void queue_iova(struct iova_domain *iovad,* flushed out on another CPU. This makes the fq_full() check below less * likely to be true. */ -fq_ring_free (iovad, fq); +fq_ring_free (cookie, fq);if (fq_full (fq)) { -iova_domain_flush (iovad); -fq_ring_free (iovad, fq); +fq_flush_iotlb (cookie); +fq_ring_free (cookie, fq); } idx =fq_ring_add (fq); fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].counter =atomic64_read (&iovad->fq_flush_start_cnt); + fq->entries[idx].counter =atomic64_read (&cookie->fq_flush_start_cnt);list_splice (freelist, &fq->entries[idx].freelist);spin_unlock_irqrestore (&fq->lock, flags);/* Avoid false sharing as much as possible. */ -if (!atomic_read (&iovad->fq_timer_on) && - !atomic_xchg (&iovad->fq_timer_on,1 )) -mod_timer (&iovad->fq_timer, +if (!atomic_read (&cookie->fq_timer_on) && + !atomic_xchg (&cookie->fq_timer_on,1 )) +mod_timer (&cookie->fq_timer, jiffies +msecs_to_jiffies (IOVA_FQ_TIMEOUT )); } -static void free_iova_flush_queue (struct iova_domain *iovad) +static void iommu_dma_free_fq (struct iommu_dma_cookie *cookie) {int cpu, idx; -if (!iovad->fq) +if (!cookie->fq)return ; -del_timer_sync (&iovad->fq_timer); -/* -* This code runs when the iova_domain is being detroyed, so don't -* bother to free iovas, just free any remaining pagetable pages. -*/ +del_timer_sync (&cookie->fq_timer); +/* The IOVAs will be torn down separately, so just free our queued pages */ for_each_possible_cpu (cpu) { -struct iova_fq *fq =per_cpu_ptr (iovad->fq, cpu); +struct iova_fq *fq =per_cpu_ptr (cookie->fq, cpu);fq_ring_for_each (idx, fq)put_pages_list (&fq->entries[idx].freelist); } -free_percpu (iovad->fq); - - iovad->fq =NULL ; - iovad->fq_domain =NULL ; +free_percpu (cookie->fq); } -int init_iova_flush_queue (struct iova_domain *iovad,struct iommu_domain *fq_domain) +/* sysfs updates are serialised by the mutex of the group owning @domain */ +int iommu_dma_init_fq (struct iommu_domain *domain) { +struct iommu_dma_cookie *cookie = domain->iova_cookie;struct iova_fq __percpu *queue;int i, cpu; -atomic64_set (&iovad->fq_flush_start_cnt,0 ); -atomic64_set (&iovad->fq_flush_finish_cnt,0 ); +if (cookie->fq_domain) +return 0 ; + +atomic64_set (&cookie->fq_flush_start_cnt,0 ); +atomic64_set (&cookie->fq_flush_finish_cnt,0 ); queue = alloc_percpu(struct iova_fq); -if (!queue) +if (!queue) { +pr_warn ("iova flush queue initialization failed\n" );return -ENOMEM ; + }for_each_possible_cpu (cpu) {struct iova_fq *fq =per_cpu_ptr (queue, cpu);
@@ -228,12 +264,16 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domINIT_LIST_HEAD (&fq->entries[i].freelist); } - iovad->fq_domain = fq_domain; - iovad->fq = queue; - -timer_setup (&iovad->fq_timer, fq_flush_timeout,0 ); -atomic_set (&iovad->fq_timer_on,0 ); + cookie->fq = queue; +timer_setup (&cookie->fq_timer, fq_flush_timeout,0 ); +atomic_set (&cookie->fq_timer_on,0 ); +/* +* Prevent incomplete fq state being observable. Pairs with path from +* __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() +*/ +smp_wmb (); +WRITE_ONCE (cookie->fq_domain, domain);return 0 ; }
@@ -318,7 +358,7 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)return ;if (cookie->type ==IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { -free_iova_flush_queue (&cookie->iovad); +iommu_dma_free_fq (cookie);put_iova_domain (&cookie->iovad); }
@@ -467,29 +507,6 @@ static bool dev_use_swiotlb(struct device *dev)return IS_ENABLED (CONFIG_SWIOTLB ) &&dev_is_untrusted (dev); } -/* sysfs updates are serialised by the mutex of the group owning @domain */ -int iommu_dma_init_fq (struct iommu_domain *domain) -{ -struct iommu_dma_cookie *cookie = domain->iova_cookie; -int ret; - -if (cookie->fq_domain) -return 0 ; - - ret =init_iova_flush_queue (&cookie->iovad, domain); -if (ret) { -pr_warn ("iova flush queue initialization failed\n" ); -return ret; - } -/* -* Prevent incomplete iovad->fq being observable. Pairs with path from -* __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() -*/ -smp_wmb (); -WRITE_ONCE (cookie->fq_domain, domain); -return 0 ; -} -/** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
@@ -628,7 +645,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,if (cookie->type ==IOMMU_DMA_MSI_COOKIE ) cookie->msi_iova -= size;else if (gather && gather->queued) -queue_iova (iovad,iova_pfn (iovad, iova), +queue_iova (cookie,iova_pfn (iovad, iova), size >>iova_shift (iovad), &gather->freelist);else
drivers/iommu/iova.c +0 -2
@@ -61,8 +61,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn =1UL << (32 -iova_shift (iovad)); iovad->max32_alloc_size = iovad->dma_32bit_pfn; - iovad->fq_domain =NULL ; - iovad->fq =NULL ; iovad->anchor.pfn_lo = iovad->anchor.pfn_hi =IOVA_ANCHOR ;rb_link_node (&iovad->anchor.node,NULL , &iovad->rbroot.rb_node);rb_insert_color (&iovad->anchor.node, &iovad->rbroot);
include/linux/iova.h +1 -43
76 lines not on this page · raw message
Signature · 6 lines
2.28.0.dirty _______________________________________________ iommu mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/iommu
Quoted 12 lines · Robin Murphy
Matthew Wilcox (Oracle) (2):
iommu/amd: Useput_pages_list
iommu/vt-d: Useput_pages_listRobin Murphy (6):
iommu/iova: Squashentry_dtorabstraction iommu/iova: Squashflush_cbabstraction iommu/amd: Simplify pagetable freeing iommu/iova: Consolidate flush queue code iommu/iova: Move flush queue code to iommu-dma
iommu: Move flush queue data intoiommu_dma_cookie
Quoted 2 lines
Xiongfeng Wang (1):
iommu/iova: Fix race between FQ timeout and teardown
Applied, thanks Robin.