mm/hmm: let users to tag specific PFNs
Subsystems (11)
PATCH v3 1/26 mm/hmm: let users to tag specific PFNs
From: Leon Romanovsky <[email protected]>
Introduce new sticky flag, which isn't overwritten by HMM range fault. Such flag allows users to tag specific PFNs with extra data in addition to already filled by HMM.
--- include/linux/hmm.h | 3 +++ mm/hmm.c | 34 +++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 13 deletions(-)
include/linux/hmm.h +3 -0
@@ -23,6 +23,7 @@ struct mmu_interval_notifier;* HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID) * HMM_PFN_ERROR - accessing the pfn is impossible and the device should * fail. ie poisoned memory, special pages, no vma, etc +* HMM_PFN_STICKY - Flag preserved on input-to-output transformation * * On input: * 0 - Return the current state of the page, do not fault it.
@@ -36,6 +37,8 @@ enum hmm_pfn_flags {HMM_PFN_VALID =1UL << (BITS_PER_LONG -1 ),HMM_PFN_WRITE =1UL << (BITS_PER_LONG -2 ),HMM_PFN_ERROR =1UL << (BITS_PER_LONG -3 ), +/* Sticky lag, carried from Input to Output */ +HMM_PFN_STICKY =1UL << (BITS_PER_LONG -7 ), HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),/* Input flags */
mm/hmm.c +21 -13
@@ -44,8 +44,10 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end, {unsigned long i = (addr - range->start) >>PAGE_SHIFT ; -for (; addr < end; addr +=PAGE_SIZE , i++) - range->hmm_pfns[i] = cpu_flags; +for (; addr < end; addr +=PAGE_SIZE , i++) { + range->hmm_pfns[i] &=HMM_PFN_STICKY ; + range->hmm_pfns[i] |= cpu_flags; + }return 0 ; }
@@ -202,8 +204,10 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,return hmm_vma_fault (addr, end, required_fault, walk); pfn =pmd_pfn (pmd) + ((addr & ~PMD_MASK ) >>PAGE_SHIFT ); -for (i =0 ; addr < end; addr +=PAGE_SIZE , i++, pfn++) - hmm_pfns[i] = pfn | cpu_flags; +for (i =0 ; addr < end; addr +=PAGE_SIZE , i++, pfn++) { + hmm_pfns[i] &=HMM_PFN_STICKY ; + hmm_pfns[i] |= pfn | cpu_flags; + } return 0; }#else /* CONFIG_TRANSPARENT_HUGEPAGE */
@@ -236,7 +240,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,hmm_pte_need_fault (hmm_vma_walk, pfn_req_flags,0 );if (required_fault) goto fault; - *hmm_pfn =0 ; + *hmm_pfn = *hmm_pfn &HMM_PFN_STICKY ;return 0 ; }
@@ -253,14 +257,14 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, cpu_flags =HMM_PFN_VALID ;if (is_writable_device_private_entry (entry)) cpu_flags |=HMM_PFN_WRITE ; - *hmm_pfn =swp_offset_pfn (entry) | cpu_flags; + *hmm_pfn = (*hmm_pfn &HMM_PFN_STICKY ) |swp_offset_pfn (entry) | cpu_flags;return 0 ; } required_fault =hmm_pte_need_fault (hmm_vma_walk, pfn_req_flags,0 );if (!required_fault) { - *hmm_pfn =0 ; + *hmm_pfn = *hmm_pfn &HMM_PFN_STICKY ;return 0 ; }
@@ -304,11 +308,11 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,pte_unmap (ptep);return -EFAULT ; } - *hmm_pfn =HMM_PFN_ERROR ; + *hmm_pfn = (*hmm_pfn &HMM_PFN_STICKY ) |HMM_PFN_ERROR ;return 0 ; } - *hmm_pfn =pte_pfn (pte) | cpu_flags; + *hmm_pfn = (*hmm_pfn &HMM_PFN_STICKY ) |pte_pfn (pte) | cpu_flags;return 0 ; fault:
@@ -448,8 +452,10 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, } pfn =pud_pfn (pud) + ((addr & ~PUD_MASK ) >>PAGE_SHIFT ); -for (i =0 ; i < npages; ++i, ++pfn) - hmm_pfns[i] = pfn | cpu_flags; +for (i =0 ; i < npages; ++i, ++pfn) { + hmm_pfns[i] &=HMM_PFN_STICKY ; + hmm_pfns[i] |= pfn | cpu_flags; + } goto out_unlock; }
@@ -507,8 +513,10 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, } pfn =pte_pfn (entry) + ((start & ~hmask) >>PAGE_SHIFT ); -for (; addr < end; addr +=PAGE_SIZE , i++, pfn++) - range->hmm_pfns[i] = pfn | cpu_flags; +for (; addr < end; addr +=PAGE_SIZE , i++, pfn++) { + range->hmm_pfns[i] &=HMM_PFN_STICKY ; + range->hmm_pfns[i] |= pfn | cpu_flags; + }spin_unlock (ptl);return 0 ;
Signature · 1 line
2.26.3
PATCH v3 2/26 dma-mapping: provide an interface to allocate IOVA
From: Leon Romanovsky <[email protected]>
Existing .map_page() callback provides two things at the same time:
allocates IOVA and links DMA pages. That combination works great for most of the callers who use it in control paths, but less effective in fast paths.
These advanced callers already manage their data in some sort of database and can perform IOVA allocation in advance, leaving range linkage operation to be in fast path.
Provide an interface to allocate/deallocate IOVA and next patch link/unlink DMA ranges to that specific IOVA.
--- include/linux/dma-map-ops.h | 3 +++ include/linux/dma-mapping.h | 20 ++++++++++++++++++++ kernel/dma/mapping.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+)
include/linux/dma-map-ops.h +3 -0
@@ -86,6 +86,9 @@ struct dma_map_ops {size_t (*max_mapping_size)(struct device *dev);size_t (*opt_mapping_size)(void );unsigned long (*get_merge_boundary)(struct device *dev); + +dma_addr_t (*alloc_iova)(struct device *dev,size_t size); +void (*free_iova)(struct device *dev,dma_addr_t dma_addr,size_t size); };#ifdef CONFIG_DMA_OPS
include/linux/dma-mapping.h +20 -0
@@ -91,6 +91,16 @@ static inline void debug_dma_map_single(struct device *dev, const void *addr, }#endif /* CONFIG_DMA_API_DEBUG */ +struct dma_iova_attrs { +/* OUT field */ +dma_addr_t addr; +/* IN fields */ +struct device *dev; +size_t size; +enum dma_data_direction dir; +unsigned long attrs; +}; +#ifdef CONFIG_HAS_DMA static inline int dma_mapping_error (struct device *dev,dma_addr_t dma_addr) {
@@ -101,6 +111,9 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)return 0 ; } +int dma_alloc_iova (struct dma_iova_attrs *iova); +void dma_free_iova (struct dma_iova_attrs *iova); +dma_addr_t dma_map_page_attrs (struct device *dev,struct page *page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs);
@@ -150,6 +163,13 @@ void dma_vunmap_noncontiguous(struct device *dev, void *vaddr);int dma_mmap_noncontiguous (struct device *dev,struct vm_area_struct *vma,size_t size,struct sg_table *sgt);#else /* CONFIG_HAS_DMA */ +static inline int dma_alloc_iova (struct dma_iova_attrs *iova) +{ +return -EOPNOTSUPP ; +} +static inline void dma_free_iova (struct dma_iova_attrs *iova) +{ +}static inline dma_addr_t dma_map_page_attrs (struct device *dev,struct page *page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs)
kernel/dma/mapping.c +30 -0
@@ -183,6 +183,36 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, }EXPORT_SYMBOL (dma_unmap_page_attrs); +int dma_alloc_iova (struct dma_iova_attrs *iova) +{ +struct device *dev = iova->dev; +const struct dma_map_ops *ops =get_dma_ops (dev); + +if (dma_map_direct (dev, ops) || !ops->alloc_iova) { + iova->addr =0 ; +return 0 ; + } + + iova->addr = ops->alloc_iova (dev, iova->size); +if (dma_mapping_error (dev, iova->addr)) +return -ENOMEM ; + +return 0 ; +} +EXPORT_SYMBOL (dma_alloc_iova); + +void dma_free_iova (struct dma_iova_attrs *iova) +{ +struct device *dev = iova->dev; +const struct dma_map_ops *ops =get_dma_ops (dev); + +if (dma_map_direct (dev, ops) || !ops->free_iova) +return ; + + ops->free_iova (dev, iova->addr, iova->size); +} +EXPORT_SYMBOL (dma_free_iova); +static int __dma_map_sg_attrs (struct device *dev,struct scatterlist *sg,int nents,enum dma_data_direction dir,unsigned long attrs) {
Signature · 1 line
2.26.3
PATCH v3 3/26 dma-mapping: provide callbacks to link/unlink pages to specific IOVA
In reply to Oak ZengFrom: Leon Romanovsky <[email protected]>
Introduce new DMA link/unlink API to provide a way for advanced users to directly map/unmap pages without ned to allocate IOVA on every map call.
--- include/linux/dma-map-ops.h | 10 +++++++ include/linux/dma-mapping.h | 13 +++++++++ kernel/dma/debug.h | 2 ++ kernel/dma/direct.h | 3 ++ kernel/dma/mapping.c | 57 +++++++++++++++++++++++++++++++++++++ 5 files changed, 85 insertions(+)
include/linux/dma-map-ops.h +10 -0
@@ -89,6 +89,13 @@ struct dma_map_ops {dma_addr_t (*alloc_iova)(struct device *dev,size_t size);void (*free_iova)(struct device *dev,dma_addr_t dma_addr,size_t size); +dma_addr_t (*link_range)(struct device *dev,struct page *page, +unsigned long offset,dma_addr_t addr, +size_t size,enum dma_data_direction dir, +unsigned long attrs); +void (*unlink_range)(struct device *dev,dma_addr_t dma_handle, +size_t size,enum dma_data_direction dir, +unsigned long attrs); };#ifdef CONFIG_DMA_OPS
@@ -440,6 +447,9 @@ bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,#define arch_dma_unmap_sg_direct (d, s, n) (false)#endif +#define arch_dma_link_range_direct arch_dma_map_page_direct +#define arch_dma_unlink_range_direct arch_dma_unmap_page_direct +#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS void arch_setup_dma_ops (struct device *dev,bool coherent);#else
include/linux/dma-mapping.h +13 -0
@@ -113,6 +113,9 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)int dma_alloc_iova (struct dma_iova_attrs *iova);void dma_free_iova (struct dma_iova_attrs *iova); +dma_addr_t dma_link_range (struct page *page,unsigned long offset, +struct dma_iova_attrs *iova,dma_addr_t dma_offset); +void dma_unlink_range (struct dma_iova_attrs *iova,dma_addr_t dma_offset); dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, size_t offset, size_t size, enum dma_data_direction dir,
@@ -170,6 +173,16 @@ static inline int dma_alloc_iova(struct dma_iova_attrs *iova)static inline void dma_free_iova (struct dma_iova_attrs *iova) { } +static inline dma_addr_t dma_link_range (struct page *page,unsigned long offset, +struct dma_iova_attrs *iova, +dma_addr_t dma_offset) +{ +return DMA_MAPPING_ERROR ; +} +static inline void dma_unlink_range (struct dma_iova_attrs *iova, +dma_addr_t dma_offset) +{ +}static inline dma_addr_t dma_map_page_attrs (struct device *dev,struct page *page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs)
kernel/dma/debug.h +2 -0
@@ -127,4 +127,6 @@ static inline void debug_dma_sync_sg_for_device(struct device *dev, { }#endif /* CONFIG_DMA_API_DEBUG */ +#define debug_dma_link_range debug_dma_map_page +#define debug_dma_unlink_range debug_dma_unmap_page#endif /* _KERNEL_DMA_DEBUG_H */
kernel/dma/direct.h +3 -0
@@ -125,4 +125,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,swiotlb_tbl_unmap_single (dev, phys, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); } + +#define dma_direct_link_range dma_direct_map_page +#define dma_direct_unlink_range dma_direct_unmap_page#endif /* _KERNEL_DMA_DIRECT_H */
kernel/dma/mapping.c +57 -0
63 lines not on this page · raw message
Signature · 1 line
2.26.3
PATCH v3 4/26 iommu/dma: Provide an interface to allow preallocate IOVA
In reply to Oak ZengFrom: Leon Romanovsky <[email protected]>
Separate IOVA allocation to dedicated callback so it will allow cache of IOVA and reuse it in fast paths for devices which support ODP (on-demand-paging) mechanism.
--- drivers/iommu/dma-iommu.c | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-)
drivers/iommu/dma-iommu.c +38 -12
@@ -357,7 +357,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain)atomic_set (&cookie->fq_timer_on,0 );/* * Prevent incomplete fq state being observable. Pairs with path from -* __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() +* __iommu_dma_unmap() through __iommu_dma_free_iova() to queue_iova() */ smp_wmb ();WRITE_ONCE (cookie->fq_domain, domain);
@@ -758,7 +758,7 @@ static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, } } -static dma_addr_t iommu_dma_alloc_iova (struct iommu_domain *domain, +static dma_addr_t __iommu_dma_alloc_iova (struct iommu_domain *domain,size_t size,u64 dma_limit,struct device *dev) {struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -804,7 +804,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,return (dma_addr_t )iova << shift; } -static void iommu_dma_free_iova (struct iommu_dma_cookie *cookie, +static void __iommu_dma_free_iova (struct iommu_dma_cookie *cookie,dma_addr_t iova,size_t size,struct iommu_iotlb_gather *gather) {struct iova_domain *iovad = &cookie->iovad;
@@ -841,7 +841,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,if (!iotlb_gather.queued)iommu_iotlb_sync (domain, &iotlb_gather); -iommu_dma_free_iova (cookie, dma_addr, size, &iotlb_gather); +__iommu_dma_free_iova (cookie, dma_addr, size, &iotlb_gather); } static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -864,12 +864,12 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, size =iova_align (iovad, size + iova_off); - iova =iommu_dma_alloc_iova (domain, size, dma_mask, dev); + iova =__iommu_dma_alloc_iova (domain, size, dma_mask, dev);if (!iova)return DMA_MAPPING_ERROR ;if (iommu_map (domain, iova, phys - iova_off, size, prot,GFP_ATOMIC )) { -iommu_dma_free_iova (cookie, iova, size,NULL ); +__iommu_dma_free_iova (cookie, iova, size,NULL );return DMA_MAPPING_ERROR ; }return iova + iova_off;
@@ -973,7 +973,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,return NULL ; size =iova_align (iovad, size); - iova =iommu_dma_alloc_iova (domain, size, dev->coherent_dma_mask, dev); + iova =__iommu_dma_alloc_iova (domain, size, dev->coherent_dma_mask, dev);if (!iova) goto out_free_pages;
@@ -1007,7 +1007,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev, out_free_sg:sg_free_table (sgt); out_free_iova: -iommu_dma_free_iova (cookie, iova, size,NULL ); +__iommu_dma_free_iova (cookie, iova, size,NULL ); out_free_pages:__iommu_dma_free_pages (pages, count);return NULL ;
@@ -1442,7 +1442,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,if (!iova_len)return __finalise_sg (dev, sg, nents,0 ); - iova =iommu_dma_alloc_iova (domain, iova_len,dma_get_mask (dev), dev); + iova =__iommu_dma_alloc_iova (domain, iova_len,dma_get_mask (dev), dev);if (!iova) { ret = -ENOMEM ; goto out_restore_sg;
@@ -1459,7 +1459,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,return __finalise_sg (dev, sg, nents, iova); out_free_iova: -iommu_dma_free_iova (cookie, iova, iova_len,NULL ); +__iommu_dma_free_iova (cookie, iova, iova_len,NULL ); out_restore_sg:__invalidate_sg (sg, nents); out:
@@ -1720,6 +1720,30 @@ static size_t iommu_dma_max_mapping_size(struct device *dev)return SIZE_MAX ; } +static dma_addr_t iommu_dma_alloc_iova (struct device *dev,size_t size) +{ +struct iommu_domain *domain =iommu_get_dma_domain (dev); +struct iommu_dma_cookie *cookie = domain->iova_cookie; +struct iova_domain *iovad = &cookie->iovad; +dma_addr_t dma_mask =dma_get_mask (dev); + + size =iova_align (iovad, size); +return __iommu_dma_alloc_iova (domain, size, dma_mask, dev); +} + +static void iommu_dma_free_iova (struct device *dev,dma_addr_t iova, +size_t size) +{ +struct iommu_domain *domain =iommu_get_dma_domain (dev); +struct iommu_dma_cookie *cookie = domain->iova_cookie; +struct iova_domain *iovad = &cookie->iovad; +struct iommu_iotlb_gather iotlb_gather; + + size =iova_align (iovad, size); +iommu_iotlb_gather_init (&iotlb_gather); +__iommu_dma_free_iova (cookie, iova, size, &iotlb_gather); +} +static const struct dma_map_ops iommu_dma_ops = { .flags =DMA_F_PCI_P2PDMA_SUPPORTED |DMA_F_CAN_SKIP_SYNC ,
@@ -1744,6 +1768,8 @@ static const struct dma_map_ops iommu_dma_ops = { .get_merge_boundary = iommu_dma_get_merge_boundary, .opt_mapping_size = iommu_dma_opt_mapping_size, .max_mapping_size = iommu_dma_max_mapping_size, + .alloc_iova = iommu_dma_alloc_iova, + .free_iova = iommu_dma_free_iova, };void iommu_setup_dma_ops (struct device *dev)
@@ -1786,7 +1812,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,if (!msi_page)return NULL ; - iova =iommu_dma_alloc_iova (domain, size,dma_get_mask (dev), dev); + iova =__iommu_dma_alloc_iova (domain, size,dma_get_mask (dev), dev);if (!iova) goto out_free_page;
@@ -1800,7 +1826,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,return msi_page; out_free_iova: -iommu_dma_free_iova (cookie, iova, size,NULL ); +__iommu_dma_free_iova (cookie, iova, size,NULL ); out_free_page:kfree (msi_page);return NULL ;
Signature · 1 line
2.26.3
PATCH v3 5/26 iommu/dma: Prepare map/unmap page functions to receive IOVA
In reply to Oak ZengFrom: Leon Romanovsky <[email protected]>
Extend the existing map_page/unmap_page function implementations to get preallocated IOVA. In such case, the IOVA allocation needs to be skipped, but rest of the code stays the same.
--- drivers/iommu/dma-iommu.c | 68 ++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 23 deletions(-)
drivers/iommu/dma-iommu.c +45 -23
@@ -822,7 +822,7 @@ static void __iommu_dma_free_iova(struct iommu_dma_cookie *cookie, }static void __iommu_dma_unmap (struct device *dev,dma_addr_t dma_addr, -size_t size) +size_t size,bool free_iova) {struct iommu_domain *domain =iommu_get_dma_domain (dev);struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -841,17 +841,19 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,if (!iotlb_gather.queued)iommu_iotlb_sync (domain, &iotlb_gather); -__iommu_dma_free_iova (cookie, dma_addr, size, &iotlb_gather); +if (free_iova) +__iommu_dma_free_iova (cookie, dma_addr, size, &iotlb_gather); }static dma_addr_t __iommu_dma_map (struct device *dev,phys_addr_t phys, -size_t size,int prot,u64 dma_mask) +dma_addr_t iova,size_t size,int prot, +u64 dma_mask) {struct iommu_domain *domain =iommu_get_dma_domain (dev);struct iommu_dma_cookie *cookie = domain->iova_cookie;struct iova_domain *iovad = &cookie->iovad;size_t iova_off =iova_offset (iovad, phys); -dma_addr_t iova; +bool no_iova = !iova;if (static_branch_unlikely (&iommu_deferred_attach_enabled) &&iommu_deferred_attach (dev, domain))
@@ -864,12 +866,14 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, size =iova_align (iovad, size + iova_off); - iova =__iommu_dma_alloc_iova (domain, size, dma_mask, dev); +if (no_iova) + iova =__iommu_dma_alloc_iova (domain, size, dma_mask, dev);if (!iova)return DMA_MAPPING_ERROR ;if (iommu_map (domain, iova, phys - iova_off, size, prot,GFP_ATOMIC )) { -__iommu_dma_free_iova (cookie, iova, size,NULL ); +if (no_iova) +__iommu_dma_free_iova (cookie, iova, size,NULL );return DMA_MAPPING_ERROR ; }return iova + iova_off;
@@ -1034,7 +1038,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,return vaddr; out_unmap: -__iommu_dma_unmap (dev, *dma_handle, size); +__iommu_dma_unmap (dev, *dma_handle, size, true);__iommu_dma_free_pages (pages,PAGE_ALIGN (size) >>PAGE_SHIFT );return NULL ; }
@@ -1063,7 +1067,7 @@ static void iommu_dma_free_noncontiguous(struct device *dev, size_t size, {struct dma_sgt_handle *sh =sgt_handle (sgt); -__iommu_dma_unmap (dev, sgt->sgl->dma_address, size); +__iommu_dma_unmap (dev, sgt->sgl->dma_address, size, true);__iommu_dma_free_pages (sh->pages,PAGE_ALIGN (size) >>PAGE_SHIFT );sg_free_table (&sh->sgt);kfree (sh);
@@ -1134,9 +1138,11 @@ static void iommu_dma_sync_sg_for_device(struct device *dev,arch_sync_dma_for_device (sg_phys (sg), sg->length, dir); } -static dma_addr_t iommu_dma_map_page (struct device *dev,struct page *page, -unsigned long offset,size_t size,enum dma_data_direction dir, -unsigned long attrs) +static dma_addr_t __iommu_dma_map_pages (struct device *dev,struct page *page, +unsigned long offset,dma_addr_t iova, +size_t size, +enum dma_data_direction dir, +unsigned long attrs) {phys_addr_t phys =page_to_phys (page) + offset;bool coherent =dev_is_dma_coherent (dev);
@@ -1144,7 +1150,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,struct iommu_domain *domain =iommu_get_dma_domain (dev);struct iommu_dma_cookie *cookie = domain->iova_cookie;struct iova_domain *iovad = &cookie->iovad; -dma_addr_t iova, dma_mask =dma_get_mask (dev); +dma_addr_t addr, dma_mask =dma_get_mask (dev);/* * If both the physical buffer start address and size are
@@ -1188,14 +1194,23 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,if (!coherent && !(attrs &DMA_ATTR_SKIP_CPU_SYNC ))arch_sync_dma_for_device (phys, size, dir); - iova =__iommu_dma_map (dev, phys, size, prot, dma_mask); -if (iova ==DMA_MAPPING_ERROR &&is_swiotlb_buffer (dev, phys)) + addr =__iommu_dma_map (dev, phys, iova, size, prot, dma_mask); +if (addr ==DMA_MAPPING_ERROR &&is_swiotlb_buffer (dev, phys))swiotlb_tbl_unmap_single (dev, phys, size, dir, attrs); -return iova; +return addr; } -static void iommu_dma_unmap_page (struct device *dev,dma_addr_t dma_handle, -size_t size,enum dma_data_direction dir,unsigned long attrs) +static dma_addr_t iommu_dma_map_page (struct device *dev,struct page *page, +unsigned long offset,size_t size, +enum dma_data_direction dir, +unsigned long attrs) +{ +return __iommu_dma_map_pages (dev, page, offset,0 , size, dir, attrs); +} + +static void __iommu_dma_unmap_pages (struct device *dev,dma_addr_t dma_handle, +size_t size,enum dma_data_direction dir, +unsigned long attrs,bool free_iova) {struct iommu_domain *domain =iommu_get_dma_domain (dev);phys_addr_t phys;
@@ -1207,12 +1222,19 @@ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,if (!(attrs &DMA_ATTR_SKIP_CPU_SYNC ) && !dev_is_dma_coherent (dev))arch_sync_dma_for_cpu (phys, size, dir); -__iommu_dma_unmap (dev, dma_handle, size); +__iommu_dma_unmap (dev, dma_handle, size, free_iova);if (unlikely (is_swiotlb_buffer (dev, phys)))swiotlb_tbl_unmap_single (dev, phys, size, dir, attrs); } +static void iommu_dma_unmap_page (struct device *dev,dma_addr_t dma_handle, +size_t size,enum dma_data_direction dir, +unsigned long attrs) +{ +__iommu_dma_unmap_pages (dev, dma_handle, size, dir, attrs, true); +} +/* * Prepare a successfully-mapped scatterlist to give back to the caller. *
@@ -1515,13 +1537,13 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg, }if (end) -__iommu_dma_unmap (dev, start, end - start); +__iommu_dma_unmap (dev, start, end - start, true); }static dma_addr_t iommu_dma_map_resource (struct device *dev,phys_addr_t phys,size_t size,enum dma_data_direction dir,unsigned long attrs) { -return __iommu_dma_map (dev, phys, size, +return __iommu_dma_map (dev, phys,0 , size,dma_info_to_prot (dir, false, attrs) |IOMMU_MMIO ,dma_get_mask (dev)); }
@@ -1529,7 +1551,7 @@ static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,static void iommu_dma_unmap_resource (struct device *dev,dma_addr_t handle,size_t size,enum dma_data_direction dir,unsigned long attrs) { -__iommu_dma_unmap (dev, handle, size); +__iommu_dma_unmap (dev, handle, size, true); }static void __iommu_dma_free (struct device *dev,size_t size,void *cpu_addr)
@@ -1566,7 +1588,7 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)static void iommu_dma_free (struct device *dev,size_t size,void *cpu_addr,dma_addr_t handle,unsigned long attrs) { -__iommu_dma_unmap (dev, handle, size); +__iommu_dma_unmap (dev, handle, size, true);__iommu_dma_free (dev, size, cpu_addr); }
@@ -1632,7 +1654,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,if (!cpu_addr)return NULL ; - *handle =__iommu_dma_map (dev,page_to_phys (page), size, ioprot, + *handle =__iommu_dma_map (dev,page_to_phys (page),0 , size, ioprot, dev->coherent_dma_mask);if (*handle ==DMA_MAPPING_ERROR ) {__iommu_dma_free (dev, size, cpu_addr);
Signature · 1 line
2.26.3
PATCH v3 6/26 iommu/dma: Implement link/unlink page callbacks
In reply to Oak ZengFrom: Leon Romanovsky <[email protected]>
Add an implementation of link/unlink interface to perform in map/unmap pages in fast patch for pre-allocated IOVA.
--- drivers/iommu/dma-iommu.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
drivers/iommu/dma-iommu.c +17 -0
@@ -1766,6 +1766,21 @@ static void iommu_dma_free_iova(struct device *dev, dma_addr_t iova,__iommu_dma_free_iova (cookie, iova, size, &iotlb_gather); } +static dma_addr_t iommu_dma_link_range (struct device *dev,struct page *page, +unsigned long offset,dma_addr_t iova, +size_t size,enum dma_data_direction dir, +unsigned long attrs) +{ +return __iommu_dma_map_pages (dev, page, offset, iova, size, dir, attrs); +} + +static void iommu_dma_unlink_range (struct device *dev,dma_addr_t addr, +size_t size,enum dma_data_direction dir, +unsigned long attrs) +{ +__iommu_dma_unmap_pages (dev, addr, size, dir, attrs, false); +} +static const struct dma_map_ops iommu_dma_ops = { .flags =DMA_F_PCI_P2PDMA_SUPPORTED |DMA_F_CAN_SKIP_SYNC ,
@@ -1792,6 +1807,8 @@ static const struct dma_map_ops iommu_dma_ops = { .max_mapping_size = iommu_dma_max_mapping_size, .alloc_iova = iommu_dma_alloc_iova, .free_iova = iommu_dma_free_iova, + .link_range = iommu_dma_link_range, + .unlink_range = iommu_dma_unlink_range, };void iommu_setup_dma_ops (struct device *dev)
Signature · 1 line
2.26.3
PATCH v3 7/26 drm: move xe_sg_segment_size to drm layer
In reply to Oak ZengMove this helper function to drm layer and rename it to drm_gem_dma_max_sg_segment, so it can be used by the coming drm patches also. No functional changes.
--- drivers/gpu/drm/xe/xe_bo.c | 3 ++- drivers/gpu/drm/xe/xe_bo.h | 24 ------------------------ drivers/gpu/drm/xe/xe_device.c | 3 ++- drivers/gpu/drm/xe/xe_hmm.c | 3 ++- include/drm/drm_gem_dma_helper.h | 25 +++++++++++++++++++++++++ 5 files changed, 31 insertions(+), 27 deletions(-)
drivers/gpu/drm/xe/xe_bo.c +2 -1
@@ -9,6 +9,7 @@#include <drm/drm_drv.h> #include <drm/drm_gem_ttm_helper.h> +#include <drm/drm_gem_dma_helper.h> #include <drm/drm_managed.h> #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h>
@@ -299,7 +300,7 @@ static int xe_tt_map_sg(struct ttm_tt *tt) ret =sg_alloc_table_from_pages_segment (&xe_tt->sgt, tt->pages, num_pages,0 , (u64 )num_pages <<PAGE_SHIFT , -xe_sg_segment_size (xe_tt->dev), +drm_gem_dma_max_sg_segment (xe_tt->dev),GFP_KERNEL );if (ret)return ret;
drivers/gpu/drm/xe/xe_bo.h +0 -24
@@ -289,30 +289,6 @@ void xe_bo_put_commit(struct llist_head *deferred);struct sg_table *xe_bo_sg (struct xe_bo *bo); -/* -* xe_sg_segment_size() - Provides upper limit for sg segment size. -* @dev: device pointer -* -* Returns the maximum segment size for the 'struct scatterlist' -* elements. -*/ -static inline unsigned int xe_sg_segment_size (struct device *dev) -{ -struct scatterlist __maybe_unused sg; -size_t max =BIT_ULL (sizeof (sg.length) *8 ) -1 ; - - max =min_t (size_t, max,dma_max_mapping_size (dev)); - -/* -* The iommu_dma_map_sg() function ensures iova allocation doesn't -* cross dma segment boundary. It does so by padding some sg elements. -* This can cause overflow, ending up with sg->length being set to 0. -* Avoid this by ensuring maximum segment size is half of 'max' -* rounded down to PAGE_SIZE. -*/ -return round_down (max /2 ,PAGE_SIZE ); -} -#define i915_gem_object_flush_if_display (obj) ((void)(obj))#if IS_ENABLED (CONFIG_DRM_XE_KUNIT_TEST )
drivers/gpu/drm/xe/xe_device.c +2 -1
@@ -12,6 +12,7 @@#include <drm/drm_atomic_helper.h> #include <drm/drm_client.h> #include <drm/drm_gem_ttm_helper.h> +#include <drm/drm_gem_dma_helper.h> #include <drm/drm_ioctl.h> #include <drm/drm_managed.h> #include <drm/drm_print.h>
@@ -407,7 +408,7 @@ static int xe_set_dma_info(struct xe_device *xe)unsigned int mask_size = xe->info.dma_mask_size;int err; -dma_set_max_seg_size (xe->drm.dev,xe_sg_segment_size (xe->drm.dev)); +dma_set_max_seg_size (xe->drm.dev,drm_gem_dma_max_sg_segment (xe->drm.dev)); err =dma_set_mask (xe->drm.dev,DMA_BIT_MASK (mask_size));if (err)
drivers/gpu/drm/xe/xe_hmm.c +2 -1
@@ -3,6 +3,7 @@* Copyright © 2024 Intel Corporation */ +#include <drm/drm_gem_dma_helper.h> #include <linux/scatterlist.h> #include <linux/mmu_notifier.h> #include <linux/dma-mapping.h>
@@ -96,7 +97,7 @@ static int xe_build_sg(struct xe_device *xe, struct hmm_range *range, } ret =sg_alloc_table_from_pages_segment (st, pages, npages,0 , npages <<PAGE_SHIFT , -xe_sg_segment_size (dev),GFP_KERNEL ); +drm_gem_dma_max_sg_segment (dev),GFP_KERNEL );if (ret) goto free_pages;
include/drm/drm_gem_dma_helper.h +25 -0
@@ -5,6 +5,7 @@#include <drm/drm_file.h> #include <drm/drm_ioctl.h> #include <drm/drm_gem.h> +#include <linux/dma-mapping.h> struct drm_mode_create_dumb ;
@@ -133,6 +134,30 @@ static inline int drm_gem_dma_object_mmap(struct drm_gem_object *obj, struct vm_return drm_gem_dma_mmap (dma_obj, vma); } +/* +* drm_gem_dma_max_sg_segment() - Provides upper limit for sg segment size. +* @dev: device pointer +* +* Returns the maximum segment size for the 'struct scatterlist' +* elements. +*/ +static inline unsigned int drm_gem_dma_max_sg_segment (struct device *dev) +{ +struct scatterlist __maybe_unused sg; +size_t max =BIT_ULL (sizeof (sg.length) *8 ) -1 ; + + max =min_t (size_t, max,dma_max_mapping_size (dev)); + +/* +* The iommu_dma_map_sg() function ensures iova allocation doesn't +* cross dma segment boundary. It does so by padding some sg elements. +* This can cause overflow, ending up with sg->length being set to 0. +* Avoid this by ensuring maximum segment size is half of 'max' +* rounded down to PAGE_SIZE. +*/ +return round_down (max /2 ,PAGE_SIZE ); +} +/* * Driver ops */
Signature · 1 line
2.26.3
PATCH v3 8/26 drm: Move GPUVA_START/LAST to drm_gpuvm.h
In reply to Oak ZengMove them to .h file so they can be used outside of drm_gpuvm.c. Also add a GPUVA_END micro.
--- drivers/gpu/drm/drm_gpuvm.c | 3 --- include/drm/drm_gpuvm.h | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-)
drivers/gpu/drm/drm_gpuvm.c +0 -3
@@ -867,9 +867,6 @@ __drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,#define to_drm_gpuva (__node) container_of((__node), struct drm_gpuva, rb.node) -#define GPUVA_START (node) ((node)->va.addr) -#define GPUVA_LAST (node) ((node)->va.addr + (node)->va.range - 1) -/* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain * about this. */
include/drm/drm_gpuvm.h +4 -0
@@ -38,6 +38,10 @@ struct drm_gpuvm;struct drm_gpuvm_bo ;struct drm_gpuvm_ops ; +#define GPUVA_START (node) ((node)->va.addr) +#define GPUVA_LAST (node) ((node)->va.addr + (node)->va.range - 1) +#define GPUVA_END (node) ((node)->va.addr + (node)->va.range) +/** * enum drm_gpuva_flags - flags for struct drm_gpuva */
Signature · 1 line
2.26.3
PATCH v3 9/26 drm/svm: add a mm field to drm_gpuvm struct
In reply to Oak ZengAdd a mm_struct field to drm_gpuvm. This is only used by shared virtual memory, where CPU program and GPU program share one process virtual address space.
--- drivers/gpu/drm/drm_gpuvm.c | 1 + include/drm/drm_gpuvm.h | 6 ++++++ 2 files changed, 7 insertions(+)
drivers/gpu/drm/drm_gpuvm.c +1 -0
@@ -1015,6 +1015,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, gpuvm->ops = ops; gpuvm->drm = drm; gpuvm->r_obj = r_obj; + gpuvm->mm = current->mm;drm_gem_object_get (r_obj);
include/drm/drm_gpuvm.h +6 -0
@@ -242,6 +242,12 @@ struct drm_gpuvm {* @drm: the &drm_device this VM lives in */ struct drm_device *drm; +/** +* @mm: the process &mm_struct which create this gpuvm. +* This is only used for shared virtual memory where virtual +* address space is shared b/t CPU and GPU program. +*/ +struct mm_struct *mm;/** * @mm_start: start of the VA space
Signature · 1 line
2.26.3
PATCH v3 10/26 drm/svm: introduce drm_mem_region concept
In reply to Oak ZengAs its name indicates, a drm_mem_region represent a memory region on a drm device, e.g., a GPU's HBM memory.
In a memory region, we have address information of this region, from both CPU (hpa_base) and GPU (dpa_base) perspectives. It also has some interfaces for drm to callback driver to allocate/free memory from this region, migrate data to/from this memory region, get a pagemap owner of a memory region, and get the drm device which own the memory region
This is introduced for system allocator implementation, so the memory allocation and free interfaces are page based.
A few helper functions are also introduced:
1) drm_mem_region_pfn_to_dpa: calculate device physical address from a page's pfn
2) drm_page_to_mem_region: retrieve drm memory region that a page resides in
--- include/drm/drm_svm.h | 156 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 include/drm/drm_svm.h
include/drm/drm_svm.h new file +156 -0
@@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright © 2024 Intel Corporation +*/ + +#include <linux/compiler_types.h> +#include <linux/memremap.h> +#include <linux/types.h> + +struct dma_fence ; +struct drm_mem_region ; + +/** +* struct migrate_vec - a migration vector is an array of addresses, +* each of which represents one page. +* When it is system memory page, the address is a dmap-mapped address. +* When it is vram page, the address is device physical address. +*/ +struct migrate_vec { +/** +* @mr: the memory region that pages reside. +* When it is system memory page, mr is NULL. +*/ +struct drm_mem_region *mr; +/** @npages: number of pages */ +u64 npages; +/** +* @addr_vec: address vector +* each item in addr_vec is an address of a page +*/ +union { +/** @dma_addr: dma-mapped address of the page, only valid for system pages*/ +dma_addr_t dma_addr; +/** @dpa: device physical address of the page, only valid for vram page*/ +phys_addr_t dpa; + } addr_vec[1 ]; +}; + +/** +* struct drm_mem_region_ops - memory region operations such as memory allocation +* and migration etc. Driver is supposed to implement those operations. +*/ +struct drm_mem_region_ops { +/** +* @drm_mem_region_alloc_pages: Called from drm to driver to allocate +* device VRAM memory +* @mr: The memory region where to allocate device VRAM memory from +* @npages: number of pages to allocate +* @pfns: Used to return pfns of each page +*/ +int (*drm_mem_region_alloc_pages) (struct drm_mem_region *mr, +unsigned long npags,unsigned long *pfns); +/** +* @drm_mem_region_free_page: Called from drm to free one page of device memory +* @page: point to the page to free +*/ +void (*drm_mem_region_free_page) (struct page *page); +/** +* @drm_mem_region_migrate: Called from drm to migrate memory from src to +* dst. Driver is supposed to implement this function using device hardware +* accelerators such as DMA. DRM subsystem call this function to migrate +* memory b/t system memory and device memory region +* +* @src_vec: source migration vector +* @dst_vec: destination migration vector +*/ +struct dma_fence * (*drm_mem_region_migrate)(struct migrate_vec *src_vec, +struct migrate_vec *dst_vec); +/** +* @drm_mem_region_pagemap_owner: Return the pagemap owner of a memory +* region. Pagemap owner is the owner of the device memory. It is +* defined by device driver and opaque to drm layer. Drm uses pagemap +* owner to set up page migrations (see hmm function migrate_vma_setup) +* and range population (see hmm functio hmm_range_fault). Driver has +* the freedom to choose the right pagemap owner. +* +* @mr: the memory region which we want to get the page map owner +*/ +void * (*drm_mem_region_pagemap_owner)(struct drm_mem_region *mr); +/** +* @drm_mem_region_get_device: Return the drm device which has this memory +* region. +* +* @mr: the memory region +*/ +struct drm_device * (*drm_mem_region_get_device)(struct drm_mem_region *mr); +}; + +/** +* struct drm_mem_region - memory region structure +* This is used to describe a memory region in drm +* device, such as HBM memory or CXL extension memory. +* +* drm_mem_region is converted from the xe_mem_region +* concept. xe_mem_region is moved to drm layer and renamed +* as drm_mem_region. +* +* drm_mem_region is supposed to be embedded in some driver struct such as +* "struct xe_tile" or "struct amdgpu_device" +*/ +struct drm_mem_region { +/** @dev_private: device private data which is opaque to drm layer */ +void *dev_private; +/** @dpa_base: This memory regions's DPA (device physical address) base */ +resource_size_t dpa_base; +/** +* @usable_size: usable size of VRAM +* +* Usable size of VRAM excluding reserved portions +* (e.g stolen mem) +*/ +resource_size_t usable_size; +/** @pagemap: Used to remap device memory as ZONE_DEVICE */ +struct dev_pagemap pagemap; +/** +* @hpa_base: base host physical address +* +* This is generated when remap device memory as ZONE_DEVICE +*/ +resource_size_t hpa_base; +/** +* @mr_ops: memory region operation function pointers +*/ +struct drm_mem_region_ops mr_ops; +}; + +/** +* drm_page_to_mem_region() - Get a page's memory region +* +* @page: a struct page pointer pointing to a page in vram memory region +*/ +static inline struct drm_mem_region *drm_page_to_mem_region (struct page *page) +{ + return container_of(page->pgmap, struct drm_mem_region, pagemap); +} + +/** +* drm_mem_region_pfn_to_dpa() - Calculate page's dpa from pfn +* +* @mr: The memory region that page resides in +* @pfn: page frame number of the page +* +* Returns: the device physical address of the page +*/ +static inline u64 drm_mem_region_pfn_to_dpa (struct drm_mem_region *mr,u64 pfn) +{ +u64 dpa; +u64 offset; + +BUG_ON ((pfn <<PAGE_SHIFT ) < mr->hpa_base ); +BUG_ON ((pfn <<PAGE_SHIFT ) >= mr->hpa_base + mr->usable_size); + offset = (pfn <<PAGE_SHIFT ) - mr->hpa_base; + dpa = mr->dpa_base + offset; + +return dpa; +}
Signature · 1 line
2.26.3
PATCH v3 11/26 drm/svm: introduce hmmptr and helper functions
In reply to Oak ZengA hmmptr is a pointer in a CPU program, like a userptr. but unlike a userptr, a hmmptr can also be migrated to device local memory. The other way to look at is, userptr is a special hmmptr without the capability of migration - userptr's backing store is always in system memory.
This is built on top of kernel HMM infrastructure thus is called hmmptr.
Helper functions are introduced to init, release, populate and dma-map hmmptr.
--- drivers/gpu/drm/Kconfig | 1 + drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/drm_svm.c | 254 ++++++++++++++++++++++++++++++++++++++ include/drm/drm_svm.h | 55 +++++++++ 4 files changed, 311 insertions(+) create mode 100644 drivers/gpu/drm/drm_svm.c
drivers/gpu/drm/Kconfig +1 -0
@@ -20,6 +20,7 @@ menuconfig DRM # device and dmabuf fd. Let's make sure that is available for our userspace. select KCMP select VIDEO + select HMM_MIRROR help Kernel-level support for the Direct Rendering Infrastructure (DRI) introduced in XFree86 4.0. If you say Y here, you need to select
drivers/gpu/drm/Makefile +1 -0
@@ -89,6 +89,7 @@ drm-$(CONFIG_DRM_PRIVACY_SCREEN) += \ drm_privacy_screen_x86.o drm-$(CONFIG_DRM_ACCEL) += ../../accel/drm_accel.o drm-$(CONFIG_DRM_PANIC) += drm_panic.o +drm-$(CONFIG_HMM_MIRROR) += ./drm_svm.o obj-$(CONFIG_DRM) += drm.o obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
drivers/gpu/drm/drm_svm.c new file +254 -0
254 lines not on this page · raw message
include/drm/drm_svm.h +55 -0
69 lines not on this page · raw message
Signature · 1 line
2.26.3
PATCH v3 12/26 drm/svm: Introduce helper to remap drm memory region
In reply to Oak ZengHelper function drm_svm_register_mem_region to remap GPU vram using devm_memremap_pages, so each GPU vram page is backed by struct page.
Those struct pages are created to allow hmm migrate buffer b/t GPU vram and CPU system memory using existing Linux migration mechanism (i.e., migrating b/t CPU system memory and hard disk).
This is prepare work to enable svm (shared virtual memory) through Linux kernel hmm framework. The memory remap's page map type is set to MEMORY_DEVICE_PRIVATE for now. This means even though each GPU vram page get a struct page and can be mapped in CPU page table, but such pages are treated as GPU's private resource, so CPU can't access them. If CPU access such page, a page fault is triggered and page will be migrate to system memory.
For GPU device which supports coherent memory protocol b/t CPU and GPU (such as CXL and CAPI protocol), we can remap device memory as MEMORY_DEVICE_COHERENT. This is TBD.
--- drivers/gpu/drm/drm_svm.c | 50 +++++++++++++++++++++++++++++++++++++++ include/drm/drm_svm.h | 2 ++ 2 files changed, 52 insertions(+)
drivers/gpu/drm/drm_svm.c +50 -0
@@ -13,6 +13,7 @@#include <linux/swap.h> #include <linux/bug.h> #include <linux/hmm.h> +#include <linux/pci.h> #include <linux/mm.h> static u64 __npages_in_range (unsigned long start,unsigned long end)
@@ -252,3 +253,52 @@ int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, ureturn ret; }EXPORT_SYMBOL_GPL (drm_svm_hmmptr_populate); + +static struct dev_pagemap_ops drm_devm_pagemap_ops; + +/** +* drm_svm_register_mem_region: Remap and provide memmap backing for device memory +* @drm: drm device who want to register a memory region +* @mr: memory region to register +* +* This remap device memory to host physical address space and create +* struct page to back device memory +* +* Return: 0 on success standard error code otherwise +*/ +int drm_svm_register_mem_region (const struct drm_device *drm,struct drm_mem_region *mr) +{ +struct device *dev = &to_pci_dev (drm->dev)->dev; +struct resource *res; +void *addr; +int ret; + + res =devm_request_free_mem_region (dev, &iomem_resource, + mr->usable_size); +if (IS_ERR (res)) { + ret =PTR_ERR (res); +return ret; + } + + drm_devm_pagemap_ops.page_free = mr->mr_ops.drm_mem_region_free_page; + mr->pagemap.type =MEMORY_DEVICE_PRIVATE ; + mr->pagemap.range.start = res->start; + mr->pagemap.range.end = res->end; + mr->pagemap.nr_range =1 ; + mr->pagemap.ops = &drm_devm_pagemap_ops; + mr->pagemap.owner = mr->mr_ops.drm_mem_region_pagemap_owner (mr); + addr =devm_memremap_pages (dev, &mr->pagemap); +if (IS_ERR (addr)) { +devm_release_mem_region (dev, res->start,resource_size (res)); + ret =PTR_ERR (addr); +drm_err (drm,"Failed to remap memory region %p, errno %d\n" , + mr, ret); +return ret; + } + mr->hpa_base = res->start; + +drm_info (drm,"Registered device memory [%llx-%llx] to devm, remapped to %pr\n" , + mr->dpa_base, mr->dpa_base + mr->usable_size, res); +return 0 ; +} +EXPORT_SYMBOL_GPL (drm_svm_register_mem_region);
include/drm/drm_svm.h +2 -0
@@ -159,6 +159,8 @@ static inline u64 drm_mem_region_pfn_to_dpa(struct drm_mem_region *mr, u64 pfn)return dpa; } +int drm_svm_register_mem_region (const struct drm_device *drm,struct drm_mem_region *mr); +/** * struct drm_hmmptr- hmmptr pointer *
Signature · 1 line
2.26.3
PATCH v3 13/26 drm/svm: handle CPU page fault
In reply to Oak ZengUnder the picture of svm, CPU and GPU program share the same virtual address space. The backing store of this virtual address space can be either in system memory or device memory. Since GPU device memory is remaped as DEVICE_PRIVATE, CPU can't access it. Any CPU access to device memory causes a page fault. Implement a page fault handler to migrate memory back to system memory and map it to CPU page table so the CPU program can proceed.
--- drivers/gpu/drm/drm_svm.c | 251 +++++++++++++++++++++++++++++++++++++- 1 file changed, 250 insertions(+), 1 deletion(-)
drivers/gpu/drm/drm_svm.c +250 -1
@@ -8,6 +8,7 @@#include <linux/mmu_notifier.h> #include <linux/dma-mapping.h> #include <linux/memremap.h> +#include <linux/migrate.h> #include <drm/drm_gem_dma_helper.h> #include <drm/drm_svm.h> #include <linux/swap.h>
@@ -254,7 +255,255 @@ int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, u }EXPORT_SYMBOL_GPL (drm_svm_hmmptr_populate); -static struct dev_pagemap_ops drm_devm_pagemap_ops; +static void __drm_svm_free_pages (unsigned long *mpfn,unsigned long npages) +{ +struct page *page; +int j; + +for (j =0 ; j < npages; j++) { + page =migrate_pfn_to_page (mpfn[j]); + mpfn[j] =0 ; +if (page) { +unlock_page (page); +put_page (page); + } + } +} + +/** +* __drm_svm_alloc_host_pages() - allocate host pages for the fault vma +* +* @vma: the fault vma that we need allocate page for +* @mpfn: used to output the pfns of the allocated pages +* +* This function allocate host pages for a specified vma. +* +* When this function returns, the page is locked. +* +* Return 0 on success +* error code otherwise +*/ +static int __drm_svm_alloc_host_pages (struct vm_area_struct *vma, +unsigned long *mpfn) +{ +unsigned long addr = vma->vm_start; +u64 npages =vma_pages (vma); +struct page *page; +int i; + +for (i =0 ; i < npages; i++) { + page =alloc_page_vma (GFP_HIGHUSER , vma, addr); +if (unlikely (!page)) + goto free_allocated; + +/**Lock page per hmm requirement, see hmm.rst*/ +lock_page (page); + + mpfn[i] =migrate_pfn (page_to_pfn (page)); + addr +=PAGE_SIZE ; + } +return 0 ; + +free_allocated: +__drm_svm_free_pages (mpfn, i); +return -ENOMEM ; +} + +static struct migrate_vec *__generate_migrate_vec_vram (unsigned long *mpfn,bool is_migrate_src,unsigned long npages) +{ +struct migrate_vec *vec; +int size =sizeof (*vec) +sizeof (vec->addr_vec[1 ]) * (npages -1 ); +struct drm_mem_region *mr; +struct page *page; +u64 dpa; +int i, j; + + page =migrate_pfn_to_page (mpfn[0 ]); +if (unlikely (!page)) +return NULL ; + + vec =kzalloc (size,GFP_KERNEL ); +if (!vec) +return NULL ; + + mr =drm_page_to_mem_region (page); +for (i =0 , j =0 ; i < npages; i++) { +if (is_migrate_src && !(mpfn[i] &MIGRATE_PFN_MIGRATE )) +continue ; + + page =migrate_pfn_to_page (mpfn[i]); + dpa =drm_mem_region_pfn_to_dpa (mr,page_to_pfn (page)); + vec->addr_vec[j++].dpa = dpa; + } + vec->mr = mr; + vec->npages = j; +return vec; +} + +static struct migrate_vec *__generate_migrate_vec_sram (struct device *dev,unsigned long *mpfn, +bool is_migrate_src,unsigned long npages) +{ +enum dma_data_direction dir = is_migrate_src ?DMA_TO_DEVICE :DMA_FROM_DEVICE ; +struct migrate_vec *vec; +int size =sizeof (*vec) +sizeof (vec->addr_vec[1 ]) * (npages -1 ); +dma_addr_t dma_addr; +struct page *page; +int i, j, k; + + page =migrate_pfn_to_page (mpfn[0 ]); +if (unlikely (!page)) +return NULL ; + + vec =kzalloc (size,GFP_KERNEL ); +if (!vec) +return NULL ; + +for (i =0 , k =0 ; i < npages; i++) { +if (is_migrate_src && !(mpfn[i] &MIGRATE_PFN_MIGRATE )) +continue ; + + page =migrate_pfn_to_page (mpfn[i]); + dma_addr =dma_map_page (dev, page,0 ,PAGE_SIZE , dir); +if (unlikely (dma_mapping_error (dev, dma_addr))) + goto undo_dma_mapping; + + vec->addr_vec[k++].dma_addr = dma_addr; + } + + vec->mr =NULL ; + vec->npages = k; +return vec; + +undo_dma_mapping: +for (j =0 ; j < k; j++) { +if (vec->addr_vec[j].dma_addr) +dma_unmap_page (dev, vec->addr_vec[j].dma_addr,PAGE_SIZE , dir); + } +kfree (vec); +return NULL ; +} + +static void __free_migrate_vec_sram (struct device *dev,struct migrate_vec *vec,bool is_migrate_src) +{ +enum dma_data_direction dir = is_migrate_src ?DMA_TO_DEVICE :DMA_FROM_DEVICE ; +int i; + +BUG_ON (vec->mr !=NULL ); + +for (i =0 ; i < vec->npages; i++) { +dma_unmap_page (dev, vec->addr_vec[i].dma_addr,PAGE_SIZE , dir); +/** No need to free host pages. migrate_vma_finalize take care */ + } +kfree (vec); +} + +static void __free_migrate_vec_vram (struct migrate_vec *vec) +{ +BUG_ON (vec->mr ==NULL ); + +kfree (vec); +} +/** +* drm_svm_migrate_to_sram() - Migrate memory back to sram on CPU page fault +* +* @vmf: cpu vm fault structure, contains fault information such as vma etc. +* +* Note, this is in CPU's vm fault handler, caller holds the mmap read lock. +* +* This function migrate one page at the fault address. This is the normal +* core mm page fault scheme. Linux doesn't aggressively prefault at CPU page +* fault time. It only fault-in one page to recover the fault address. Even +* if we migrate more than one page, core mm still only program one pte entry +* (covers one page).See logic in function handle_pte_fault. do_swap_page +* eventually calls to drm_svm_migrate_to_sram in our case. +* +* We call migrate_vma_setup to set up the migration. During migrate_vma_setup, +* device page table is invalidated before migration (by calling the driver registered +* mmu notifier) +* +* We call migrate_vma_finalize to finalize the migration. During migrate_vma_finalize, +* device pages of the source buffer is freed (by calling memory region's +* drm_mem_region_free_page callback function) +* +* Return: +* 0 on success +* VM_FAULT_SIGBUS: failed to migrate page to system memory, application +* will be signaled a SIGBUG +*/ +static vm_fault_t drm_svm_migrate_to_sram (struct vm_fault *vmf) +{ +struct drm_mem_region *mr =drm_page_to_mem_region (vmf->page); +struct drm_device *drm = mr->mr_ops.drm_mem_region_get_device (mr); +unsigned long src_pfn =0 , dst_pfn =0 ; +struct device *dev = drm->dev; +struct vm_area_struct *vma = vmf->vma; +struct migrate_vec *src; +struct migrate_vec *dst; +struct dma_fence *fence; +vm_fault_t ret =0 , r; + +struct migrate_vma migrate_vma = { + .vma = vma, + .start =ALIGN_DOWN (vmf->address,PAGE_SIZE ), + .end =ALIGN_DOWN (vmf->address,PAGE_SIZE ) +PAGE_SIZE , + .src = &src_pfn, + .dst = &dst_pfn, + .pgmap_owner = mr->mr_ops.drm_mem_region_pagemap_owner (mr), + .flags =MIGRATE_VMA_SELECT_DEVICE_PRIVATE , + .fault_page = vmf->page, + }; + +if (migrate_vma_setup (&migrate_vma) <0 ) +return VM_FAULT_SIGBUS ; + +if (!migrate_vma.cpages) +return 0 ; + + r =__drm_svm_alloc_host_pages (vma, migrate_vma.dst); +if (r) { + ret =VM_FAULT_OOM ; + goto migrate_pages; + } + + src =__generate_migrate_vec_vram (migrate_vma.src, true,1 ); +if (!src) { + ret =VM_FAULT_OOM ; + goto free_host_pages; + } + + dst =__generate_migrate_vec_sram (dev, migrate_vma.dst, false,1 ); +if (!dst) { + ret =VM_FAULT_OOM ; + goto free_migrate_src; + } + + fence = mr->mr_ops.drm_mem_region_migrate (src, dst); +if (IS_ERR (fence)) { + ret =VM_FAULT_SIGBUS ; + goto free_migrate_dst; + } +dma_fence_wait (fence, false); +dma_fence_put (fence); + +free_migrate_dst: +__free_migrate_vec_sram (dev, dst, false); +free_migrate_src: +__free_migrate_vec_vram (src); +free_host_pages: +/** +* On migration success, hmm take care of life cycle of src and dst pages +* See migrate_vma_finalize(). +*/ +if (ret) +__drm_svm_free_pages (migrate_vma.dst,1 ); +migrate_pages: +migrate_vma_pages (&migrate_vma); +migrate_vma_finalize (&migrate_vma); +return ret; +} +static struct dev_pagemap_ops drm_devm_pagemap_ops = { + .migrate_to_ram = drm_svm_migrate_to_sram, +};/** * drm_svm_register_mem_region: Remap and provide memmap backing for device memory
Signature · 1 line
2.26.3
PATCH v3 14/26 drm/svm: Migrate a range of hmmptr to vram
In reply to Oak ZengIntroduce a helper function drm_svm_migrate_hmmptr_to_vram to migrate any sub-range of a hmmptr to vram. The range has to be at page boundary. This supposed to be called by driver to migrate a hmmptr to vram.
--- drivers/gpu/drm/drm_svm.c | 122 ++++++++++++++++++++++++++++++++++++++ include/drm/drm_svm.h | 3 + 2 files changed, 125 insertions(+)
drivers/gpu/drm/drm_svm.c +122 -0
@@ -551,3 +551,125 @@ int drm_svm_register_mem_region(const struct drm_device *drm, struct drm_mem_regreturn 0 ; }EXPORT_SYMBOL_GPL (drm_svm_register_mem_region); + +static void __drm_svm_init_device_pages (unsigned long *pfn,unsigned long npages) +{ +struct page *page; +int i; + +for (i =0 ; i < npages; i++) { + page =pfn_to_page (pfn[i]); +zone_device_page_init (page); + pfn[i] =migrate_pfn (pfn[i]); + } +} + +/** +* drm_svm_migrate_hmmptr_to_vram() - migrate a sub-range of a hmmptr to vram +* Must be called with mmap_read_lock held. +* +* @vm: the vm that the hmmptr belongs to +* @mr: the destination memory region we want to migrate to +* @hmmptr: the hmmptr to migrate. +* @start: start(virtual address, inclusive) of the range to migrate +* @end: end(virtual address, exclusive) of the range to migrate +* +* Returns: negative errno on faiure, 0 on success +*/ +int drm_svm_migrate_hmmptr_to_vram (struct drm_gpuvm *vm, +struct drm_mem_region *mr, +struct drm_hmmptr *hmmptr,unsigned long start,unsigned long end) +{ +struct drm_device *drm = mr->mr_ops.drm_mem_region_get_device (mr); +struct drm_gpuva *gpuva = hmmptr->get_gpuva (hmmptr); +struct mm_struct *mm = vm->mm; +unsigned long npages =__npages_in_range (start, end); +struct vm_area_struct *vas; +struct migrate_vma migrate = { + .start =ALIGN_DOWN (start,PAGE_SIZE ), + .end =ALIGN (end,PAGE_SIZE ), + .pgmap_owner = mr->mr_ops.drm_mem_region_pagemap_owner (mr), + .flags =MIGRATE_VMA_SELECT_SYSTEM , + }; +struct device *dev = drm->dev; +struct dma_fence *fence; +struct migrate_vec *src; +struct migrate_vec *dst; +int ret =0 ; +void *buf; + +mmap_assert_locked (mm); + +BUG_ON (start <GPUVA_START (gpuva)); +BUG_ON (end >GPUVA_END (gpuva)); + + vas =find_vma_intersection (mm, start, end); +if (!vas) +return -ENOENT ; + + migrate.vma = vas; + buf =kvcalloc (npages,2 *sizeof (*migrate.src),GFP_KERNEL ); +if (!buf) +return -ENOMEM ; + + migrate.src = buf; + migrate.dst = migrate.src + npages; + ret =migrate_vma_setup (&migrate); +if (ret) { +drm_warn (drm,"vma setup returned %d for range [%lx - %lx]\n" , + ret, start, end); + goto free_buf; + } + +/** +* Partial migration is just normal. Print a message for now. +* Once this behavior is verified, delete this warning. +*/ +if (migrate.cpages != npages) +drm_warn (drm,"Partial migration for range [%lx - %lx], range is %ld pages, migrate only %ld pages\n" , + start, end, npages, migrate.cpages); + + ret = mr->mr_ops.drm_mem_region_alloc_pages (mr, migrate.cpages, migrate.dst); +if (ret) { +memset (migrate.dst,0 ,sizeof (*migrate.dst)); + goto migrate_pages; + } + +__drm_svm_init_device_pages (migrate.dst, migrate.cpages); + + src =__generate_migrate_vec_sram (dev, migrate.src, true, npages); +if (!src) { + ret =VM_FAULT_OOM ; + goto free_device_pages; + } + + dst =__generate_migrate_vec_vram (migrate.dst, false, migrate.cpages); +if (!dst) { + ret =VM_FAULT_OOM ; + goto free_migrate_src; + } + + fence = mr->mr_ops.drm_mem_region_migrate (src, dst); +if (IS_ERR (fence)) { + ret =VM_FAULT_SIGBUS ; + goto free_migrate_dst; + } +dma_fence_wait (fence, false); +dma_fence_put (fence); + +free_migrate_dst: +__free_migrate_vec_vram (dst); +free_migrate_src: +__free_migrate_vec_sram (dev, src, true); + +free_device_pages: +if (ret) +__drm_svm_free_pages (migrate.dst, migrate.cpages); +migrate_pages: +migrate_vma_pages (&migrate); +migrate_vma_finalize (&migrate); +free_buf: +kvfree (buf); +return ret; +} +EXPORT_SYMBOL_GPL (drm_svm_migrate_hmmptr_to_vram);
include/drm/drm_svm.h +3 -0
@@ -211,3 +211,6 @@ void drm_svm_hmmptr_release(struct drm_hmmptr *hmmptr);void drm_svm_hmmptr_map_dma_pages (struct drm_hmmptr *hmmptr,u64 page_idx,u64 npages);int drm_svm_hmmptr_populate (struct drm_hmmptr *hmmptr,void *owner,u64 start,u64 end,bool write); +int drm_svm_migrate_hmmptr_to_vram (struct drm_gpuvm *vm, +struct drm_mem_region *mr, +struct drm_hmmptr *hmmptr,unsigned long start,unsigned long end);
Signature · 1 line
2.26.3
PATCH v3 15/26 drm/svm: Add DRM SVM documentation
In reply to Oak ZengThe purpose of DRM SVM design is to provide helpers and infrastructure to facilitate driver implementation of SVM (shared virtual memory) function.
The overview part described the helpers that DRM layer provdes and the virtual function interfaces that driver has to implement if driver use drm SVM layer to implement SVM functionality.
Unlike legacy DRM BO (buffer object) based physical memory management, DRM SVM use a complete page centric memory management. This has fundamental impact to the memory allocate, free, eviction and migration logic. It also requires the driver to support range (which is required to be page boundary) based GPU page table update and invalidation. The page centric design section described this design.
Lock design is one of the key of the design, which is described in the lock design section.
There are other sections described the memory attributes API design and the memory eviction design.
Create a DRM SVM section in drm-mm.rst and generate SVM documents under this document section.
--- Documentation/gpu/drm-mm.rst | 42 +++++ drivers/gpu/drm/drm_svm.c | 309 +++++++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+)
Documentation/gpu/drm-mm.rst +42 -0
@@ -573,3 +573,45 @@ Scheduler Function References .. kernel-doc:: drivers/gpu/drm/scheduler/sched_entity.c :export: + +DRM SVM +====== + +Overview +------- + +.. kernel-doc:: drivers/gpu/drm/drm_svm.c + :doc: Overview + +Page centric design +------------------ + +.. kernel-doc:: drivers/gpu/drm/drm_svm.c + :doc: page centric design + +Lock design +----------- + +.. kernel-doc:: drivers/gpu/drm/drm_svm.c + :doc: lock design + +Memory hints +----------- + +.. kernel-doc:: drivers/gpu/drm/drm_svm.c + :doc: Memory hints + +Memory eviction +----------- + +.. kernel-doc:: drivers/gpu/drm/drm_svm.c + :doc: pag granularity eviction + +Function References +----------------------------- + +.. kernel-doc:: include/drm/drm_svm.h + :internal: + +.. kernel-doc:: drivers/gpu/drm/drm_svm.c + :export:
drivers/gpu/drm/drm_svm.c +309 -0
315 lines not on this page · raw message
Signature · 1 line
2.26.3
PATCH v3 16/26 drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
Engine is old nomenclature, replace with exec queue.
--- drivers/gpu/drm/xe/xe_migrate.c | 9 ++++----- drivers/gpu/drm/xe/xe_migrate.h | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-)
drivers/gpu/drm/xe/xe_migrate.c +4 -5
@@ -84,15 +84,14 @@ struct xe_migrate {#define MAX_PTE_PER_SDI 0x1FE/** -* xe_tile_migrate_engine() - Get this tile's migrate engine. +* xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue. * @tile: The tile. * -* Returns the default migrate engine of this tile. -* TODO: Perhaps this function is slightly misplaced, and even unneeded? +* Returns the default migrate exec queue of this tile. * -* Return: The default migrate engine +* Return: The default migrate exec queue */ -struct xe_exec_queue *xe_tile_migrate_engine (struct xe_tile *tile) +struct xe_exec_queue *xe_tile_migrate_exec_queue (struct xe_tile *tile) {return tile->migrate->q; }
drivers/gpu/drm/xe/xe_migrate.h +1 -1
@@ -106,5 +106,5 @@ xe_migrate_update_pgtables(struct xe_migrate *m,void xe_migrate_wait (struct xe_migrate *m); -struct xe_exec_queue *xe_tile_migrate_engine (struct xe_tile *tile); +struct xe_exec_queue *xe_tile_migrate_exec_queue (struct xe_tile *tile);#endif
Signature · 1 line
2.26.3
PATCH v3 17/26 drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
Each xe_vma_op resolves to 0-3 pt_ops. Add storage for the pt_ops to xe_vma_ops which is dynamically allocated based the number and types of xe_vma_op in the xe_vma_ops list. Allocation only implemented in this patch.
This will help with converting xe_vma_ops (multiple xe_vma_op) in a atomic update unit.
--- drivers/gpu/drm/xe/xe_pt_types.h | 12 ++++++ drivers/gpu/drm/xe/xe_vm.c | 66 +++++++++++++++++++++++++++++++- drivers/gpu/drm/xe/xe_vm_types.h | 8 ++++ 3 files changed, 84 insertions(+), 2 deletions(-)
drivers/gpu/drm/xe/xe_pt_types.h +12 -0
@@ -74,4 +74,16 @@ struct xe_vm_pgtable_update { u32 flags; }; +/** struct xe_vm_pgtable_update_op - Page table update operation */ +struct xe_vm_pgtable_update_op { +/** @entries: entries to update for this operation */ +struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL *2 +1 ]; +/** @num_entries: number of entries for this update operation */ +u32 num_entries; +/** @bind: is a bind */ +bool bind; +/** @rebind: is a rebind */ +bool rebind; +}; +#endif
drivers/gpu/drm/xe/xe_vm.c +64 -2
143 lines not on this page · raw message
drivers/gpu/drm/xe/xe_vm_types.h +8 -0
@@ -21,6 +21,7 @@ struct xe_bo;struct xe_sync_entry ;struct xe_user_fence ;struct xe_vm ; +struct xe_vm_pgtable_update_op ;#define XE_VMA_READ_ONLY DRM_GPUVA_USERBITS#define XE_VMA_DESTROYED (DRM_GPUVA_USERBITS << 1)
@@ -368,6 +369,13 @@ struct xe_vma_ops {struct xe_sync_entry *syncs;/** @num_syncs: number of syncs */ u32 num_syncs; +/** @pt_update_ops: page table update operations */ +struct { +/** @ops: operations */ +struct xe_vm_pgtable_update_op *ops; +/** @num_ops: number of operations */ +u32 num_ops; + } pt_update_ops[XE_MAX_TILES_PER_DEVICE]; };#endif
Signature · 1 line
2.26.3
PATCH v3 18/26 drm/xe: Convert multiple bind ops into single job
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
This aligns with the uAPI of an array of binds or single bind that results in multiple GPUVA ops to be considered a single atomic operations.
The implemenation is roughly:
- xe_vma_ops is a list of xe_vma_op (GPUVA op)
- each xe_vma_op resolves to 0-3 PT ops
- xe_vma_ops creates a single job
- if at any point during binding a failure occurs, xe_vma_ops contains the information necessary unwind the PT and VMA (GPUVA) state
v2:
- add missing dma-resv slot reservation (CI, testing)
--- drivers/gpu/drm/xe/xe_bo_types.h | 2 + drivers/gpu/drm/xe/xe_migrate.c | 301 ++++---- drivers/gpu/drm/xe/xe_migrate.h | 32 +- drivers/gpu/drm/xe/xe_pt.c | 1108 +++++++++++++++++++----------- drivers/gpu/drm/xe/xe_pt.h | 14 +- drivers/gpu/drm/xe/xe_pt_types.h | 36 + drivers/gpu/drm/xe/xe_vm.c | 519 +++----------- drivers/gpu/drm/xe/xe_vm.h | 2 + drivers/gpu/drm/xe/xe_vm_types.h | 45 +- 9 files changed, 1036 insertions(+), 1023 deletions(-)
drivers/gpu/drm/xe/xe_bo_types.h +2 -0
@@ -58,6 +58,8 @@ struct xe_bo {#endif /** @freed: List node for delayed put. */ struct llist_node freed; +/** @update_index: Update index if PT BO */ +int update_index;/** @created: Whether the bo has passed initial creation */ bool created;
drivers/gpu/drm/xe/xe_migrate.c +137 -164
415 lines not on this page · raw message
drivers/gpu/drm/xe/xe_migrate.h +20 -12
@@ -47,6 +47,24 @@ struct xe_migrate_pt_update_ops { struct xe_tile *tile, struct iosys_map *map, void *pos, u32 ofs, u32 num_qwords, const struct xe_vm_pgtable_update *update); +/** +* @clear: Clear a command buffer or page-table with ptes. +* @pt_update: Embeddable callback argument. +* @tile: The tile for the current operation. +* @map: struct iosys_map into the memory to be populated. +* @pos: If @map is NULL, map into the memory to be populated. +* @ofs: qword offset into @map, unused if @map is NULL. +* @num_qwords: Number of qwords to write. +* @update: Information about the PTEs to be inserted. +* +* This interface is intended to be used as a callback into the +* page-table system to populate command buffers or shared +* page-tables with PTEs. +*/ +void (*clear)(struct xe_migrate_pt_update *pt_update, +struct xe_tile *tile,struct iosys_map *map, +void *pos,u32 ofs,u32 num_qwords, +const struct xe_vm_pgtable_update *update);/** * @pre_commit: Callback to be called just before arming the
@@ -67,14 +85,10 @@ struct xe_migrate_pt_update_ops {struct xe_migrate_pt_update {/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */ const struct xe_migrate_pt_update_ops *ops; -/** @vma: The vma we're updating the pagetable for. */ -struct xe_vma *vma; +/** @vops: VMA operations */ +struct xe_vma_ops *vops;/** @job: The job if a GPU page-table update. NULL otherwise */ struct xe_sched_job *job; -/** @start: Start of update for the range fence */ -u64 start; -/** @last: Last of update for the range fence */ -u64 last;/** @tile_id: Tile ID of the update */ u8 tile_id; };
@@ -96,12 +110,6 @@ struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);struct dma_fence *xe_migrate_update_pgtables (struct xe_migrate *m, -struct xe_vm *vm, -struct xe_bo *bo, -struct xe_exec_queue *q, -const struct xe_vm_pgtable_update *updates, -u32 num_updates, -struct xe_sync_entry *syncs,u32 num_syncs,struct xe_migrate_pt_update *pt_update);void xe_migrate_wait (struct xe_migrate *m);
drivers/gpu/drm/xe/xe_pt.c +706 -402
1,260 lines not on this page · raw message
drivers/gpu/drm/xe/xe_pt.h +6 -8
@@ -17,6 +17,7 @@ struct xe_sync_entry;struct xe_tile ;struct xe_vm ;struct xe_vma ; +struct xe_vma_ops ;/* Largest huge pte is currently 1GiB. May become device dependent. */ #define MAX_HUGEPTE_LEVEL 2
@@ -34,14 +35,11 @@ void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,void xe_pt_destroy (struct xe_pt *pt,u32 flags,struct llist_head *deferred); -struct dma_fence * -__xe_pt_bind_vma (struct xe_tile *tile,struct xe_vma *vma,struct xe_exec_queue *q, -struct xe_sync_entry *syncs,u32 num_syncs, -bool rebind); - -struct dma_fence * -__xe_pt_unbind_vma (struct xe_tile *tile,struct xe_vma *vma,struct xe_exec_queue *q, -struct xe_sync_entry *syncs,u32 num_syncs); +int xe_pt_update_ops_prepare (struct xe_tile *tile,struct xe_vma_ops *vops); +struct dma_fence *xe_pt_update_ops_run (struct xe_tile *tile, +struct xe_vma_ops *vops); +void xe_pt_update_ops_fini (struct xe_tile *tile,struct xe_vma_ops *vops); +void xe_pt_update_ops_abort (struct xe_tile *tile,struct xe_vma_ops *vops);bool xe_pt_zap_ptes (struct xe_tile *tile,struct xe_vma *vma);
drivers/gpu/drm/xe/xe_pt_types.h +36 -0
@@ -78,6 +78,8 @@ struct xe_vm_pgtable_update {struct xe_vm_pgtable_update_op {/** @entries: entries to update for this operation */ struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL *2 +1 ]; +/** @vma: VMA for operation, operation not valid if NULL */ +struct xe_vma *vma;/** @num_entries: number of entries for this update operation */ u32 num_entries;/** @bind: is a bind */
@@ -86,4 +88,38 @@ struct xe_vm_pgtable_update_op { bool rebind; }; +/** struct xe_vm_pgtable_update_ops: page table update operations */ +struct xe_vm_pgtable_update_ops { +/** @ops: operations */ +struct xe_vm_pgtable_update_op *ops; +/** @deferred: deferred list to destroy PT entries */ +struct llist_head deferred; +/** @q: exec queue for PT operations */ +struct xe_exec_queue *q; +/** @start: start address of ops */ +u64 start; +/** @last: last address of ops */ +u64 last; +/** @num_ops: number of operations */ +u32 num_ops; +/** @current_op: current operations */ +u32 current_op; +/** @needs_userptr_lock: Needs userptr lock */ +bool needs_userptr_lock; +/** @needs_invalidation: Needs invalidation */ +bool needs_invalidation; +/** +* @wait_vm_bookkeep: PT operations need to wait until VM is idle +* (bookkeep dma-resv slots are idle) and stage all future VM activity +* behind these operations (install PT operations into VM kernel +* dma-resv slot). +*/ +bool wait_vm_bookkeep; +/** +* @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv +* slots are idle. +*/ +bool wait_vm_kernel; +}; +#endif
drivers/gpu/drm/xe/xe_vm.c +113 -406
633 lines not on this page · raw message
drivers/gpu/drm/xe/xe_vm.h +2 -0
@@ -259,6 +259,8 @@ static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)return drm_gpuvm_resv (&vm->gpuvm); } +void xe_vm_kill (struct xe_vm *vm,bool unlocked); +/** * xe_vm_assert_held(vm) - Assert that the vm's reservation object is held. * @vm: The vm
drivers/gpu/drm/xe/xe_vm_types.h +14 -31
84 lines not on this page · raw message
Signature · 1 line
2.26.3
PATCH v3 19/26 drm/xe: Update VM trace events
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
The trace events have changed moving to a single job per VM bind IOCTL, update the trace events align with old behavior as much as possible.
--- drivers/gpu/drm/xe/xe_trace.h | 10 ++++----- drivers/gpu/drm/xe/xe_vm.c | 42 +++++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-)
drivers/gpu/drm/xe/xe_trace.h +5 -5
@@ -426,11 +426,6 @@ DEFINE_EVENT(xe_vma, xe_vma_acc,TP_ARGS (vma) ); -DEFINE_EVENT (xe_vma, xe_vma_fail, - TP_PROTO(struct xe_vma *vma), -TP_ARGS (vma) -); -DEFINE_EVENT (xe_vma, xe_vma_bind, TP_PROTO(struct xe_vma *vma), TP_ARGS(vma)
@@ -544,6 +539,11 @@ DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_exit,TP_ARGS (vm) ); +DEFINE_EVENT (xe_vm, xe_vm_ops_fail, + TP_PROTO(struct xe_vm *vm), +TP_ARGS (vm) +); +/* GuC */ DECLARE_EVENT_CLASS(xe_guc_ct_flow_control, TP_PROTO(u32 _head, u32 _tail, u32 size, u32 space, u32 len),
drivers/gpu/drm/xe/xe_vm.c +40 -2
66 lines not on this page · raw message
Signature · 1 line
2.26.3
PATCH v3 20/26 drm/xe: Update PT layer with better error handling
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
Update PT layer so if a memory allocation for a PTE fails the error can be propagated to the user without requiring to be killed.
--- drivers/gpu/drm/xe/xe_pt.c | 208 ++++++++++++++++++++++++++++--------- 1 file changed, 160 insertions(+), 48 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +160 -48
@@ -845,19 +845,27 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *t } } -static void xe_pt_abort_bind (struct xe_vma *vma, -struct xe_vm_pgtable_update *entries, -u32 num_entries) +static void xe_pt_cancel_bind (struct xe_vma *vma, +struct xe_vm_pgtable_update *entries, +u32 num_entries) {u32 i, j;for (i =0 ; i < num_entries; i++) { -if (!entries[i].pt_entries) +struct xe_pt *pt = entries[i].pt; + +if (!pt)continue ; -for (j =0 ; j < entries[i].qwords; j++) -xe_pt_destroy (entries[i].pt_entries[j].pt,xe_vma_vm (vma)->flags,NULL ); +if (pt->level) { +for (j =0 ; j < entries[i].qwords; j++) +xe_pt_destroy (entries[i].pt_entries[j].pt, +xe_vma_vm (vma)->flags,NULL ); + } +kfree (entries[i].pt_entries); + entries[i].pt_entries =NULL ; + entries[i].qwords =0 ; } }
@@ -873,10 +881,61 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma)xe_vm_assert_held (vm); } -static void xe_pt_commit_bind (struct xe_vma *vma, -struct xe_vm_pgtable_update *entries, -u32 num_entries,bool rebind, -struct llist_head *deferred) +static void xe_pt_commit (struct xe_vma *vma, +struct xe_vm_pgtable_update *entries, +u32 num_entries,struct llist_head *deferred) +{ +u32 i, j; + +xe_pt_commit_locks_assert (vma); + +for (i =0 ; i < num_entries; i++) { +struct xe_pt *pt = entries[i].pt; + +if (!pt->level) +continue ; + +for (j =0 ; j < entries[i].qwords; j++) { +struct xe_pt *oldpte = entries[i].pt_entries[j].pt; + +xe_pt_destroy (oldpte,xe_vma_vm (vma)->flags, deferred); + } + } +} + +static void xe_pt_abort_bind (struct xe_vma *vma, +struct xe_vm_pgtable_update *entries, +u32 num_entries,bool rebind) +{ +int i, j; + +xe_pt_commit_locks_assert (vma); + +for (i = num_entries -1 ; i >=0 ; --i) { +struct xe_pt *pt = entries[i].pt; +struct xe_pt_dir *pt_dir; + +if (!rebind) + pt->num_live -= entries[i].qwords; + +if (!pt->level) +continue ; + + pt_dir =as_xe_pt_dir (pt); +for (j =0 ; j < entries[i].qwords; j++) { +u32 j_ = j + entries[i].ofs; +struct xe_pt *newpte =xe_pt_entry (pt_dir, j_); +struct xe_pt *oldpte = entries[i].pt_entries[j].pt; + + pt_dir->children[j_] = oldpte ? &oldpte->base :0 ; +xe_pt_destroy (newpte,xe_vma_vm (vma)->flags,NULL ); + } + } +} + +static void xe_pt_commit_prepare_bind (struct xe_vma *vma, +struct xe_vm_pgtable_update *entries, +u32 num_entries,bool rebind) {u32 i, j;
@@ -896,12 +955,13 @@ static void xe_pt_commit_bind(struct xe_vma *vma,for (j =0 ; j < entries[i].qwords; j++) {u32 j_ = j + entries[i].ofs;struct xe_pt *newpte = entries[i].pt_entries[j].pt; +struct xe_pt *oldpte =NULL ;if (xe_pt_entry (pt_dir, j_)) -xe_pt_destroy (xe_pt_entry (pt_dir, j_), -xe_vma_vm (vma)->flags, deferred); + oldpte =xe_pt_entry (pt_dir, j_); pt_dir->children[j_] = &newpte->base; + entries[i].pt_entries[j].pt = oldpte; } } }
@@ -925,8 +985,6 @@ xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma, err =xe_pt_stage_bind (tile, vma, entries, num_entries);if (!err)xe_tile_assert (tile, *num_entries); -else /* abort! */ -xe_pt_abort_bind (vma, entries, *num_entries);return err; }
@@ -1447,7 +1505,7 @@ xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, &end_offset))return 0 ; - (void )xe_pt_new_shared (&xe_walk->wupd, xe_child, offset, false); + (void )xe_pt_new_shared (&xe_walk->wupd, xe_child, offset, true); xe_walk->wupd.updates[level].update->qwords = end_offset - offset;return 0 ;
@@ -1515,32 +1573,57 @@ xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,memset64 (ptr, empty, num_qwords); } +static void xe_pt_abort_unbind (struct xe_vma *vma, +struct xe_vm_pgtable_update *entries, +u32 num_entries) +{ +int j, i; + +xe_pt_commit_locks_assert (vma); + +for (j = num_entries -1 ; j >=0 ; --j) { +struct xe_vm_pgtable_update *entry = &entries[j]; +struct xe_pt *pt = entry->pt; +struct xe_pt_dir *pt_dir =as_xe_pt_dir (pt); + + pt->num_live += entry->qwords; + +if (!pt->level) +continue ; + +for (i = entry->ofs; i < entry->ofs + entry->qwords; i++) + pt_dir->children[i] = + entries[j].pt_entries[i - entry->ofs].pt ? + &entries[j].pt_entries[i - entry->ofs].pt->base :0 ; + } +} +static void -xe_pt_commit_unbind (struct xe_vma *vma, -struct xe_vm_pgtable_update *entries,u32 num_entries, -struct llist_head *deferred) +xe_pt_commit_prepare_unbind (struct xe_vma *vma, +struct xe_vm_pgtable_update *entries, +u32 num_entries) { -u32 j; +int j, i;xe_pt_commit_locks_assert (vma);for (j =0 ; j < num_entries; ++j) {struct xe_vm_pgtable_update *entry = &entries[j];struct xe_pt *pt = entry->pt; +struct xe_pt_dir *pt_dir; pt->num_live -= entry->qwords; -if (pt->level) { -struct xe_pt_dir *pt_dir =as_xe_pt_dir (pt); -u32 i; - -for (i = entry->ofs; i < entry->ofs + entry->qwords; - i++) { -if (xe_pt_entry (pt_dir, i)) -xe_pt_destroy (xe_pt_entry (pt_dir, i), -xe_vma_vm (vma)->flags, deferred); +if (!pt->level) +continue ; - pt_dir->children[i] =NULL ; - } + pt_dir =as_xe_pt_dir (pt); +for (i = entry->ofs; i < entry->ofs + entry->qwords; i++) { +if (xe_pt_entry (pt_dir, i)) + entries[j].pt_entries[i - entry->ofs].pt = +xe_pt_entry (pt_dir, i); +else + entries[j].pt_entries[i - entry->ofs].pt =NULL ; + pt_dir->children[i] =NULL ; } } }
@@ -1586,7 +1669,6 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, {u32 current_op = pt_update_ops->current_op;struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; -struct llist_head *deferred = &pt_update_ops->deferred;int err;xe_bo_assert_held (xe_vma_bo (vma));
@@ -1633,11 +1715,12 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, (!pt_op->rebind && vm->scratch_pt[tile->id] &&xe_vm_in_preempt_fence_mode (vm)); -/* FIXME: Don't commit right away */ vma->tile_staged |=BIT (tile->id); pt_op->vma = vma; -xe_pt_commit_bind (vma, pt_op->entries, pt_op->num_entries, - pt_op->rebind, deferred); +xe_pt_commit_prepare_bind (vma, pt_op->entries, + pt_op->num_entries, pt_op->rebind); + }else { +xe_pt_cancel_bind (vma, pt_op->entries, pt_op->num_entries); }return err;
@@ -1649,7 +1732,6 @@ static int unbind_op_prepare(struct xe_tile *tile, {u32 current_op = pt_update_ops->current_op;struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; -struct llist_head *deferred = &pt_update_ops->deferred;int err;if (!((vma->tile_present | vma->tile_staged) &BIT (tile->id)))
@@ -1685,9 +1767,7 @@ static int unbind_op_prepare(struct xe_tile *tile, pt_update_ops->needs_userptr_lock |=xe_vma_is_userptr (vma); pt_update_ops->needs_invalidation = true; -/* FIXME: Don't commit right away */ -xe_pt_commit_unbind (vma, pt_op->entries, pt_op->num_entries, - deferred); +xe_pt_commit_prepare_unbind (vma, pt_op->entries, pt_op->num_entries);return 0 ; }
@@ -1908,7 +1988,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)struct invalidation_fence *ifence =NULL ;struct xe_range_fence *rfence;struct xe_vma_op *op; -int err =0 ; +int err =0 , i;struct xe_migrate_pt_update update = { .ops = pt_update_ops->needs_userptr_lock ? &userptr_migrate_ops :
@@ -1928,8 +2008,10 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)if (pt_update_ops->needs_invalidation) { ifence =kzalloc (sizeof (*ifence),GFP_KERNEL ); -if (!ifence) -return ERR_PTR (-ENOMEM ); +if (!ifence) { + err = -ENOMEM ; + goto kill_vm_tile1; + } } rfence =kzalloc (sizeof (*rfence),GFP_KERNEL );
@@ -1944,6 +2026,15 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops) goto free_rfence; } +/* Point of no return - VM killed if failure after this */ +for (i =0 ; i < pt_update_ops->current_op; ++i) { +struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; + +xe_pt_commit (pt_op->vma, pt_op->entries, + pt_op->num_entries, &pt_update_ops->deferred); + pt_op->vma =NULL ;/* skip in xe_pt_update_ops_abort */ + } + err = xe_range_fence_insert(&vm->rftree[tile->id], rfence, &xe_range_fence_kfree_ops, pt_update_ops->start,
@@ -1979,10 +2070,15 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)if (pt_update_ops->needs_userptr_lock)up_read (&vm->userptr.notifier_lock);dma_fence_put (fence); +if (!tile->id) +xe_vm_kill (vops->vm, false); free_rfence:kfree (rfence); free_ifence:kfree (ifence); +kill_vm_tile1: +if (err != -EAGAIN && tile->id) +xe_vm_kill (vops->vm, false);return ERR_PTR (err); }
@@ -2003,12 +2099,10 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)lockdep_assert_held (&vops->vm->lock);xe_vm_assert_held (vops->vm); -/* FIXME: Not 100% correct */ -for (i =0 ; i < pt_update_ops->num_ops; ++i) { +for (i =0 ; i < pt_update_ops->current_op; ++i) {struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; -if (pt_op->bind) -xe_pt_free_bind (pt_op->entries, pt_op->num_entries); +xe_pt_free_bind (pt_op->entries, pt_op->num_entries); }xe_bo_put_commit (&vops->pt_update_ops[tile->id].deferred); }
@@ -2022,10 +2116,28 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)*/ void xe_pt_update_ops_abort (struct xe_tile *tile,struct xe_vma_ops *vops) { +struct xe_vm_pgtable_update_ops *pt_update_ops = + &vops->pt_update_ops[tile->id]; +int i; +lockdep_assert_held (&vops->vm->lock);xe_vm_assert_held (vops->vm); -/* FIXME: Just kill VM for now + cleanup PTs */ +for (i = pt_update_ops->num_ops -1 ; i >=0 ; --i) { +struct xe_vm_pgtable_update_op *pt_op = + &pt_update_ops->ops[i]; + +if (!pt_op->vma || i >= pt_update_ops->current_op) +continue ; + +if (pt_op->bind) +xe_pt_abort_bind (pt_op->vma, pt_op->entries, + pt_op->num_entries, + pt_op->rebind); +else +xe_pt_abort_unbind (pt_op->vma, pt_op->entries, + pt_op->num_entries); + } +xe_bo_put_commit (&vops->pt_update_ops[tile->id].deferred); -xe_vm_kill (vops->vm, false); - } +}
Signature · 1 line
2.26.3
PATCH v3 21/26 drm/xe: Retry BO allocation
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
TTM doesn't support fair eviction via WW locking, this mitigated in by using retry loops in exec and preempt rebind worker. Extend this retry loop to BO allocation. Once TTM supports fair eviction this patch can be reverted.
--- drivers/gpu/drm/xe/xe_bo.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/xe/xe_bo.c +7 -1
@@ -1930,6 +1930,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,struct xe_file *xef =to_xe_file (file);struct drm_xe_gem_create *args = data;struct xe_vm *vm =NULL ; +ktime_t end =0 ;struct xe_bo *bo;unsigned int bo_flags;u32 handle;
@@ -1995,11 +1996,14 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, vm =xe_vm_lookup (xef, args->vm_id);if (XE_IOCTL_DBG (xe, !vm))return -ENOENT ; + } + +retry: +if (vm) { err =xe_vm_lock (vm, true);if (err) goto out_vm; } - bo =xe_bo_create_user (xe,NULL , vm, args->size, args->cpu_caching, ttm_bo_type_device, bo_flags);
@@ -2008,6 +2012,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,if (IS_ERR (bo)) { err =PTR_ERR (bo); +if (xe_vm_validate_should_retry (NULL , err, &end)) + goto retry; goto out_vm; }
Signature · 1 line
2.26.3
PATCH v3 22/26 drm/xe: Rework GPU page fault handling
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
Add helper function to implement VMA (user binding) page faults, remove unnecessary userptr.invalidate_link list del operation, retry on memory pressure, remove unnecessary xe_vma_userptr_check_repin after rebinding, remove unnecessary TLB invalidation, and always use vm->lock in write mode. Changes help facilitate SVM page faults.
--- drivers/gpu/drm/xe/xe_gt_pagefault.c | 148 +++++++++++++-------------- 1 file changed, 69 insertions(+), 79 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +69 -79
@@ -125,126 +125,116 @@ static int xe_pf_begin(struct drm_exec *exec, struct xe_vma *vma,return 0 ; } -static int handle_pagefault (struct xe_gt *gt,struct pagefault *pf) +static int handle_vma_pagefault (struct xe_tile *tile,struct pagefault *pf, +struct xe_vma *vma) { -struct xe_device *xe =gt_to_xe (gt); -struct xe_tile *tile =gt_to_tile (gt); +struct xe_vm *vm =xe_vma_vm (vma);struct drm_exec exec; -struct xe_vm *vm; -struct xe_vma *vma =NULL ;struct dma_fence *fence; -bool write_locked; -int ret =0 ; +ktime_t end =0 ; +int err;bool atomic; -/* SW isn't expected to handle TRTT faults */ -if (pf->trva_fault) -return -EFAULT ; - -/* ASID to VM */ -mutex_lock (&xe->usm.lock); - vm =xa_load (&xe->usm.asid_to_vm, pf->asid); -if (vm &&xe_vm_in_fault_mode (vm)) -xe_vm_get (vm); -else - vm =NULL ; -mutex_unlock (&xe->usm.lock); -if (!vm) -return -EINVAL ; - -retry_userptr: -/* -* TODO: Avoid exclusive lock if VM doesn't have userptrs, or -* start out read-locked? -*/ -down_write (&vm->lock); - write_locked = true; - vma =lookup_vma (vm, pf->page_addr); -if (!vma) { - ret = -EINVAL ; - goto unlock_vm; - } - -if (!xe_vma_is_userptr (vma) || - !xe_vma_userptr_check_repin (to_userptr_vma (vma))) { -downgrade_write (&vm->lock); - write_locked = false; - } +lockdep_assert_held_write (&vm->lock);trace_xe_vma_pagefault (vma); - atomic =access_is_atomic (pf->access_type);/* Check if VMA is valid */ if (vma_is_valid (tile, vma) && !atomic) - goto unlock_vm; - -/* TODO: Validate fault */ +return 0 ; -if (xe_vma_is_userptr (vma) && write_locked) { +retry_userptr: +if (xe_vma_is_userptr (vma) && +xe_vma_userptr_check_repin (to_userptr_vma (vma))) {struct xe_userptr_vma *uvma =to_userptr_vma (vma); -spin_lock (&vm->userptr.invalidated_lock); -list_del_init (&uvma->userptr.invalidate_link); -spin_unlock (&vm->userptr.invalidated_lock); - - ret =xe_vma_userptr_pin_pages (uvma); -if (ret) - goto unlock_vm; - -downgrade_write (&vm->lock); - write_locked = false; + err =xe_vma_userptr_pin_pages (uvma); +if (err) +return err; }/* Lock VM and BOs dma-resv */ drm_exec_init (&exec,0 ,0 ); drm_exec_until_all_locked(&exec) { - ret =xe_pf_begin (&exec, vma, atomic, tile->id); + err =xe_pf_begin (&exec, vma, atomic, tile->id);drm_exec_retry_on_contention (&exec); -if (ret) +if (xe_vm_validate_should_retry (&exec, err, &end)) + err = -EAGAIN ; +if (err) goto unlock_dma_resv;/* Bind VMA only to the GT that has faulted */ trace_xe_vma_pf_bind (vma); fence =xe_vma_rebind (vm, vma,BIT (tile->id));if (IS_ERR (fence)) { - ret =PTR_ERR (fence); + err =PTR_ERR (fence); +if (xe_vm_validate_should_retry (&exec, err, &end)) + err = -EAGAIN ; goto unlock_dma_resv; } } -/* -* XXX: Should we drop the lock before waiting? This only helps if doing -* GPU binds which is currently only done if we have to wait for more -* than 10ms on a move. -*/ dma_fence_wait (fence, false);dma_fence_put (fence); - -if (xe_vma_is_userptr (vma)) - ret =xe_vma_userptr_check_repin (to_userptr_vma (vma)); vma->tile_invalidated &= ~BIT (tile->id); unlock_dma_resv:drm_exec_fini (&exec); -unlock_vm: -if (!ret) - vm->usm.last_fault_vma = vma; -if (write_locked) -up_write (&vm->lock); -else -up_read (&vm->lock); -if (ret == -EAGAIN ) +if (err == -EAGAIN ) goto retry_userptr; -if (!ret) { - ret =xe_gt_tlb_invalidation_vma (gt,NULL , vma); -if (ret >=0 ) - ret =0 ; +return err; +} + +static int handle_pagefault (struct xe_gt *gt,struct pagefault *pf) +{ +struct xe_device *xe =gt_to_xe (gt); +struct xe_tile *tile =gt_to_tile (gt); +struct xe_vm *vm; +struct xe_vma *vma =NULL ; +int err; + +/* SW isn't expected to handle TRTT faults */ +if (pf->trva_fault) +return -EFAULT ; + +/* ASID to VM */ +mutex_lock (&xe->usm.lock); + vm =xa_load (&xe->usm.asid_to_vm, pf->asid); +if (vm &&xe_vm_in_fault_mode (vm)) +xe_vm_get (vm); +else + vm =NULL ; +mutex_unlock (&xe->usm.lock); +if (!vm) +return -EINVAL ; + +/* +* TODO: Change to read lock? Using write lock for simplicity. +*/ +down_write (&vm->lock); + +if (xe_vm_is_closed_or_banned (vm)) { + err = -ENOENT ; + goto unlock_vm; + } + + vma =lookup_vma (vm, pf->page_addr); +if (!vma) { + err = -EINVAL ; + goto unlock_vm; } + + err =handle_vma_pagefault (tile, pf, vma); + +unlock_vm: +if (!err) + vm->usm.last_fault_vma = vma; +up_write (&vm->lock);xe_vm_put (vm); -return ret; +return err; } static int send_pagefault_reply(struct xe_guc *guc,
Signature · 1 line
2.26.3
PATCH v3 23/26 drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
Add the DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag, which is used to create unpopulated virtual memory areas (VMAs) without memory backing or GPU page tables. These VMAs are referred to as system allocator VMAs. The idea is that upon a page fault, the memory backing and GPU page tables will be populated.
System allocator VMAs only update GPUVM state; they do not have an internal page table (PT) state, nor do they have GPU mappings.
It is expected that system allocator VMAs will be mixed with buffer object (BO) VMAs within a single VM. In other words, system allocations and runtime allocations can be mixed within a single user-mode driver (UMD) program.
Expected usage:
- Bind the entire virtual address (VA) space upon program load using the DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag and DRM_XE_VM_BIND_OP_MAP op.
- If a buffer object (BO) requires GPU mapping, reserve an address range using mmap, unbind the address range from system allocator using DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag and DRM_XE_VM_BIND_OP_UNMAP op, and bind the BO to the reserved address using existing bind IOCTLs (runtime allocation).
- If a BO no longer requires GPU mapping, bind the mapping address with the DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag.
- Any malloc'd address accessed by the GPU will be faulted in via the SVM implementation (system allocation).
- Upon freeing any malloc'd data, the SVM implementation will remove GPU mappings.
v1:
fix commit message (Oak)
rebase with lastest xekmd, fix conflict in xe_analyze_vm (Oak)
FIXME: Not enforcing DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR for fault mode VMs
FIXME: Only supporting 1 to 1 mapping between user address space and GPU address space
--- drivers/gpu/drm/xe/xe_pt.c | 73 +++++++++++++++++---- drivers/gpu/drm/xe/xe_vm.c | 107 ++++++++++++++++++++----------- drivers/gpu/drm/xe/xe_vm.h | 8 ++- drivers/gpu/drm/xe/xe_vm_types.h | 3 + include/uapi/drm/xe_drm.h | 11 +++- 5 files changed, 147 insertions(+), 55 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +59 -14
@@ -1064,6 +1064,11 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, {int err =0 ; +/* +* No need to check for is_system_allocator here as vma_add_deps is a +* NOP if VMA is_system_allocator +*/ +switch (op->base.op) {case DRM_GPUVA_OP_MAP :if (!op->map.immediate &&xe_vm_in_fault_mode (vm))
@@ -1671,6 +1676,7 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];int err; +xe_tile_assert (tile, !xe_vma_is_system_allocator (vma));xe_bo_assert_held (xe_vma_bo (vma)); vm_dbg(&xe_vma_vm(vma)->xe->drm,
@@ -1737,6 +1743,7 @@ static int unbind_op_prepare(struct xe_tile *tile,if (!((vma->tile_present | vma->tile_staged) &BIT (tile->id)))return 0 ; +xe_tile_assert (tile, !xe_vma_is_system_allocator (vma));xe_bo_assert_held (xe_vma_bo (vma)); vm_dbg(&xe_vma_vm(vma)->xe->drm,
@@ -1783,15 +1790,21 @@ static int op_prepare(struct xe_vm *vm,switch (op->base.op) {case DRM_GPUVA_OP_MAP : -if (!op->map.immediate &&xe_vm_in_fault_mode (vm)) +if ((!op->map.immediate &&xe_vm_in_fault_mode (vm)) || + op->map.is_system_allocator)break ; err =bind_op_prepare (vm, tile, pt_update_ops, op->map.vma); pt_update_ops->wait_vm_kernel = true;break ;case DRM_GPUVA_OP_REMAP : - err =unbind_op_prepare (tile, pt_update_ops, -gpuva_to_vma (op->base.remap.unmap->va)); + { +struct xe_vma *old =gpuva_to_vma (op->base.remap.unmap->va); + +if (xe_vma_is_system_allocator (old)) +break ; + + err =unbind_op_prepare (tile, pt_update_ops, old);if (!err && op->remap.prev) { err = bind_op_prepare(vm, tile, pt_update_ops,
@@ -1804,15 +1817,28 @@ static int op_prepare(struct xe_vm *vm, pt_update_ops->wait_vm_bookkeep = true; }break ; + }case DRM_GPUVA_OP_UNMAP : - err =unbind_op_prepare (tile, pt_update_ops, -gpuva_to_vma (op->base.unmap.va)); + { +struct xe_vma *vma =gpuva_to_vma (op->base.unmap.va); + +if (xe_vma_is_system_allocator (vma)) +break ; + + err =unbind_op_prepare (tile, pt_update_ops, vma);break ; + }case DRM_GPUVA_OP_PREFETCH : - err =bind_op_prepare (vm, tile, pt_update_ops, -gpuva_to_vma (op->base.prefetch.va)); + { +struct xe_vma *vma =gpuva_to_vma (op->base.prefetch.va); + +if (xe_vma_is_system_allocator (vma)) +break ; + + err =bind_op_prepare (vm, tile, pt_update_ops, vma); pt_update_ops->wait_vm_kernel = true;break ; + }default :drm_warn (&vm->xe->drm,"NOT POSSIBLE" ); }
@@ -1873,6 +1899,8 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_vma *vma, struct dma_fence *fence) { + xe_tile_assert(tile, !xe_vma_is_system_allocator(vma)); + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, pt_update_ops->wait_vm_bookkeep ?
@@ -1899,6 +1927,8 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,struct xe_vm_pgtable_update_ops *pt_update_ops, struct xe_vma *vma, struct dma_fence *fence) { + xe_tile_assert(tile, !xe_vma_is_system_allocator(vma)); + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, pt_update_ops->wait_vm_bookkeep ?
@@ -1926,14 +1956,20 @@ static void op_commit(struct xe_vm *vm,switch (op->base.op) {case DRM_GPUVA_OP_MAP : -if (!op->map.immediate &&xe_vm_in_fault_mode (vm)) +if ((!op->map.immediate &&xe_vm_in_fault_mode (vm)) || + op->map.is_system_allocator)break ;bind_op_commit (vm, tile, pt_update_ops, op->map.vma, fence);break ;case DRM_GPUVA_OP_REMAP : -unbind_op_commit (vm, tile, pt_update_ops, -gpuva_to_vma (op->base.remap.unmap->va), fence); + { +struct xe_vma *old =gpuva_to_vma (op->base.remap.unmap->va); + +if (xe_vma_is_system_allocator (old)) +break ; + +unbind_op_commit (vm, tile, pt_update_ops, old, fence);if (op->remap.prev) bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
@@ -1942,14 +1978,23 @@ static void op_commit(struct xe_vm *vm,bind_op_commit (vm, tile, pt_update_ops, op->remap.next, fence);break ; + }case DRM_GPUVA_OP_UNMAP : -unbind_op_commit (vm, tile, pt_update_ops, -gpuva_to_vma (op->base.unmap.va), fence); + { +struct xe_vma *vma =gpuva_to_vma (op->base.unmap.va); + +if (!xe_vma_is_system_allocator (vma)) +unbind_op_commit (vm, tile, pt_update_ops, vma, fence);break ; + }case DRM_GPUVA_OP_PREFETCH : -bind_op_commit (vm, tile, pt_update_ops, -gpuva_to_vma (op->base.prefetch.va), fence); + { +struct xe_vma *vma =gpuva_to_vma (op->base.prefetch.va); + +if (!xe_vma_is_system_allocator (vma)) +bind_op_commit (vm, tile, pt_update_ops, vma, fence);break ; + }default :drm_warn (&vm->xe->drm,"NOT POSSIBLE" ); }
drivers/gpu/drm/xe/xe_vm.c +68 -39
240 lines not on this page · raw message
drivers/gpu/drm/xe/xe_vm.h +7 -1
@@ -151,6 +151,11 @@ static inline bool xe_vma_is_null(struct xe_vma *vma)return vma->gpuva.flags &DRM_GPUVA_SPARSE ; } +static inline bool xe_vma_is_system_allocator (struct xe_vma *vma) +{ +return vma->gpuva.flags &XE_VMA_SYSTEM_ALLOCATOR ; +} +static inline bool xe_vma_has_no_bo (struct xe_vma *vma) {return !xe_vma_bo (vma);
@@ -158,7 +163,8 @@ static inline bool xe_vma_has_no_bo(struct xe_vma *vma)static inline bool xe_vma_is_userptr (struct xe_vma *vma) { -return xe_vma_has_no_bo (vma) && !xe_vma_is_null (vma); +return xe_vma_has_no_bo (vma) && !xe_vma_is_null (vma) && + !xe_vma_is_system_allocator (vma); }/**
drivers/gpu/drm/xe/xe_vm_types.h +3 -0
@@ -32,6 +32,7 @@ struct xe_vm_pgtable_update_op;#define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6)#define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7)#define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) +#define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9)/** struct xe_userptr - User pointer */ struct xe_userptr {
@@ -284,6 +285,8 @@ struct xe_vma_op_map {bool read_only;/** @is_null: is NULL binding */ bool is_null; +/** @is_system_allocator: is system allocator binding */ +bool is_system_allocator;/** @dumpable: whether BO is dumped on GPU hang */ bool dumpable;/** @pat_index: The pat index to use for this operation. */
include/uapi/drm/xe_drm.h +10 -1
@@ -885,6 +885,12 @@ struct drm_xe_vm_destroy {* will only be valid for DRM_XE_VM_BIND_OP_MAP operations, the BO * handle MBZ, and the BO offset MBZ. This flag is intended to * implement VK sparse bindings. +* - %DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR - When the system allocator flag is +* set, no mappings are created rather the range is reserved for system +* allocations which will be populated on GPU page faults. Only valid on VMs +* with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The system allocator flag are +* only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO handle MBZ, and +* the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */
@@ -937,7 +943,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;
@@ -975,6 +983,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_IMMEDIATE (1 << 1)#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3) +#define DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR (1 << 4)/** @flags: Bind flags */ __u32 flags;
Signature · 1 line
2.26.3
PATCH v3 24/26 drm/xe: Add dma_addr res cursor
In reply to Oak ZengFrom: Matthew Brost <[email protected]>
v1: Modify dma_addr res cursor to use new dma map API (Oak)
With the new dma map API, we don't need a dma_address array to hold dma addresses. Instead, dma address is pre-allocated from iova address space. Modify dma_addr res cursor for this purpose.
--- drivers/gpu/drm/xe/xe_res_cursor.h | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-)
drivers/gpu/drm/xe/xe_res_cursor.h +37 -1
@@ -46,6 +46,7 @@ struct xe_res_cursor {u32 mem_type;struct scatterlist *sgl;struct drm_buddy *mm; +bool dma_addr; };static struct drm_buddy *xe_res_get_buddy (struct ttm_resource *res)
@@ -71,6 +72,7 @@ static inline void xe_res_first(struct ttm_resource *res, struct xe_res_cursor *cur) { cur->sgl =NULL ; + cur->dma_addr = false;if (!res) goto fallback;
@@ -158,6 +160,7 @@ static inline void xe_res_first_sg(const struct sg_table *sg, {XE_WARN_ON (!sg); cur->node =NULL ; + cur->dma_addr = false; cur->start = start; cur->remaining = size; cur->size =0 ;
@@ -166,6 +169,29 @@ static inline void xe_res_first_sg(const struct sg_table *sg,__xe_res_sg_next (cur); } +/** +* xe_res_first_dma - initialize a xe_res_cursor with dma_addr +* +* @dma_addr: dma address +* @start: Start of the range +* @size: Size of the range +* @cur: cursor object to initialize +* +* Start walking over the range of allocations between @start and @size. +*/ +static inline void xe_res_first_dma (const dma_addr_t dma_addr, +u64 size,struct xe_res_cursor *cur) +{ +XE_WARN_ON (!IS_ALIGNED (size,PAGE_SIZE )); + + cur->node =NULL ; + cur->start = dma_addr; + cur->remaining = size; + cur->sgl =NULL ; + cur->dma_addr = true; + cur->mem_type =XE_PL_TT ; +} +/** * xe_res_next - advance the cursor *
@@ -192,6 +218,11 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)return ; } +if (cur->dma_addr) { + cur->start += size; +return ; + } +if (cur->sgl) { cur->start += size;__xe_res_sg_next (cur);
@@ -233,6 +264,11 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)*/ static inline u64 xe_res_dma (const struct xe_res_cursor *cur) { -return cur->sgl ?sg_dma_address (cur->sgl) + cur->start : cur->start; +if (cur->dma_addr) +return cur->start; +else if (cur->sgl) +return sg_dma_address (cur->sgl) + cur->start; +else +return cur->start; }#endif
Signature · 1 line
2.26.3
PATCH v3 25/26 drm/xe: Use drm_mem_region for xe
In reply to Oak Zengdrm_mem_region was introduced to move some memory management codes to drm layer so it can be shared b/t different vendor drivers. This patch apply drm_mem_region concept to xekmd driver.
drm_mem_region is the parent class of xe_mem_region. Some xe_mem_region member such as dpa_base is deleted as it is already in the parent class.
--- drivers/gpu/drm/xe/display/xe_fb_pin.c | 2 +- drivers/gpu/drm/xe/display/xe_plane_initial.c | 2 +- drivers/gpu/drm/xe/xe_bo.c | 6 +++--- drivers/gpu/drm/xe/xe_device_types.h | 11 ++--------- drivers/gpu/drm/xe/xe_migrate.c | 6 +++--- drivers/gpu/drm/xe/xe_mmio.c | 12 ++++++------ drivers/gpu/drm/xe/xe_query.c | 2 +- drivers/gpu/drm/xe/xe_tile.c | 2 +- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 2 +- 9 files changed, 19 insertions(+), 26 deletions(-)
drivers/gpu/drm/xe/display/xe_fb_pin.c +1 -1
@@ -272,7 +272,7 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,* accessible. This is important on small-bar systems where * only some subset of VRAM is CPU accessible. */ -if (tile->mem.vram.io_size < tile->mem.vram.usable_size) { +if (tile->mem.vram.io_size < tile->mem.vram.drm_mr.usable_size) { ret = -EINVAL ; goto err; }
drivers/gpu/drm/xe/display/xe_plane_initial.c +1 -1
@@ -86,7 +86,7 @@ initial_plane_bo(struct xe_device *xe,* We don't currently expect this to ever be placed in the * stolen portion. */ -if (phys_base >= tile0->mem.vram.usable_size) { +if (phys_base >= tile0->mem.vram.drm_mr.usable_size) {drm_err (&xe->drm,"Initial plane programming using invalid range, phys_base=%pa\n" , &phys_base);
drivers/gpu/drm/xe/xe_bo.c +3 -3
@@ -173,7 +173,7 @@ static void add_vram(struct xe_device *xe, struct xe_bo *bo,xe_assert (xe, *c <ARRAY_SIZE (bo->placements)); vram =to_xe_ttm_vram_mgr (ttm_manager_type (&xe->ttm, mem_type))->vram; -xe_assert (xe, vram && vram->usable_size); +xe_assert (xe, vram && vram->drm_mr.usable_size); io_size = vram->io_size;/*
@@ -184,7 +184,7 @@ static void add_vram(struct xe_device *xe, struct xe_bo *bo, XE_BO_FLAG_GGTT)) place.flags |= TTM_PL_FLAG_CONTIGUOUS; -if (io_size < vram->usable_size) { +if (io_size < vram->drm_mr.usable_size) {if (bo_flags &XE_BO_FLAG_NEEDS_CPU_ACCESS ) { place.fpfn =0 ; place.lpfn = io_size >>PAGE_SHIFT ;
@@ -1638,7 +1638,7 @@ uint64_t vram_region_gpu_offset(struct ttm_resource *res)if (res->mem_type ==XE_PL_STOLEN )return xe_ttm_stolen_gpu_offset (xe); -return res_to_mem_region (res)->dpa_base; +return res_to_mem_region (res)->drm_mr.dpa_base; }/**
drivers/gpu/drm/xe/xe_device_types.h +2 -9
@@ -11,6 +11,7 @@#include <drm/drm_device.h> #include <drm/drm_file.h> #include <drm/ttm/ttm_device.h> +#include <drm/drm_svm.h> #include "xe_devcoredump_types.h" #include "xe_heci_gsc.h"
@@ -69,6 +70,7 @@ struct xe_pat_ops;* device, such as HBM memory or CXL extension memory. */ struct xe_mem_region { +struct drm_mem_region drm_mr;/** @io_start: IO start address of this VRAM instance */ resource_size_t io_start;/**
@@ -81,15 +83,6 @@ struct xe_mem_region {* configuration is known as small-bar. */ resource_size_t io_size; -/** @dpa_base: This memory regions's DPA (device physical address) base */ -resource_size_t dpa_base; -/** -* @usable_size: usable size of VRAM -* -* Usable size of VRAM excluding reserved portions -* (e.g stolen mem) -*/ -resource_size_t usable_size;/** * @actual_physical_size: Actual VRAM size *
drivers/gpu/drm/xe/xe_migrate.c +3 -3
@@ -126,7 +126,7 @@ static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr)* Remove the DPA to get a correct offset into identity table for the * migrate offset */ - addr -= xe->mem.vram.dpa_base; + addr -= xe->mem.vram.drm_mr.dpa_base;return addr + (256ULL <<xe_pt_shift (2 )); }
@@ -261,8 +261,8 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,* Use 1GB pages, it shouldn't matter the physical amount of * vram is less, when we don't access it. */ -for (pos = xe->mem.vram.dpa_base; - pos < xe->mem.vram.actual_physical_size + xe->mem.vram.dpa_base; +for (pos = xe->mem.vram.drm_mr.dpa_base; + pos < xe->mem.vram.actual_physical_size + xe->mem.vram.drm_mr.dpa_base; pos +=SZ_1G , ofs +=8 )xe_map_wr (xe, &bo->vmap, ofs, u64, pos | flags); }
drivers/gpu/drm/xe/xe_mmio.c +6 -6
@@ -160,7 +160,7 @@ static int xe_determine_lmem_bar_size(struct xe_device *xe)return -EIO ;/* XXX: Need to change when xe link code is ready */ - xe->mem.vram.dpa_base =0 ; + xe->mem.vram.drm_mr.dpa_base =0 ;/* set up a map to the total memory area. */ xe->mem.vram.mapping =ioremap_wc (xe->mem.vram.io_start, xe->mem.vram.io_size);
@@ -319,16 +319,16 @@ int xe_mmio_probe_vram(struct xe_device *xe)return -ENODEV ; } - tile->mem.vram.dpa_base = xe->mem.vram.dpa_base + tile_offset; - tile->mem.vram.usable_size = vram_size; + tile->mem.vram.drm_mr.dpa_base = xe->mem.vram.drm_mr.dpa_base + tile_offset; + tile->mem.vram.drm_mr.usable_size = vram_size; tile->mem.vram.mapping = xe->mem.vram.mapping + tile_offset; -if (tile->mem.vram.io_size < tile->mem.vram.usable_size) +if (tile->mem.vram.io_size < tile->mem.vram.drm_mr.usable_size)drm_info (&xe->drm,"Small BAR device\n" );drm_info (&xe->drm,"VRAM[%u, %u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n" , id, - tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.usable_size, &tile->mem.vram.io_size); + tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.drm_mr.usable_size, &tile->mem.vram.io_size);drm_info (&xe->drm,"VRAM[%u, %u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n" , id, tile->id, - &tile->mem.vram.dpa_base, tile->mem.vram.dpa_base + (u64 )tile->mem.vram.actual_physical_size, + &tile->mem.vram.drm_mr.dpa_base, tile->mem.vram.drm_mr.dpa_base + (u64 )tile->mem.vram.actual_physical_size, &tile->mem.vram.io_start, tile->mem.vram.io_start + (u64 )tile->mem.vram.io_size);/* calculate total size using tile size to get the correct HW sizing */
drivers/gpu/drm/xe/xe_query.c +1 -1
@@ -334,7 +334,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->num_params = num_params; config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID ] = xe->info.devid | (xe->info.revid <<16 ); -if (xe_device_get_root_tile (xe)->mem.vram.usable_size) +if (xe_device_get_root_tile (xe)->mem.vram.drm_mr.usable_size) config->info[DRM_XE_QUERY_CONFIG_FLAGS ] =DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM ; config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
drivers/gpu/drm/xe/xe_tile.c +1 -1
@@ -132,7 +132,7 @@ static int tile_ttm_mgr_init(struct xe_tile *tile)struct xe_device *xe =tile_to_xe (tile);int err; -if (tile->mem.vram.usable_size) { +if (tile->mem.vram.drm_mr.usable_size) { err =xe_ttm_vram_mgr_init (tile, tile->mem.vram_mgr);if (err)return err;
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +1 -1
@@ -364,7 +364,7 @@ int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr) mgr->vram = vram;return __xe_ttm_vram_mgr_init (xe, mgr,XE_PL_VRAM0 + tile->id, - vram->usable_size, vram->io_size, + vram->drm_mr.usable_size, vram->io_size,PAGE_SIZE ); }
Signature · 1 line
2.26.3
PATCH v3 26/26 drm/xe: use drm_hmmptr in xe
In reply to Oak Zengdrm_hmmptr was created to move some userptr and svm logic such as mmu notifier registration, range population etc to drm layer, so those logic can be shared b/t vendor drivers.
This patch apply the drm_hmmptr to xekmd driver. drm_hmmptr is the parent class of xe_userptr.
Most of the changes are straight forward, such as some xe_userptr members are now moved to drm_hmmptr. Since drm_hmmptr doesn't use scatter-gather table for dma address anymore, we changed to the dma-address cursor from the sg cursor during page table building.
xe_hmm.c and xe_hmm.h are removed, as the logic is now moved to drm_svm.c
--- drivers/gpu/drm/xe/Makefile | 2 - drivers/gpu/drm/xe/xe_gt_pagefault.c | 2 + drivers/gpu/drm/xe/xe_hmm.c | 254 --------------------------- drivers/gpu/drm/xe/xe_hmm.h | 11 -- drivers/gpu/drm/xe/xe_pt.c | 8 +- drivers/gpu/drm/xe/xe_vm.c | 64 ++++--- drivers/gpu/drm/xe/xe_vm.h | 2 + drivers/gpu/drm/xe/xe_vm_types.h | 11 +- 8 files changed, 49 insertions(+), 305 deletions(-) delete mode 100644 drivers/gpu/drm/xe/xe_hmm.c delete mode 100644 drivers/gpu/drm/xe/xe_hmm.h
drivers/gpu/drm/xe/Makefile +0 -2
@@ -148,8 +148,6 @@ xe-y += xe_bb.o \ xe_wa.o \ xe_wopcm.o -xe-$(CONFIG_HMM_MIRROR) += xe_hmm.o - # graphics hardware monitoring (HWMON) support xe-$(CONFIG_HWMON) += xe_hwmon.o
drivers/gpu/drm/xe/xe_gt_pagefault.c +2 -0
@@ -152,6 +152,8 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf, err =xe_vma_userptr_pin_pages (uvma);if (err)return err; + +xe_vma_userptr_dma_map_pages (uvma); }/* Lock VM and BOs dma-resv */
drivers/gpu/drm/xe/xe_hmm.c deleted +0 -254
254 lines not on this page · raw message
drivers/gpu/drm/xe/xe_hmm.h deleted +0 -11
@@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: MIT -* -* Copyright © 2024 Intel Corporation -*/ - -#include <linux/types.h> - -struct xe_userptr_vma ; - -int xe_hmm_userptr_populate_range (struct xe_userptr_vma *uvma,bool is_mm_mmap_locked); -void xe_hmm_userptr_free_sg (struct xe_userptr_vma *uvma);
drivers/gpu/drm/xe/xe_pt.c +4 -4
@@ -669,7 +669,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,if (!xe_vma_is_null (vma)) {if (xe_vma_is_userptr (vma)) -xe_res_first_sg (to_userptr_vma (vma)->userptr.sg,0 , +xe_res_first_dma (to_userptr_vma (vma)->userptr.hmmptr.iova.addr,xe_vma_size (vma), &curs);else if (xe_bo_is_vram (bo) ||xe_bo_is_stolen (bo)) xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
@@ -1221,12 +1221,12 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,return 0 ; uvma =to_userptr_vma (vma); - notifier_seq = uvma->userptr.notifier_seq; + notifier_seq = uvma->userptr.hmmptr.notifier_seq;if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode (vm))return 0 ; -if (!mmu_interval_read_retry (&uvma->userptr.notifier, +if (!mmu_interval_read_retry (&uvma->userptr.hmmptr.notifier, notifier_seq) && !xe_pt_userptr_inject_eagain (uvma))return 0 ;
@@ -1755,7 +1755,7 @@ static int unbind_op_prepare(struct xe_tile *tile,* state if an invalidation is running while preparing an unbind. */ if (xe_vma_is_userptr (vma) &&xe_vm_in_fault_mode (xe_vma_vm (vma))) -mmu_interval_read_begin (&to_userptr_vma (vma)->userptr.notifier); +mmu_interval_read_begin (&to_userptr_vma (vma)->userptr.hmmptr.notifier); pt_op->vma = vma; pt_op->bind = false;
drivers/gpu/drm/xe/xe_vm.c +40 -24
134 lines not on this page · raw message
drivers/gpu/drm/xe/xe_vm.h +2 -0
@@ -245,6 +245,8 @@ static inline void xe_vm_reactivate_rebind(struct xe_vm *vm)int xe_vma_userptr_pin_pages (struct xe_userptr_vma *uvma); +void xe_vma_userptr_dma_map_pages (struct xe_userptr_vma *uvma); +int xe_vma_userptr_check_repin (struct xe_userptr_vma *uvma);bool xe_vm_validate_should_retry (struct drm_exec *exec,int err,ktime_t *end);
drivers/gpu/drm/xe/xe_vm_types.h +1 -10
@@ -36,20 +36,11 @@ struct xe_vm_pgtable_update_op;/** struct xe_userptr - User pointer */ struct xe_userptr { +struct drm_hmmptr hmmptr;/** @invalidate_link: Link for the vm::userptr.invalidated list */ struct list_head invalidate_link;/** @userptr: link into VM repin list if userptr. */ struct list_head repin_link; -/** -* @notifier: MMU notifier for user pointer (invalidation call back) -*/ -struct mmu_interval_notifier notifier; -/** @sgt: storage for a scatter gather table */ -struct sg_table sgt; -/** @sg: allocated scatter gather table */ -struct sg_table *sg; -/** @notifier_seq: notifier sequence number */ -unsigned long notifier_seq;/** * @initial_bind: user pointer has been bound at least once. * write: vm->userptr.notifier_lock in read mode and vm->resv held.
Signature · 1 line
2.26.3
Patchwork
CI✓ CI.Patch_applied: success for series starting with [CI,v3,01/26] mm/hmm: let users to tag specific PFNs
In reply to Oak Zeng== Series Details ==
== Series Details ==
Series: series starting with [CI,v3,01/26] mm/hmm: let users to tag specific PFNs
URL : https://patchwork.freedesktop.org/series/134215/
State : success
== Summary ==
=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: 55d6179b96e0 drm-tip: 2024y-05m-29d-17h-29m-17s UTC integration manifest === git am output follows ===
Applying: mm/hmm: let users to tag specific PFNs
Applying: dma-mapping: provide an interface to allocate IOVA
Applying: dma-mapping: provide callbacks to link/unlink pages to specific IOVA
Applying: iommu/dma: Provide an interface to allow preallocate IOVA
Applying: iommu/dma: Prepare map/unmap page functions to receive IOVA
Applying: iommu/dma: Implement link/unlink page callbacks
Applying: drm: move xe_sg_segment_size to drm layer
Applying: drm: Move GPUVA_START/LAST to drm_gpuvm.h
Applying: drm/svm: add a mm field to drm_gpuvm struct
Applying: drm/svm: introduce drm_mem_region concept
Applying: drm/svm: introduce hmmptr and helper functions
Applying: drm/svm: Introduce helper to remap drm memory region
Applying: drm/svm: handle CPU page fault
Applying: drm/svm: Migrate a range of hmmptr to vram
Applying: drm/svm: Add DRM SVM documentation
Applying: drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
Applying: drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
Applying: drm/xe: Convert multiple bind ops into single job
Applying: drm/xe: Update VM trace events
Applying: drm/xe: Update PT layer with better error handling
Applying: drm/xe: Retry BO allocation
Applying: drm/xe: Rework GPU page fault handling
Applying: drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag
Applying: drm/xe: Add dma_addr res cursor
Applying: drm/xe: Use drm_mem_region for xe
Applying: drm/xe: use drm_hmmptr in xe
Patchwork
CI✗ CI.checkpatch: warning for series starting with [CI,v3,01/26] mm/hmm: let users to tag specific PFNs
In reply to Oak Zeng== Series Details ==
== Series Details ==
Series: series starting with [CI,v3,01/26] mm/hmm: let users to tag specific PFNs
URL : https://patchwork.freedesktop.org/series/134215/
State : warning
== Summary ==
+ KERNEL=/kernel
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools mt Cloning into 'mt'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ git -C mt rev-list -n1 origin/master 51ce9f6cd981d42d7467409d7dbc559a450abc1e
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ git log -n1
commit f394c75ee71643b62a31b99ee3ea81b695238214
Author: Oak Zeng <[email protected]> Date: Wed May 29 20:47:32 2024 -0400
drm/xe: use drm_hmmptr in xe
drm_hmmptr was created to move some userptr and svm logic such as mmu notifier registration, range population etc to drm layer, so those logic can be shared b/t vendor drivers.
This patch apply the drm_hmmptr to xekmd driver. drm_hmmptr is the parent class of xe_userptr.
Most of the changes are straight forward, such as some xe_userptr members are now moved to drm_hmmptr. Since drm_hmmptr doesn't use scatter-gather table for dma address anymore, we changed to the dma-address cursor from the sg cursor during page table building.
xe_hmm.c and xe_hmm.h are removed, as the logic is now moved to drm_svm.c
+ /mt/dim checkpatch 55d6179b96e0390025f2ba101c03b94b50cab7a1 drm-intel bc3046f88f1e mm/hmm: let users to tag specific PFNs0e08e7fa4a8a dma-mapping: provide an interface to allocate IOVA
-:73: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #73: FILE: include/linux/dma-mapping.h:170:
+}
+static inline void dma_free_iova(struct dma_iova_attrs *iova)
total: 0 errors, 0 warnings, 1 checks, 83 lines checkedd6f35ec71d2d dma-mapping: provide callbacks to link/unlink pages to specific IOVA -:59: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #59: FILE: include/linux/dma-mapping.h:176:
}
+static inline dma_addr_t dma_link_range(struct page *page, unsigned long offset,
-:65: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #65: FILE: include/linux/dma-mapping.h:182:
+}
+static inline void dma_unlink_range(struct dma_iova_attrs *iova,
total: 0 errors, 0 warnings, 2 checks, 123 lines checked 9bd2ddc4e8cd iommu/dma: Provide an interface to allow preallocate IOVA -:31: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #31: FILE: drivers/iommu/dma-iommu.c:762:
+static dma_addr_t __iommu_dma_alloc_iova(struct iommu_domain *domain,
size_t size, u64 dma_limit, struct device *dev)
-:40: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #40: FILE: drivers/iommu/dma-iommu.c:808:
+static void __iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
total: 0 errors, 0 warnings, 2 checks, 132 lines checked e80879f31d0b iommu/dma: Prepare map/unmap page functions to receive IOVA -:197: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #197: FILE: drivers/iommu/dma-iommu.c:1658:
+ *handle = __iommu_dma_map(dev, page_to_phys(page), 0, size, ioprot,
dev->coherent_dma_mask);
total: 0 errors, 0 warnings, 1 checks, 171 lines checked 0330423a5149 iommu/dma: Implement link/unlink page callbacks a4154deb4767 drm: move xe_sg_segment_size to drm layerd64d0c98d892 drm: Move GPUVA_START/LAST to drm_gpuvm.h
-:41: CHECK:MACRO_ARG_REUSE: Macro argument reuse 'node' - possible side-effects? #41: FILE: include/drm/drm_gpuvm.h:42:
+#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
-:42: CHECK:MACRO_ARG_REUSE: Macro argument reuse 'node' - possible side-effects? #42: FILE: include/drm/drm_gpuvm.h:43:
+#define GPUVA_END(node) ((node)->va.addr + (node)->va.range)
total: 0 errors, 0 warnings, 2 checks, 19 lines checked5ca221e158d0 drm/svm: add a mm field to drm_gpuvm struct
-:9: WARNING:COMMIT_LOG_LONG_LINE: Prefer a maximum 75 chars per line (possible unwrapped commit description?) #9:
Add a mm_struct field to drm_gpuvm. This is only used by shared virtual memory,
total: 0 errors, 1 warnings, 0 checks, 19 lines checkede89d08977db9 drm/svm: introduce drm_mem_region concept
-:35: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #35:
new file mode 100644
-:40: WARNING:SPDX_LICENSE_TAG: Improper SPDX comment style for 'include/drm/drm_svm.h', please use '/*' instead #40: FILE: include/drm/drm_svm.h:1:
+// SPDX-License-Identifier: MIT
-:40: WARNING:SPDX_LICENSE_TAG: Missing or malformed SPDX-License-Identifier tag in line 1 #40: FILE: include/drm/drm_svm.h:1:
+// SPDX-License-Identifier: MIT
-:76: ERROR:FLEXIBLE_ARRAY: Use C99 flexible arrays - see https://docs.kernel.org/process/deprecated.html#zero-length-and-one-element-arrays
#76: FILE: include/drm/drm_svm.h:37:
+ } addr_vec[1];
+};
-:90: WARNING:SPACING: Unnecessary space before function pointer arguments #90: FILE: include/drm/drm_svm.h:51:
+ int (*drm_mem_region_alloc_pages) (struct drm_mem_region *mr,
-:91: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #91: FILE: include/drm/drm_svm.h:52:
+ int (*drm_mem_region_alloc_pages) (struct drm_mem_region *mr,
+ unsigned long npags, unsigned long *pfns);
-:96: WARNING:SPACING: Unnecessary space before function pointer arguments #96: FILE: include/drm/drm_svm.h:57:
+ void (*drm_mem_region_free_page) (struct page *page);
-:107: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #107: FILE: include/drm/drm_svm.h:68:
+ struct dma_fence* (*drm_mem_region_migrate)(struct migrate_vec *src_vec,
+ struct migrate_vec *dst_vec);
-:189: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #189: FILE: include/drm/drm_svm.h:150:
+ BUG_ON((pfn << PAGE_SHIFT) < mr->hpa_base );
-:189: ERROR:SPACING: space prohibited before that close parenthesis ')' #189: FILE: include/drm/drm_svm.h:150:
+ BUG_ON((pfn << PAGE_SHIFT) < mr->hpa_base );
-:190: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #190: FILE: include/drm/drm_svm.h:151:
+ BUG_ON((pfn << PAGE_SHIFT) >= mr->hpa_base + mr->usable_size);
total: 2 errors, 7 warnings, 2 checks, 156 lines checkeda4a84ff69a7a drm/svm: introduce hmmptr and helper functions
Traceback (most recent call last):
File "scripts/spdxcheck.py", line 6, in <module>
from ply import lex, yacc
ModuleNotFoundError: No module named 'ply'
-:58: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #58:
new file mode 100644
-:68: CHECK:LINE_SPACING: Please don't use multiple blank lines #68: FILE: drivers/gpu/drm/drm_svm.c:6:
+
+
-:157: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #157: FILE: drivers/gpu/drm/drm_svm.c:95:
+ BUG_ON(page_idx + npages > tpages);
-:160: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #160: FILE: drivers/gpu/drm/drm_svm.c:98:
+ BUG_ON(!page);
-:161: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #161: FILE: drivers/gpu/drm/drm_svm.c:99:
+ BUG_ON(is_device_private_page(page));
-:174: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #174: FILE: drivers/gpu/drm/drm_svm.c:112:
+int drm_svm_hmmptr_init(struct drm_hmmptr *hmmptr,
+ const struct mmu_interval_notifier_ops *ops)
-:200: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #200: FILE: drivers/gpu/drm/drm_svm.c:138:
+ ret = mmu_interval_notifier_insert(&hmmptr->notifier, current->mm,
+ start, end - start, ops);
-:275: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #275: FILE: drivers/gpu/drm/drm_svm.c:213:
+ BUG_ON(start < __hmmptr_start(hmmptr));
-:276: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #276: FILE: drivers/gpu/drm/drm_svm.c:214:
+ BUG_ON(end > __hmmptr_end(hmmptr));
-:330: CHECK:LINE_SPACING: Please don't use multiple blank lines #330: FILE: include/drm/drm_svm.h:12:
+
-:383: WARNING:SPACING: Unnecessary space before function pointer arguments #383: FILE: include/drm/drm_svm.h:203:
+ struct drm_gpuva * (*get_gpuva) (struct drm_hmmptr *hmmptr);
-:387: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #387: FILE: include/drm/drm_svm.h:207:
+int drm_svm_hmmptr_init(struct drm_hmmptr *hmmptr,
+ const struct mmu_interval_notifier_ops *ops);
-:391: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #391: FILE: include/drm/drm_svm.h:211:
+int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner,
+ u64 start, u64 end, bool write);
total: 0 errors, 7 warnings, 6 checks, 337 lines checked 981f0b935803 drm/svm: Introduce helper to remap drm memory region -:90: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #90: FILE: drivers/gpu/drm/drm_svm.c:295:
+ drm_err(drm, "Failed to remap memory region %p, errno %d\n",
+ mr, ret);
-:96: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #96: FILE: drivers/gpu/drm/drm_svm.c:301:
+ drm_info(drm, "Registered device memory [%llx-%llx] to devm, remapped to %pr\n",
+ mr->dpa_base, mr->dpa_base + mr->usable_size, res);
total: 0 errors, 0 warnings, 2 checks, 67 lines checked c57566060c29 drm/svm: handle CPU page fault
-:69: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #69: FILE: drivers/gpu/drm/drm_svm.c:287:
+static int __drm_svm_alloc_host_pages(struct vm_area_struct *vma,
+ unsigned long *mpfn)
-:94: WARNING:LONG_LINE: line length of 118 exceeds 100 columns
#94: FILE: drivers/gpu/drm/drm_svm.c:312:
+static struct migrate_vec *__generate_migrate_vec_vram(unsigned long *mpfn, bool is_migrate_src, unsigned long npages)
-:112: ERROR:SPACING: space required before the open parenthesis '(' #112: FILE: drivers/gpu/drm/drm_svm.c:330:
+ for(i = 0, j = 0; i < npages; i++) {
-:126: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #126: FILE: drivers/gpu/drm/drm_svm.c:344:
+static struct migrate_vec *__generate_migrate_vec_sram(struct device *dev, unsigned long *mpfn,
+ bool is_migrate_src, unsigned long npages)
-:143: ERROR:SPACING: spaces required around that '=' (ctx:WxV) #143: FILE: drivers/gpu/drm/drm_svm.c:361:
+ for(i = 0, k =0 ; i < npages; i++) {
^
-:143: ERROR:SPACING: space required before the open parenthesis '(' #143: FILE: drivers/gpu/drm/drm_svm.c:361:
+ for(i = 0, k =0 ; i < npages; i++) {
-:168: WARNING:LONG_LINE: line length of 101 exceeds 100 columns
#168: FILE: drivers/gpu/drm/drm_svm.c:386:
+static void __free_migrate_vec_sram(struct device *dev, struct migrate_vec *vec, bool is_migrate_src)
-:173: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #173: FILE: drivers/gpu/drm/drm_svm.c:391:
+ BUG_ON(vec->mr != NULL);
-:173: CHECK:COMPARISON_TO_NULL: Comparison to NULL could be written "vec->mr" #173: FILE: drivers/gpu/drm/drm_svm.c:391:
+ BUG_ON(vec->mr != NULL);
-:184: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #184: FILE: drivers/gpu/drm/drm_svm.c:402:
+ BUG_ON(vec->mr == NULL);
-:184: CHECK:COMPARISON_TO_NULL: Comparison to NULL could be written "!vec->mr" #184: FILE: drivers/gpu/drm/drm_svm.c:402:
+ BUG_ON(vec->mr == NULL);
-:188: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #188: FILE: drivers/gpu/drm/drm_svm.c:406:
+}
+/**
-:286: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #286: FILE: drivers/gpu/drm/drm_svm.c:504:
+}
+static struct dev_pagemap_ops drm_devm_pagemap_ops = {
total: 3 errors, 4 warnings, 6 checks, 263 lines checked b09616ca0f1c drm/svm: Migrate a range of hmmptr to vram -:39: ERROR:SPACING: space required before the open parenthesis '(' #39: FILE: drivers/gpu/drm/drm_svm.c:560:
+ for(i = 0; i < npages; i++) {
-:59: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #59: FILE: drivers/gpu/drm/drm_svm.c:580:
+int drm_svm_migrate_hmmptr_to_vram(struct drm_gpuvm *vm,
+ struct drm_mem_region *mr,
-:82: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #82: FILE: drivers/gpu/drm/drm_svm.c:603:
+ BUG_ON(start < GPUVA_START(gpuva));
-:83: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #83: FILE: drivers/gpu/drm/drm_svm.c:604:
+ BUG_ON(end > GPUVA_END(gpuva));
-:90: CHECK:SPACING: spaces preferred around that '*' (ctx:VxW) #90: FILE: drivers/gpu/drm/drm_svm.c:611:
+ buf = kvcalloc(npages, 2* sizeof(*migrate.src), GFP_KERNEL);
^
-:91: ERROR:SPACING: space required before the open parenthesis '(' #91: FILE: drivers/gpu/drm/drm_svm.c:612:
+ if(!buf)
-:99: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #99: FILE: drivers/gpu/drm/drm_svm.c:620:
+ drm_warn(drm, "vma setup returned %d for range [%lx - %lx]\n",
+ ret, start, end);
-:109: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #109: FILE: drivers/gpu/drm/drm_svm.c:630:
+ drm_warn(drm, "Partial migration for range [%lx - %lx], range is %ld pages, migrate only %ld pages\n",
+ start, end, npages, migrate.cpages);
-:164: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #164: FILE: include/drm/drm_svm.h:215:
+int drm_svm_migrate_hmmptr_to_vram(struct drm_gpuvm *vm,
+ struct drm_mem_region *mr,
total: 2 errors, 2 warnings, 5 checks, 131 lines checked53e4de7cf2a8 drm/svm: Add DRM SVM documentation
-:217: WARNING:TYPO_SPELLING: 'delibrately' may be misspelled - perhaps 'deliberately'? #217: FILE: drivers/gpu/drm/drm_svm.c:147:
+ * in DRM SVM design. We delibrately choose this implementation option to achieve true
^^^^^^^^^^^
-:315: WARNING:LONG_LINE_COMMENT: line length of 121 exceeds 100 columns #315: FILE: drivers/gpu/drm/drm_svm.c:245:
+ * In GPU page fault handler: munmap from user:
-:318: WARNING:LONG_LINE_COMMENT: line length of 121 exceeds 100 columns #318: FILE: drivers/gpu/drm/drm_svm.c:248:
+ * Mmap_read_lock() mmap_write_lock()
-:319: WARNING:LONG_LINE_COMMENT: line length of 168 exceeds 100 columns #319: FILE: drivers/gpu/drm/drm_svm.c:249:
+ * call drm_svm_migrate_hmmptr_to_vram to migrate hmmptr if needed mmu_notifier_invalidate_range_start(), call hmmptr->invalidate()
-:320: WARNING:LONG_LINE_COMMENT: line length of 136 exceeds 100 columns #320: FILE: drivers/gpu/drm/drm_svm.c:250:
+ * hmmptr.notifier_seq = mmu_interval_read_begin(&hmmptr.notifier) down_write(page_table_lock);
-:321: WARNING:LONG_LINE_COMMENT: line length of 131 exceeds 100 columns #321: FILE: drivers/gpu/drm/drm_svm.c:251:
+ * call drm_svm_populate_hmmptr to populate hmmptr mmu_interval_set_seq();
-:322: WARNING:LONG_LINE_COMMENT: line length of 145 exceeds 100 columns #322: FILE: drivers/gpu/drm/drm_svm.c:252:
+ * Mmap_read_unlock invalidate hmmptr from GPU page table
-:323: WARNING:LONG_LINE_COMMENT: line length of 134 exceeds 100 columns #323: FILE: drivers/gpu/drm/drm_svm.c:253:
+ * up_write(page_table_lock);
-:324: WARNING:LONG_LINE_COMMENT: line length of 139 exceeds 100 columns #324: FILE: drivers/gpu/drm/drm_svm.c:254:
+ * dma_resv_lock() mmu_notifier_invalidate_range_end()
-:325: WARNING:LONG_LINE_COMMENT: line length of 126 exceeds 100 columns #325: FILE: drivers/gpu/drm/drm_svm.c:255:
+ * down_read(page_table_lock); mmap_write_unlock();
total: 0 errors, 10 warnings, 0 checks, 360 lines checkedc54223667b3f drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue 211a7aaf6e02 drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops25e62c8526cd drm/xe: Convert multiple bind ops into single job
-:13: WARNING:TYPO_SPELLING: 'implemenation' may be misspelled - perhaps 'implementation'? #13:
The implemenation is roughly:
^^^^^^^^^^^^^
-:885: WARNING:SUSPECT_CODE_INDENT: suspect code indent for conditional statements (8, 15) #885: FILE: drivers/gpu/drm/xe/xe_pt.c:1163:
+ if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))
+ return 0;
-:886: ERROR:CODE_INDENT: code indent should use tabs where possible #886: FILE: drivers/gpu/drm/xe/xe_pt.c:1164:
+ return 0;$
-:886: WARNING:LEADING_SPACE: please, no spaces at the start of a line #886: FILE: drivers/gpu/drm/xe/xe_pt.c:1164:
+ return 0;$
-:1667: WARNING:BLOCK_COMMENT_STYLE: Block comments should align the * on each line #1667: FILE: drivers/gpu/drm/xe/xe_pt.c:1898:
+ * installing job fence in various places.
+ *
-:1668: WARNING:BLOCK_COMMENT_STYLE: Block comments should align the * on each line #1668: FILE: drivers/gpu/drm/xe/xe_pt.c:1899:
+ *
+ * Return: fence on success, negative ERR_PTR on error.
-:1800: WARNING:LEADING_SPACE: please, no spaces at the start of a line #1800: FILE: drivers/gpu/drm/xe/xe_pt.c:2031:
+ }$
total: 1 errors, 6 warnings, 0 checks, 2532 lines checkedf04332685569 drm/xe: Update VM trace eventsc316f6986241 drm/xe: Update PT layer with better error handling9e9fcf221575 drm/xe: Retry BO allocation314c773e48e5 drm/xe: Rework GPU page fault handlingf1be750203f3 drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag
-:40: WARNING:COMMIT_LOG_LONG_LINE: Prefer a maximum 75 chars per line (possible unwrapped commit description?) #40:
FIXME: Not enforcing DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR for fault mode VMs
total: 0 errors, 1 warnings, 0 checks, 451 lines checked fb91b8c587bb drm/xe: Add dma_addr res cursor f8d69d83215a drm/xe: Use drm_mem_region for xe
-:162: WARNING:LONG_LINE: line length of 134 exceeds 100 columns #162: FILE: drivers/gpu/drm/xe/xe_mmio.c:329:
+ tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.drm_mr.usable_size, &tile->mem.vram.io_size);
-:165: WARNING:LONG_LINE: line length of 132 exceeds 100 columns #165: FILE: drivers/gpu/drm/xe/xe_mmio.c:331:
+ &tile->mem.vram.drm_mr.dpa_base, tile->mem.vram.drm_mr.dpa_base + (u64)tile->mem.vram.actual_physical_size,
total: 0 errors, 2 warnings, 0 checks, 140 lines checkedf394c75ee716 drm/xe: use drm_hmmptr in xe
-:52: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #52:
deleted file mode 100644
-:338: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #338: FILE: drivers/gpu/drm/xe/xe_pt.c:673:
+ xe_res_first_dma(to_userptr_vma(vma)->userptr.hmmptr.iova.addr,
xe_vma_size(vma), &curs);
-:402: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #402: FILE: drivers/gpu/drm/xe/xe_vm.c:79:
+ ret = drm_svm_hmmptr_populate(hmmptr, NULL, xe_vma_start(vma),
+ xe_vma_end(vma), !xe_vma_read_only(vma));
-:464: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #464: FILE: drivers/gpu/drm/xe/xe_vm.c:1015:
+ err = drm_svm_hmmptr_init(&userptr->hmmptr,
+ &vma_userptr_notifier_ops);
total: 0 errors, 1 warnings, 3 checks, 209 lines checked
Patchwork
CI✗ CI.KUnit: failure for series starting with [CI,v3,01/26] mm/hmm: let users to tag specific PFNs
In reply to Oak Zeng== Series Details ==
== Series Details ==
Series: series starting with [CI,v3,01/26] mm/hmm: let users to tag specific PFNs
URL : https://patchwork.freedesktop.org/series/134215/
State : failure
== Summary ==
+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig ERROR:root:In file included from ../drivers/gpu/drm/xe/xe_device_types.h:14,
from ../drivers/gpu/drm/xe/xe_assert.h:13,
from ../drivers/gpu/drm/xe/xe_bb.c:9:../include/drm/drm_svm.h:189:31: error: field ‘notifier’ has incomplete type
189 | struct mmu_interval_notifier notifier;
| ^~~~~~~~
make[7]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/xe/xe_bb.o] Error 1 make[7]: *** Waiting for unfinished jobs....
In file included from ../drivers/gpu/drm/xe/xe_device_types.h:14,
from ../drivers/gpu/drm/xe/xe_device.h:11,
from ../drivers/gpu/drm/xe/tests/xe_pci_test.c:13:../include/drm/drm_svm.h:189:31: error: field ‘notifier’ has incomplete type
189 | struct mmu_interval_notifier notifier;
| ^~~~~~~~
make[8]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/xe/tests/xe_pci_test.o] Error 1 make[8]: *** Waiting for unfinished jobs....
In file included from ../drivers/gpu/drm/xe/xe_device_types.h:14,
from ../drivers/gpu/drm/xe/xe_device.h:11,
from ../drivers/gpu/drm/xe/tests/xe_rtp_test.c:16:../include/drm/drm_svm.h:189:31: error: field ‘notifier’ has incomplete type
189 | struct mmu_interval_notifier notifier;
| ^~~~~~~~
make[8]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/xe/tests/xe_rtp_test.o] Error 1 In file included from ../drivers/gpu/drm/xe/xe_device_types.h:14,
from ../drivers/gpu/drm/xe/xe_device.h:11,
from ../drivers/gpu/drm/xe/tests/xe_wa_test.c:11:../include/drm/drm_svm.h:189:31: error: field ‘notifier’ has incomplete type
189 | struct mmu_interval_notifier notifier;
| ^~~~~~~~
make[8]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/xe/tests/xe_wa_test.o] Error 1 make[7]: *** [../scripts/Makefile.build:485: drivers/gpu/drm/xe/tests] Error 2 make[6]: *** [../scripts/Makefile.build:485: drivers/gpu/drm/xe] Error 2 make[6]: *** Waiting for unfinished jobs....
make[5]: *** [../scripts/Makefile.build:485: drivers/gpu/drm] Error 2
make[4]: *** [../scripts/Makefile.build:485: drivers/gpu] Error 2
make[4]: *** Waiting for unfinished jobs....
make[3]: *** [../scripts/Makefile.build:485: drivers] Error 2
make[3]: *** Waiting for unfinished jobs....../lib/iomap.c:156:5: warning: no previous prototype for ‘ioread64_lo_hi’ [-Wmissing-prototypes]
156 | u64 ioread64_lo_hi(const void __iomem *addr)
| ^~~~~~~~~~~~~~
../lib/iomap.c:163:5: warning: no previous prototype for ‘ioread64_hi_lo’ [-Wmissing-prototypes]
163 | u64 ioread64_hi_lo(const void __iomem *addr)
| ^~~~~~~~~~~~~~
../lib/iomap.c:170:5: warning: no previous prototype for ‘ioread64be_lo_hi’ [-Wmissing-prototypes]
170 | u64 ioread64be_lo_hi(const void __iomem *addr)
| ^~~~~~~~~~~~~~~~
../lib/iomap.c:178:5: warning: no previous prototype for ‘ioread64be_hi_lo’ [-Wmissing-prototypes]
178 | u64 ioread64be_hi_lo(const void __iomem *addr)
| ^~~~~~~~~~~~~~~~
../lib/iomap.c:264:6: warning: no previous prototype for ‘iowrite64_lo_hi’ [-Wmissing-prototypes]
264 | void iowrite64_lo_hi(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~
../lib/iomap.c:272:6: warning: no previous prototype for ‘iowrite64_hi_lo’ [-Wmissing-prototypes]
272 | void iowrite64_hi_lo(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~
../lib/iomap.c:280:6: warning: no previous prototype for ‘iowrite64be_lo_hi’ [-Wmissing-prototypes]
280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~~~
../lib/iomap.c:288:6: warning: no previous prototype for ‘iowrite64be_hi_lo’ [-Wmissing-prototypes]
288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~~~make[2]: *** [/kernel/Makefile:1934: .] Error 2 make[1]: *** [/kernel/Makefile:240: __sub-make] Error 2
make: *** [Makefile:240: __sub-make] Error 2
[00:51:16] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig [00:51:21] Building KUnit Kernel ... Populating config with: $ make ARCH=um O=.kunit olddefconfig Building with: $ make ARCH=um O=.kunit --jobs=48 + cleanup ++ stat -c %u:%g /kernel + chown -R 1003:1003 /kernel