Skip to contentSkip to conversation

mm/hmm: let users to tag specific PFNs

PATCH New Oak Zeng 2 participants 45 messages Jun 13, 2024

Subsystems (12)
lore
Share & tools
lore

Oak Zeng

author
#0 lore raw

PATCH 1/42 mm/hmm: let users to tag specific PFNs

From: Leon Romanovsky <[email protected]>

Introduce new sticky flag, which isn't overwritten by HMM range fault. Such flag allows users to tag specific PFNs with extra data in addition to already filled by HMM.

---
 include/linux/hmm.h |  3 +++
 mm/hmm.c            | 34 +++++++++++++++++++++-------------
 2 files changed, 24 insertions(+), 13 deletions(-)
include/linux/hmm.h +3 -0
index 126a36571667..b90902baa593 100644
@@ -23,6 +23,7 @@ struct mmu_interval_notifier;
  * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
  * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
  *                 fail. ie poisoned memory, special pages, no vma, etc
+ * HMM_PFN_STICKY - Flag preserved on input-to-output transformation
  *
  * On input:
  * 0                 - Return the current state of the page, do not fault it.
@@ -36,6 +37,8 @@ enum hmm_pfn_flags {
 	HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
 	HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
 	HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
+	/* Sticky lag, carried from Input to Output */
+	HMM_PFN_STICKY = 1UL << (BITS_PER_LONG - 7),
 	HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
 
 	/* Input flags */
mm/hmm.c +21 -13
index 93aebd9cc130..a244071085b1 100644
@@ -44,8 +44,10 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end,
 {
 	unsigned long i = (addr - range->start) >> PAGE_SHIFT;
 
-	for (; addr < end; addr += PAGE_SIZE, i++)
-		range->hmm_pfns[i] = cpu_flags;
+	for (; addr < end; addr += PAGE_SIZE, i++) {
+		range->hmm_pfns[i] &= HMM_PFN_STICKY;
+		range->hmm_pfns[i] |= cpu_flags;
+	}
 	return 0;
 }
 
@@ -202,8 +204,10 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
 		return hmm_vma_fault(addr, end, required_fault, walk);
 
 	pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
-	for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
-		hmm_pfns[i] = pfn | cpu_flags;
+	for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
+		hmm_pfns[i] &= HMM_PFN_STICKY;
+		hmm_pfns[i] |= pfn | cpu_flags;
+	}
 	return 0;
 }
 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
@@ -236,7 +240,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
 			hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
 		if (required_fault)
 			goto fault;
-		*hmm_pfn = 0;
+		*hmm_pfn = *hmm_pfn & HMM_PFN_STICKY;
 		return 0;
 	}
 
@@ -253,14 +257,14 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
 			cpu_flags = HMM_PFN_VALID;
 			if (is_writable_device_private_entry(entry))
 				cpu_flags |= HMM_PFN_WRITE;
-			*hmm_pfn = swp_offset_pfn(entry) | cpu_flags;
+			*hmm_pfn = (*hmm_pfn & HMM_PFN_STICKY) | swp_offset_pfn(entry) | cpu_flags;
 			return 0;
 		}
 
 		required_fault =
 			hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
 		if (!required_fault) {
-			*hmm_pfn = 0;
+			*hmm_pfn = *hmm_pfn & HMM_PFN_STICKY;
 			return 0;
 		}
 
@@ -304,11 +308,11 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
 			pte_unmap(ptep);
 			return -EFAULT;
 		}
-		*hmm_pfn = HMM_PFN_ERROR;
+		*hmm_pfn = (*hmm_pfn & HMM_PFN_STICKY) | HMM_PFN_ERROR;
 		return 0;
 	}
 
-	*hmm_pfn = pte_pfn(pte) | cpu_flags;
+	*hmm_pfn = (*hmm_pfn & HMM_PFN_STICKY) | pte_pfn(pte) | cpu_flags;
 	return 0;
 
 fault:
@@ -448,8 +452,10 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
 		}
 
 		pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
-		for (i = 0; i < npages; ++i, ++pfn)
-			hmm_pfns[i] = pfn | cpu_flags;
+		for (i = 0; i < npages; ++i, ++pfn) {
+			hmm_pfns[i] &= HMM_PFN_STICKY;
+			hmm_pfns[i] |= pfn | cpu_flags;
+		}
 		goto out_unlock;
 	}
 
@@ -507,8 +513,10 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
 	}
 
 	pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
-	for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
-		range->hmm_pfns[i] = pfn | cpu_flags;
+	for (; addr < end; addr += PAGE_SIZE, i++, pfn++) {
+		range->hmm_pfns[i] &= HMM_PFN_STICKY;
+		range->hmm_pfns[i] |= pfn | cpu_flags;
+	}
 
 	spin_unlock(ptl);
 	return 0;
Signature · 1 line
2.26.3
author
#1 lore raw

PATCH 2/42 dma-mapping: provide an interface to allocate IOVA

From: Leon Romanovsky <[email protected]>

Existing .map_page() callback provides two things at the same time:
allocates IOVA and links DMA pages. That combination works great for most of the callers who use it in control paths, but less effective in fast paths.

These advanced callers already manage their data in some sort of database and can perform IOVA allocation in advance, leaving range linkage operation to be in fast path.

Provide an interface to allocate/deallocate IOVA and next patch link/unlink DMA ranges to that specific IOVA.

---
 include/linux/dma-map-ops.h |  3 +++
 include/linux/dma-mapping.h | 20 ++++++++++++++++++++
 kernel/dma/mapping.c        | 30 ++++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+)
include/linux/dma-map-ops.h +3 -0
index 02a1c825896b..23e5e2f63a1c 100644
@@ -86,6 +86,9 @@ struct dma_map_ops {
 	size_t (*max_mapping_size)(struct device *dev);
 	size_t (*opt_mapping_size)(void);
 	unsigned long (*get_merge_boundary)(struct device *dev);
+
+	dma_addr_t (*alloc_iova)(struct device *dev, size_t size);
+	void (*free_iova)(struct device *dev, dma_addr_t dma_addr, size_t size);
 };
 
 #ifdef CONFIG_DMA_OPS
include/linux/dma-mapping.h +20 -0
index f693aafe221f..34a3b6420606 100644
@@ -91,6 +91,16 @@ static inline void debug_dma_map_single(struct device *dev, const void *addr,
 }
 #endif /* CONFIG_DMA_API_DEBUG */
 
+struct dma_iova_attrs {
+	/* OUT field */
+	dma_addr_t addr;
+	/* IN fields */
+	struct device *dev;
+	size_t size;
+	enum dma_data_direction dir;
+	unsigned long attrs;
+};
+
 #ifdef CONFIG_HAS_DMA
 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
@@ -101,6 +111,9 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 	return 0;
 }
 
+int dma_alloc_iova(struct dma_iova_attrs *iova);
+void dma_free_iova(struct dma_iova_attrs *iova);
+
 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
 		size_t offset, size_t size, enum dma_data_direction dir,
 		unsigned long attrs);
@@ -150,6 +163,13 @@ void dma_vunmap_noncontiguous(struct device *dev, void *vaddr);
 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma,
 		size_t size, struct sg_table *sgt);
 #else /* CONFIG_HAS_DMA */
+static inline int dma_alloc_iova(struct dma_iova_attrs *iova)
+{
+	return -EOPNOTSUPP;
+}
+static inline void dma_free_iova(struct dma_iova_attrs *iova)
+{
+}
 static inline dma_addr_t dma_map_page_attrs(struct device *dev,
 		struct page *page, size_t offset, size_t size,
 		enum dma_data_direction dir, unsigned long attrs)
kernel/dma/mapping.c +30 -0
index 81de84318ccc..4d14637d186b 100644
@@ -183,6 +183,36 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
 }
 EXPORT_SYMBOL(dma_unmap_page_attrs);
 
+int dma_alloc_iova(struct dma_iova_attrs *iova)
+{
+	struct device *dev = iova->dev;
+	const struct dma_map_ops *ops = get_dma_ops(dev);
+
+	if (dma_map_direct(dev, ops) || !ops->alloc_iova) {
+		iova->addr = 0;
+		return 0;
+	}
+
+	iova->addr = ops->alloc_iova(dev, iova->size);
+	if (dma_mapping_error(dev, iova->addr))
+		return -ENOMEM;
+
+	return 0;
+}
+EXPORT_SYMBOL(dma_alloc_iova);
+
+void dma_free_iova(struct dma_iova_attrs *iova)
+{
+	struct device *dev = iova->dev;
+	const struct dma_map_ops *ops = get_dma_ops(dev);
+
+	if (dma_map_direct(dev, ops) || !ops->free_iova)
+		return;
+
+	ops->free_iova(dev, iova->addr, iova->size);
+}
+EXPORT_SYMBOL(dma_free_iova);
+
 static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 	 int nents, enum dma_data_direction dir, unsigned long attrs)
 {
Signature · 1 line
2.26.3
author
#2 lore raw

PATCH 3/42 dma-mapping: provide callbacks to link/unlink pages to specific IOVA

In reply to Oak Zeng

From: Leon Romanovsky <[email protected]>

Introduce new DMA link/unlink API to provide a way for advanced users to directly map/unmap pages without ned to allocate IOVA on every map call.

---
 include/linux/dma-map-ops.h | 10 +++++++
 include/linux/dma-mapping.h | 13 +++++++++
 kernel/dma/debug.h          |  2 ++
 kernel/dma/direct.h         |  3 ++
 kernel/dma/mapping.c        | 57 +++++++++++++++++++++++++++++++++++++
 5 files changed, 85 insertions(+)
include/linux/dma-map-ops.h +10 -0
index 23e5e2f63a1c..292326ac5a12 100644
@@ -89,6 +89,13 @@ struct dma_map_ops {
 
 	dma_addr_t (*alloc_iova)(struct device *dev, size_t size);
 	void (*free_iova)(struct device *dev, dma_addr_t dma_addr, size_t size);
+	dma_addr_t (*link_range)(struct device *dev, struct page *page,
+				 unsigned long offset, dma_addr_t addr,
+				 size_t size, enum dma_data_direction dir,
+				 unsigned long attrs);
+	void (*unlink_range)(struct device *dev, dma_addr_t dma_handle,
+			     size_t size, enum dma_data_direction dir,
+			     unsigned long attrs);
 };
 
 #ifdef CONFIG_DMA_OPS
@@ -440,6 +447,9 @@ bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,
 #define arch_dma_unmap_sg_direct(d, s, n)	(false)
 #endif
 
+#define arch_dma_link_range_direct arch_dma_map_page_direct
+#define arch_dma_unlink_range_direct arch_dma_unmap_page_direct
+
 #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
 void arch_setup_dma_ops(struct device *dev, bool coherent);
 #else
include/linux/dma-mapping.h +13 -0
index 34a3b6420606..223b5477e36e 100644
@@ -113,6 +113,9 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 
 int dma_alloc_iova(struct dma_iova_attrs *iova);
 void dma_free_iova(struct dma_iova_attrs *iova);
+dma_addr_t dma_link_range(struct page *page, unsigned long offset,
+			  struct dma_iova_attrs *iova, dma_addr_t dma_offset);
+void dma_unlink_range(struct dma_iova_attrs *iova, dma_addr_t dma_offset);
 
 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
 		size_t offset, size_t size, enum dma_data_direction dir,
@@ -170,6 +173,16 @@ static inline int dma_alloc_iova(struct dma_iova_attrs *iova)
 static inline void dma_free_iova(struct dma_iova_attrs *iova)
 {
 }
+static inline dma_addr_t dma_link_range(struct page *page, unsigned long offset,
+					struct dma_iova_attrs *iova,
+					dma_addr_t dma_offset)
+{
+	return DMA_MAPPING_ERROR;
+}
+static inline void dma_unlink_range(struct dma_iova_attrs *iova,
+				    dma_addr_t dma_offset)
+{
+}
 static inline dma_addr_t dma_map_page_attrs(struct device *dev,
 		struct page *page, size_t offset, size_t size,
 		enum dma_data_direction dir, unsigned long attrs)
kernel/dma/debug.h +2 -0
index f525197d3cae..3d529f355c6d 100644
@@ -127,4 +127,6 @@ static inline void debug_dma_sync_sg_for_device(struct device *dev,
 {
 }
 #endif /* CONFIG_DMA_API_DEBUG */
+#define debug_dma_link_range debug_dma_map_page
+#define debug_dma_unlink_range debug_dma_unmap_page
 #endif /* _KERNEL_DMA_DEBUG_H */
kernel/dma/direct.h +3 -0
index 18d346118fe8..1c30e1cd607a 100644
@@ -125,4 +125,7 @@ static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
 		swiotlb_tbl_unmap_single(dev, phys, size, dir,
 					 attrs | DMA_ATTR_SKIP_CPU_SYNC);
 }
+
+#define dma_direct_link_range dma_direct_map_page
+#define dma_direct_unlink_range dma_direct_unmap_page
 #endif /* _KERNEL_DMA_DIRECT_H */
kernel/dma/mapping.c +57 -0

63 lines not on this page · raw message

Signature · 1 line
2.26.3
author
#3 lore raw

PATCH 4/42 iommu/dma: Provide an interface to allow preallocate IOVA

In reply to Oak Zeng

From: Leon Romanovsky <[email protected]>

Separate IOVA allocation to dedicated callback so it will allow cache of IOVA and reuse it in fast paths for devices which support ODP (on-demand-paging) mechanism.

---
 drivers/iommu/dma-iommu.c | 50 +++++++++++++++++++++++++++++----------
 1 file changed, 38 insertions(+), 12 deletions(-)
drivers/iommu/dma-iommu.c +38 -12
index 43520e7275cc..9ce8298047f5 100644
@@ -357,7 +357,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain)
 	atomic_set(&cookie->fq_timer_on, 0);
 	/*
 	 * Prevent incomplete fq state being observable. Pairs with path from
-	 * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
+	 * __iommu_dma_unmap() through __iommu_dma_free_iova() to queue_iova()
 	 */
 	smp_wmb();
 	WRITE_ONCE(cookie->fq_domain, domain);
@@ -758,7 +758,7 @@ static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
 	}
 }
 
-static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
+static dma_addr_t __iommu_dma_alloc_iova(struct iommu_domain *domain,
 		size_t size, u64 dma_limit, struct device *dev)
 {
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -804,7 +804,7 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
 	return (dma_addr_t)iova << shift;
 }
 
-static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
+static void __iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
 		dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
 {
 	struct iova_domain *iovad = &cookie->iovad;
@@ -841,7 +841,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
 
 	if (!iotlb_gather.queued)
 		iommu_iotlb_sync(domain, &iotlb_gather);
-	iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
+	__iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
 }
 
 static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -864,12 +864,12 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
 
 	size = iova_align(iovad, size + iova_off);
 
-	iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev);
+	iova = __iommu_dma_alloc_iova(domain, size, dma_mask, dev);
 	if (!iova)
 		return DMA_MAPPING_ERROR;
 
 	if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) {
-		iommu_dma_free_iova(cookie, iova, size, NULL);
+		__iommu_dma_free_iova(cookie, iova, size, NULL);
 		return DMA_MAPPING_ERROR;
 	}
 	return iova + iova_off;
@@ -973,7 +973,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
 		return NULL;
 
 	size = iova_align(iovad, size);
-	iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
+	iova = __iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
 	if (!iova)
 		goto out_free_pages;
 
@@ -1007,7 +1007,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
 out_free_sg:
 	sg_free_table(sgt);
 out_free_iova:
-	iommu_dma_free_iova(cookie, iova, size, NULL);
+	__iommu_dma_free_iova(cookie, iova, size, NULL);
 out_free_pages:
 	__iommu_dma_free_pages(pages, count);
 	return NULL;
@@ -1442,7 +1442,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
 	if (!iova_len)
 		return __finalise_sg(dev, sg, nents, 0);
 
-	iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
+	iova = __iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
 	if (!iova) {
 		ret = -ENOMEM;
 		goto out_restore_sg;
@@ -1459,7 +1459,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
 	return __finalise_sg(dev, sg, nents, iova);
 
 out_free_iova:
-	iommu_dma_free_iova(cookie, iova, iova_len, NULL);
+	__iommu_dma_free_iova(cookie, iova, iova_len, NULL);
 out_restore_sg:
 	__invalidate_sg(sg, nents);
 out:
@@ -1720,6 +1720,30 @@ static size_t iommu_dma_max_mapping_size(struct device *dev)
 	return SIZE_MAX;
 }
 
+static dma_addr_t iommu_dma_alloc_iova(struct device *dev, size_t size)
+{
+	struct iommu_domain *domain = iommu_get_dma_domain(dev);
+	struct iommu_dma_cookie *cookie = domain->iova_cookie;
+	struct iova_domain *iovad = &cookie->iovad;
+	dma_addr_t dma_mask = dma_get_mask(dev);
+
+	size = iova_align(iovad, size);
+	return __iommu_dma_alloc_iova(domain, size, dma_mask, dev);
+}
+
+static void iommu_dma_free_iova(struct device *dev, dma_addr_t iova,
+				size_t size)
+{
+	struct iommu_domain *domain = iommu_get_dma_domain(dev);
+	struct iommu_dma_cookie *cookie = domain->iova_cookie;
+	struct iova_domain *iovad = &cookie->iovad;
+	struct iommu_iotlb_gather iotlb_gather;
+
+	size = iova_align(iovad, size);
+	iommu_iotlb_gather_init(&iotlb_gather);
+	__iommu_dma_free_iova(cookie, iova, size, &iotlb_gather);
+}
+
 static const struct dma_map_ops iommu_dma_ops = {
 	.flags			= DMA_F_PCI_P2PDMA_SUPPORTED |
 				  DMA_F_CAN_SKIP_SYNC,
@@ -1744,6 +1768,8 @@ static const struct dma_map_ops iommu_dma_ops = {
 	.get_merge_boundary	= iommu_dma_get_merge_boundary,
 	.opt_mapping_size	= iommu_dma_opt_mapping_size,
 	.max_mapping_size       = iommu_dma_max_mapping_size,
+	.alloc_iova		= iommu_dma_alloc_iova,
+	.free_iova		= iommu_dma_free_iova,
 };
 
 void iommu_setup_dma_ops(struct device *dev)
@@ -1786,7 +1812,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
 	if (!msi_page)
 		return NULL;
 
-	iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
+	iova = __iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
 	if (!iova)
 		goto out_free_page;
 
@@ -1800,7 +1826,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
 	return msi_page;
 
 out_free_iova:
-	iommu_dma_free_iova(cookie, iova, size, NULL);
+	__iommu_dma_free_iova(cookie, iova, size, NULL);
 out_free_page:
 	kfree(msi_page);
 	return NULL;
Signature · 1 line
2.26.3
author
#4 lore raw

PATCH 5/42 iommu/dma: Prepare map/unmap page functions to receive IOVA

In reply to Oak Zeng

From: Leon Romanovsky <[email protected]>

Extend the existing map_page/unmap_page function implementations to get preallocated IOVA. In such case, the IOVA allocation needs to be skipped, but rest of the code stays the same.

---
 drivers/iommu/dma-iommu.c | 68 ++++++++++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 23 deletions(-)
drivers/iommu/dma-iommu.c +45 -23
index 9ce8298047f5..dbef2581a98c 100644
@@ -822,7 +822,7 @@ static void __iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
 }
 
 static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
-		size_t size)
+			      size_t size, bool free_iova)
 {
 	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -841,17 +841,19 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
 
 	if (!iotlb_gather.queued)
 		iommu_iotlb_sync(domain, &iotlb_gather);
-	__iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
+	if (free_iova)
+		__iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
 }
 
 static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
-		size_t size, int prot, u64 dma_mask)
+				  dma_addr_t iova, size_t size, int prot,
+				  u64 dma_mask)
 {
 	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 	struct iova_domain *iovad = &cookie->iovad;
 	size_t iova_off = iova_offset(iovad, phys);
-	dma_addr_t iova;
+	bool no_iova = !iova;
 
 	if (static_branch_unlikely(&iommu_deferred_attach_enabled) &&
 	    iommu_deferred_attach(dev, domain))
@@ -864,12 +866,14 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
 
 	size = iova_align(iovad, size + iova_off);
 
-	iova = __iommu_dma_alloc_iova(domain, size, dma_mask, dev);
+	if (no_iova)
+		iova = __iommu_dma_alloc_iova(domain, size, dma_mask, dev);
 	if (!iova)
 		return DMA_MAPPING_ERROR;
 
 	if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) {
-		__iommu_dma_free_iova(cookie, iova, size, NULL);
+		if (no_iova)
+			__iommu_dma_free_iova(cookie, iova, size, NULL);
 		return DMA_MAPPING_ERROR;
 	}
 	return iova + iova_off;
@@ -1034,7 +1038,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
 	return vaddr;
 
 out_unmap:
-	__iommu_dma_unmap(dev, *dma_handle, size);
+	__iommu_dma_unmap(dev, *dma_handle, size, true);
 	__iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
 	return NULL;
 }
@@ -1063,7 +1067,7 @@ static void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
 {
 	struct dma_sgt_handle *sh = sgt_handle(sgt);
 
-	__iommu_dma_unmap(dev, sgt->sgl->dma_address, size);
+	__iommu_dma_unmap(dev, sgt->sgl->dma_address, size, true);
 	__iommu_dma_free_pages(sh->pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
 	sg_free_table(&sh->sgt);
 	kfree(sh);
@@ -1134,9 +1138,11 @@ static void iommu_dma_sync_sg_for_device(struct device *dev,
 			arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
 }
 
-static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
-		unsigned long offset, size_t size, enum dma_data_direction dir,
-		unsigned long attrs)
+static dma_addr_t __iommu_dma_map_pages(struct device *dev, struct page *page,
+					unsigned long offset, dma_addr_t iova,
+					size_t size,
+					enum dma_data_direction dir,
+					unsigned long attrs)
 {
 	phys_addr_t phys = page_to_phys(page) + offset;
 	bool coherent = dev_is_dma_coherent(dev);
@@ -1144,7 +1150,7 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
 	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 	struct iova_domain *iovad = &cookie->iovad;
-	dma_addr_t iova, dma_mask = dma_get_mask(dev);
+	dma_addr_t addr, dma_mask = dma_get_mask(dev);
 
 	/*
 	 * If both the physical buffer start address and size are
@@ -1188,14 +1194,23 @@ static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
 	if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		arch_sync_dma_for_device(phys, size, dir);
 
-	iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
-	if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
+	addr = __iommu_dma_map(dev, phys, iova, size, prot, dma_mask);
+	if (addr == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
 		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
-	return iova;
+	return addr;
 }
 
-static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
-		size_t size, enum dma_data_direction dir, unsigned long attrs)
+static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
+				     unsigned long offset, size_t size,
+				     enum dma_data_direction dir,
+				     unsigned long attrs)
+{
+	return __iommu_dma_map_pages(dev, page, offset, 0, size, dir, attrs);
+}
+
+static void __iommu_dma_unmap_pages(struct device *dev, dma_addr_t dma_handle,
+				    size_t size, enum dma_data_direction dir,
+				    unsigned long attrs, bool free_iova)
 {
 	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 	phys_addr_t phys;
@@ -1207,12 +1222,19 @@ static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
 	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev))
 		arch_sync_dma_for_cpu(phys, size, dir);
 
-	__iommu_dma_unmap(dev, dma_handle, size);
+	__iommu_dma_unmap(dev, dma_handle, size, free_iova);
 
 	if (unlikely(is_swiotlb_buffer(dev, phys)))
 		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
 }
 
+static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
+				 size_t size, enum dma_data_direction dir,
+				 unsigned long attrs)
+{
+	__iommu_dma_unmap_pages(dev, dma_handle, size, dir, attrs, true);
+}
+
 /*
  * Prepare a successfully-mapped scatterlist to give back to the caller.
  *
@@ -1515,13 +1537,13 @@ static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
 	}
 
 	if (end)
-		__iommu_dma_unmap(dev, start, end - start);
+		__iommu_dma_unmap(dev, start, end - start, true);
 }
 
 static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
 		size_t size, enum dma_data_direction dir, unsigned long attrs)
 {
-	return __iommu_dma_map(dev, phys, size,
+	return __iommu_dma_map(dev, phys, 0, size,
 			dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
 			dma_get_mask(dev));
 }
@@ -1529,7 +1551,7 @@ static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
 static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
 		size_t size, enum dma_data_direction dir, unsigned long attrs)
 {
-	__iommu_dma_unmap(dev, handle, size);
+	__iommu_dma_unmap(dev, handle, size, true);
 }
 
 static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
@@ -1566,7 +1588,7 @@ static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
 static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
 		dma_addr_t handle, unsigned long attrs)
 {
-	__iommu_dma_unmap(dev, handle, size);
+	__iommu_dma_unmap(dev, handle, size, true);
 	__iommu_dma_free(dev, size, cpu_addr);
 }
 
@@ -1632,7 +1654,7 @@ static void *iommu_dma_alloc(struct device *dev, size_t size,
 	if (!cpu_addr)
 		return NULL;
 
-	*handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
+	*handle = __iommu_dma_map(dev, page_to_phys(page), 0, size, ioprot,
 			dev->coherent_dma_mask);
 	if (*handle == DMA_MAPPING_ERROR) {
 		__iommu_dma_free(dev, size, cpu_addr);
Signature · 1 line
2.26.3
author
#5 lore raw

PATCH 6/42 iommu/dma: Implement link/unlink page callbacks

In reply to Oak Zeng

From: Leon Romanovsky <[email protected]>

Add an implementation of link/unlink interface to perform in map/unmap pages in fast patch for pre-allocated IOVA.

---
 drivers/iommu/dma-iommu.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
drivers/iommu/dma-iommu.c +17 -0
index dbef2581a98c..ddf402e7c8a3 100644
@@ -1766,6 +1766,21 @@ static void iommu_dma_free_iova(struct device *dev, dma_addr_t iova,
 	__iommu_dma_free_iova(cookie, iova, size, &iotlb_gather);
 }
 
+static dma_addr_t iommu_dma_link_range(struct device *dev, struct page *page,
+				       unsigned long offset, dma_addr_t iova,
+				       size_t size, enum dma_data_direction dir,
+				       unsigned long attrs)
+{
+	return __iommu_dma_map_pages(dev, page, offset, iova, size, dir, attrs);
+}
+
+static void iommu_dma_unlink_range(struct device *dev, dma_addr_t addr,
+				   size_t size, enum dma_data_direction dir,
+				   unsigned long attrs)
+{
+	__iommu_dma_unmap_pages(dev, addr, size, dir, attrs, false);
+}
+
 static const struct dma_map_ops iommu_dma_ops = {
 	.flags			= DMA_F_PCI_P2PDMA_SUPPORTED |
 				  DMA_F_CAN_SKIP_SYNC,
@@ -1792,6 +1807,8 @@ static const struct dma_map_ops iommu_dma_ops = {
 	.max_mapping_size       = iommu_dma_max_mapping_size,
 	.alloc_iova		= iommu_dma_alloc_iova,
 	.free_iova		= iommu_dma_free_iova,
+	.link_range		= iommu_dma_link_range,
+	.unlink_range		= iommu_dma_unlink_range,
 };
 
 void iommu_setup_dma_ops(struct device *dev)
Signature · 1 line
2.26.3
author
#6 lore raw

PATCH 7/42 drm: Move GPUVA_START/LAST to drm_gpuvm.h

In reply to Oak Zeng

Move them to .h file so they can be used outside of drm_gpuvm.c. Also add a GPUVA_END micro.

Cc Matthew Brost <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/drm_gpuvm.c | 3 ---
 include/drm/drm_gpuvm.h     | 4 ++++
 2 files changed, 4 insertions(+), 3 deletions(-)
drivers/gpu/drm/drm_gpuvm.c +0 -3
index f9eb56f24bef..7402ed6f1d33 100644
@@ -867,9 +867,6 @@ __drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,
 
 #define to_drm_gpuva(__node)	container_of((__node), struct drm_gpuva, rb.node)
 
-#define GPUVA_START(node) ((node)->va.addr)
-#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
-
 /* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain
  * about this.
  */
include/drm/drm_gpuvm.h +4 -0
index 00d4e43b76b6..429dc0d82eba 100644
@@ -38,6 +38,10 @@ struct drm_gpuvm;
 struct drm_gpuvm_bo;
 struct drm_gpuvm_ops;
 
+#define GPUVA_START(node) ((node)->va.addr)
+#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
+#define GPUVA_END(node) ((node)->va.addr + (node)->va.range)
+
 /**
  * enum drm_gpuva_flags - flags for struct drm_gpuva
  */
Signature · 1 line
2.26.3
author
#7 lore raw

PATCH 8/42 drm/svm: Mark drm_gpuvm to participate SVM

In reply to Oak Zeng

A mm_struct field is added to drm_gpuvm. Also add a parameter to drm_gpuvm_init to say whether this gpuvm participate svm (shared virtual memory with CPU process). Under SVM, CPU program and GPU program share one process virtual address space.

Cc Dave Airlie <[email protected]>
Cc Daniel Vetter <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
---
 drivers/gpu/drm/drm_gpuvm.c            | 7 ++++++-
 drivers/gpu/drm/nouveau/nouveau_uvmm.c | 2 +-
 drivers/gpu/drm/xe/xe_vm.c             | 2 +-
 include/drm/drm_gpuvm.h                | 8 +++++++-
 4 files changed, 15 insertions(+), 4 deletions(-)
drivers/gpu/drm/drm_gpuvm.c +6 -1
index 7402ed6f1d33..5f246ca472a8 100644
@@ -984,6 +984,8 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
  * @reserve_offset: the start of the kernel reserved GPU VA area
  * @reserve_range: the size of the kernel reserved GPU VA area
  * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap
+ * @participate_svm: whether this gpuvm participat shared virtual memory
+ * with CPU mm
  *
  * The &drm_gpuvm must be initialized with this function before use.
  *
@@ -997,7 +999,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
 	       struct drm_gem_object *r_obj,
 	       u64 start_offset, u64 range,
 	       u64 reserve_offset, u64 reserve_range,
-	       const struct drm_gpuvm_ops *ops)
+	       const struct drm_gpuvm_ops *ops, bool participate_svm)
 {
 	gpuvm->rb.tree = RB_ROOT_CACHED;
 	INIT_LIST_HEAD(&gpuvm->rb.list);
@@ -1016,6 +1018,9 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
 	gpuvm->drm = drm;
 	gpuvm->r_obj = r_obj;
 
+	if (participate_svm)
+		gpuvm->mm = current->mm;
+
 	drm_gem_object_get(r_obj);
 
 	drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range);
drivers/gpu/drm/nouveau/nouveau_uvmm.c +1 -1
index ee02cd833c5e..0d11f1733e29 100644
@@ -1861,7 +1861,7 @@ nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
 		       NOUVEAU_VA_SPACE_END,
 		       init->kernel_managed_addr,
 		       init->kernel_managed_size,
-		       &gpuvm_ops);
+		       &gpuvm_ops, false);
 	/* GPUVM takes care from here on. */
 	drm_gem_object_put(r_obj);
 
drivers/gpu/drm/xe/xe_vm.c +1 -1
index ffda487653d8..bcb0a38b31ae 100644
@@ -1363,7 +1363,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 	}
 
 	drm_gpuvm_init(&vm->gpuvm, "Xe VM", DRM_GPUVM_RESV_PROTECTED, &xe->drm,
-		       vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops);
+		       vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops, false);
 
 	drm_gem_object_put(vm_resv_obj);
 
include/drm/drm_gpuvm.h +7 -1
index 429dc0d82eba..838dd7137f07 100644
@@ -242,6 +242,12 @@ struct drm_gpuvm {
 	 * @drm: the &drm_device this VM lives in
 	 */
 	struct drm_device *drm;
+	/**
+	 * @mm: the process &mm_struct which create this gpuvm.
+	 * This is only used for shared virtual memory where virtual
+	 * address space is shared b/t CPU and GPU program.
+	 */
+	struct mm_struct *mm;
 
 	/**
 	 * @mm_start: start of the VA space
@@ -342,7 +348,7 @@ void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
 		    struct drm_gem_object *r_obj,
 		    u64 start_offset, u64 range,
 		    u64 reserve_offset, u64 reserve_range,
-		    const struct drm_gpuvm_ops *ops);
+		    const struct drm_gpuvm_ops *ops, bool participate_svm);
 
 /**
  * drm_gpuvm_get() - acquire a struct drm_gpuvm reference
Signature · 1 line
2.26.3
author
#8 lore raw

PATCH 9/42 drm/svm: introduce drm_mem_region concept

In reply to Oak Zeng

As its name indicates, a drm_mem_region represent a memory region on a drm device, e.g., a GPU's HBM memory.

In a memory region, we have address information of this region, from both CPU (hpa_base) and GPU (dpa_base) perspectives. It has a pagemap member which is used to memremap memory region to ZONE_DEVICE. It also has some interfaces for drm to callback driver to allocate/free memory from this region, migrate data to/from this memory region, get a pagemap owner of a memory region, and get the drm device which own the memory region.

This is introduced for system allocator implementation, so the memory allocation and free interfaces are page based.

A few helper functions are also introduced:
1) drm_mem_region_page_to_dpa: calculate device physical address for a page
2) drm_page_to_mem_region: retrieve drm memory region that a page resides in

v1: use page parameter instead of pfn (Matt) Add BUG_ON(mr != drm_page_to_mem_region(page)) (Matt)

Cc Dave Airlie <[email protected]>
Cc Daniel Vetter <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Christian König <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
---
 include/drm/drm_svm.h | 162 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 162 insertions(+)
 create mode 100644 include/drm/drm_svm.h
include/drm/drm_svm.h new file +162 -0
new file mode 100644
index 000000000000..a383c7251e2b
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _DRM_SVM__
+#define _DRM_SVM__
+
+#include <linux/compiler_types.h>
+#include <linux/memremap.h>
+#include <linux/types.h>
+
+struct dma_fence;
+struct drm_mem_region;
+
+/**
+ * struct migrate_vec - a migration vector is an array of addresses,
+ * each of which represents one page.
+ * When it is system memory page, the address is a dmap-mapped address.
+ * When it is vram page, the address is device physical address.
+ */
+struct migrate_vec {
+	/**
+	 * @mr: the memory region that pages reside.
+	 * When it is system memory page, mr is NULL.
+	 */
+	struct drm_mem_region *mr;
+	/** @npages: number of pages */
+	u64 npages;
+	/**
+	 * @addr_vec: address vector
+	 * each item in addr_vec is an address of a page
+	 */
+	union {
+		/** @dma_addr: dma-mapped address of the page, only valid for system pages*/
+		dma_addr_t dma_addr;
+		/** @dpa: device physical address of the page, only valid for vram page*/
+		phys_addr_t dpa;
+	} addr_vec[1];
+};
+
+/**
+ * struct drm_mem_region_ops - memory region operations such as memory allocation
+ * and migration etc. Driver is supposed to implement those operations.
+ */
+struct drm_mem_region_ops {
+	/**
+	 * @drm_mem_region_alloc_pages: Called from drm to driver to allocate
+	 * device VRAM memory
+	 * @mr: The memory region where to allocate device VRAM memory from
+	 * @npages: number of pages to allocate
+	 * @pfns: Used to return pfns of each page
+	 */
+	int (*drm_mem_region_alloc_pages) (struct drm_mem_region *mr,
+						unsigned long npags, unsigned long *pfns);
+	/**
+	 * @drm_mem_region_free_page: Called from drm to free one page of device memory
+	 * @page: point to the page to free
+	 */
+	void (*drm_mem_region_free_page) (struct page *page);
+	/**
+	 * @drm_mem_region_migrate: Called from drm to migrate memory from src to
+	 * dst. Driver is supposed to implement this function using device hardware
+	 * accelerators such as DMA. DRM subsystem call this function to migrate
+	 * memory b/t system memory and device memory region
+	 *
+	 * @src_vec: source migration vector
+	 * @dst_vec: destination migration vector
+	 */
+	struct dma_fence* (*drm_mem_region_migrate)(struct migrate_vec *src_vec,
+			struct migrate_vec *dst_vec);
+	/**
+	 * @drm_mem_region_pagemap_owner: Return the pagemap owner of a memory
+	 * region. Pagemap owner is the owner of the device memory. It is
+	 * defined by device driver and opaque to drm layer. Drm uses pagemap
+	 * owner to set up page migrations (see hmm function migrate_vma_setup)
+	 * and range population (see hmm functio hmm_range_fault). Driver has
+	 * the freedom to choose the right pagemap owner.
+	 *
+	 * @mr: the memory region which we want to get the page map owner
+	 */
+	void* (*drm_mem_region_pagemap_owner)(struct drm_mem_region *mr);
+	/**
+	 * @drm_mem_region_get_device: Return the drm device which has this memory
+	 * region.
+	 *
+	 * @mr: the memory region
+	 */
+	struct drm_device* (*drm_mem_region_get_device)(struct drm_mem_region *mr);
+};
+
+/**
+ * struct drm_mem_region - memory region structure
+ * This is used to describe a memory region in drm
+ * device, such as HBM memory or CXL extension memory.
+ *
+ * drm_mem_region is converted from the xe_mem_region
+ * concept. xe_mem_region is moved to drm layer and renamed
+ * as drm_mem_region.
+ *
+ * drm_mem_region is supposed to be embedded in some driver struct such as
+ * "struct xe_tile" or "struct amdgpu_device"
+ */
+struct drm_mem_region {
+	/** @dev_private: device private data which is opaque to drm layer */
+	void *dev_private;
+	/** @dpa_base: This memory regions's DPA (device physical address) base */
+	resource_size_t dpa_base;
+	/**
+	 * @usable_size: usable size of VRAM
+	 *
+	 * Usable size of VRAM excluding reserved portions
+	 * (e.g stolen mem)
+	 */
+	resource_size_t usable_size;
+	/** @pagemap: Used to remap device memory as ZONE_DEVICE */
+	struct dev_pagemap pagemap;
+	/**
+	 * @hpa_base: base host physical address
+	 *
+	 * This is generated when remap device memory as ZONE_DEVICE
+	 */
+	resource_size_t hpa_base;
+	/**
+	 * @mr_ops: memory region operation function pointers
+	 */
+	struct drm_mem_region_ops mr_ops;
+};
+
+/**
+ * drm_page_to_mem_region() - Get a page's memory region
+ *
+ * @page: a struct page pointer pointing to a page in vram memory region
+ */
+static inline struct drm_mem_region *drm_page_to_mem_region(struct page *page)
+{
+	return container_of(page->pgmap, struct drm_mem_region, pagemap);
+}
+
+/**
+ * drm_mem_region_page_to_dpa() - Calculate page's device physical address
+ *
+ * @mr: The memory region that page resides in
+ * @page: page to calculate dpa for
+ *
+ * Returns: the device physical address of the page
+ */
+static inline u64 drm_mem_region_page_to_dpa(struct drm_mem_region *mr, struct page *page)
+{
+	u64 pfn = page_to_pfn(page);
+	u64 offset;
+	u64 dpa;
+
+	BUG_ON(mr != drm_page_to_mem_region(page));
+	BUG_ON((pfn << PAGE_SHIFT) < mr->hpa_base );
+	BUG_ON((pfn << PAGE_SHIFT) >=  mr->hpa_base + mr->usable_size);
+	offset = (pfn << PAGE_SHIFT) - mr->hpa_base;
+	dpa = mr->dpa_base + offset;
+
+	return dpa;
+}
+#endif
Signature · 1 line
2.26.3
author
#9 lore raw

PATCH 10/42 drm/svm: introduce hmmptr and helper functions

In reply to Oak Zeng

A hmmptr is a pointer in a CPU program, like a userptr. but unlike a userptr, a hmmptr can also be migrated to device local memory. The other way to look at is, userptr is a special hmmptr without the capability of migration - userptr's backing store is always in system memory.

This is built on top of kernel HMM infrastructure thus is called hmmptr.

This is the key concept to implement SVM (shared virtual memory) at drm drivers. With SVM, all the valid virtual address in a CPU program is also valid for GPU program, if the GPUVM participate SVM. This is implemented thru hmmptr concept.

Helper functions are introduced to init, release, populate and dma-map /unmap hmmptr. Helper will also be introduced to migrate a range of hmmptr to device memory. With those helpers, driver can easily implement the SVM address space mirroring and migration functionalities.

Cc Daniel Vetter <[email protected]>
Cc Dave Airlie <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Christian König <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Leon Romanovsky <[email protected]>
Cc Brian Welty <[email protected]>
Cc Krishna Bommu <[email protected]>
---
 drivers/gpu/drm/Kconfig   |   1 +
 drivers/gpu/drm/Makefile  |   1 +
 drivers/gpu/drm/drm_svm.c | 316 ++++++++++++++++++++++++++++++++++++++
 include/drm/drm_svm.h     |  62 ++++++++
 4 files changed, 380 insertions(+)
 create mode 100644 drivers/gpu/drm/drm_svm.c
drivers/gpu/drm/Kconfig +1 -0
index 981f43d4ca8c..2c18c8d88444 100644
@@ -20,6 +20,7 @@ menuconfig DRM
 # device and dmabuf fd. Let's make sure that is available for our userspace.
 	select KCMP
 	select VIDEO
+	select HMM_MIRROR
 	help
 	  Kernel-level support for the Direct Rendering Infrastructure (DRI)
 	  introduced in XFree86 4.0. If you say Y here, you need to select
drivers/gpu/drm/Makefile +1 -0
index 68cc9258ffc4..0006a37c662a 100644
@@ -89,6 +89,7 @@ drm-$(CONFIG_DRM_PRIVACY_SCREEN) += \
 	drm_privacy_screen_x86.o
 drm-$(CONFIG_DRM_ACCEL) += ../../accel/drm_accel.o
 drm-$(CONFIG_DRM_PANIC) += drm_panic.o
+drm-$(CONFIG_HMM_MIRROR) += ./drm_svm.o
 obj-$(CONFIG_DRM)	+= drm.o
 
 obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
drivers/gpu/drm/drm_svm.c new file +316 -0

316 lines not on this page · raw message

include/drm/drm_svm.h +62 -0

75 lines not on this page · raw message

Signature · 1 line
2.26.3
author
#10 lore raw

PATCH 11/42 drm/svm: Introduce helper to remap drm memory region

In reply to Oak Zeng

Helper function drm_svm_register_mem_region to remap GPU vram using devm_memremap_pages, so each GPU vram page is backed by struct page.

Those struct pages are created to allow hmm migrate buffer b/t GPU vram and CPU system memory using existing Linux migration mechanism (i.e., migrating b/t CPU system memory and hard disk).

This is prepare work to enable svm (shared virtual memory) through Linux kernel hmm framework. The memory remap's page map type is set to MEMORY_DEVICE_PRIVATE for now. This means even though each GPU vram page get a struct page and can be mapped in CPU page table, but such pages are treated as GPU's private resource, so CPU can't access them. If CPU access such page, a page fault is triggered and page will be migrate to system memory.

For GPU device which supports coherent memory protocol b/t CPU and GPU (such as CXL and CAPI protocol), we can remap device memory as MEMORY_DEVICE_COHERENT. This is TBD.

v1: Support a memory type interface for register_mem_region (Himal)

Cc Daniel Vetter <[email protected]>
Cc Dave Airlie <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Christian König <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Leon Romanovsky <[email protected]>
Cc Brian Welty <[email protected]>
---
 drivers/gpu/drm/drm_svm.c | 56 +++++++++++++++++++++++++++++++++++++++
 include/drm/drm_svm.h     |  3 +++
 2 files changed, 59 insertions(+)
drivers/gpu/drm/drm_svm.c +56 -0
index 9a164615f866..741874689e32 100644
@@ -13,6 +13,7 @@
 #include <linux/swap.h>
 #include <linux/bug.h>
 #include <linux/hmm.h>
+#include <linux/pci.h>
 #include <linux/mm.h>
 
 static u64 __npages_in_range(unsigned long start, unsigned long end)
@@ -314,3 +315,58 @@ int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, u
 	return ret;
 }
 EXPORT_SYMBOL_GPL(drm_svm_hmmptr_populate);
+
+static struct dev_pagemap_ops drm_devm_pagemap_ops;
+
+/**
+ * drm_svm_register_mem_region: Remap and provide memmap backing for device memory
+ * @drm: drm device who want to register a memory region
+ * @mr: memory region to register
+ * @type: ZONE_DEVICE memory type
+ *
+ * This remap device memory to host physical address space and create
+ * struct page to back device memory
+ *
+ * Return: 0 on success standard error code otherwise
+ */
+int drm_svm_register_mem_region(const struct drm_device *drm,
+		struct drm_mem_region *mr, enum memory_type type)
+{
+	struct device *dev = &to_pci_dev(drm->dev)->dev;
+	struct resource *res;
+	void *addr;
+	int ret;
+
+	/**FIXME: support MEMORY_DEVICE_COHERENT in the future*/
+	if (type != MEMORY_DEVICE_PRIVATE)
+		return -EINVAL;
+
+	res = devm_request_free_mem_region(dev, &iomem_resource,
+					   mr->usable_size);
+	if (IS_ERR(res)) {
+		ret = PTR_ERR(res);
+		return ret;
+	}
+
+	drm_devm_pagemap_ops.page_free = mr->mr_ops.drm_mem_region_free_page;
+	mr->pagemap.type = type;
+	mr->pagemap.range.start = res->start;
+	mr->pagemap.range.end = res->end;
+	mr->pagemap.nr_range = 1;
+	mr->pagemap.ops = &drm_devm_pagemap_ops;
+	mr->pagemap.owner = mr->mr_ops.drm_mem_region_pagemap_owner(mr);
+	addr = devm_memremap_pages(dev, &mr->pagemap);
+	if (IS_ERR(addr)) {
+		devm_release_mem_region(dev, res->start, resource_size(res));
+		ret = PTR_ERR(addr);
+		drm_err(drm, "Failed to remap memory region %p, errno %d\n",
+				mr, ret);
+		return ret;
+	}
+	mr->hpa_base = res->start;
+
+	drm_info(drm, "Registered device memory [%llx-%llx] to devm, remapped to %pr\n",
+			mr->dpa_base, mr->dpa_base + mr->usable_size, res);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_svm_register_mem_region);
include/drm/drm_svm.h +3 -0
index d443f20b5510..04552cc1c67f 100644
@@ -164,6 +164,9 @@ static inline u64 drm_mem_region_page_to_dpa(struct drm_mem_region *mr, struct p
 	return dpa;
 }
 
+int drm_svm_register_mem_region(const struct drm_device *drm,
+		struct drm_mem_region *mr, enum memory_type type);
+
 /**
  * struct drm_hmmptr- hmmptr pointer
  *
Signature · 1 line
2.26.3
author
#11 lore raw

PATCH 12/42 drm/svm: handle CPU page fault

In reply to Oak Zeng

Under the picture of svm, CPU and GPU program share the same virtual address space. The backing store of this virtual address space can be either in system memory or device memory. Since GPU device memory is remaped as DEVICE_PRIVATE, CPU can't access it. Any CPU access to device memory causes a page fault. Implement a page fault handler to migrate memory back to system memory and map it to CPU page table so the CPU program can proceed.

Cc Daniel Vetter <[email protected]>
Cc Dave Airlie <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Christian König <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Leon Romanovsky <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/drm_svm.c | 272 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 271 insertions(+), 1 deletion(-)
drivers/gpu/drm/drm_svm.c +271 -1
index 741874689e32..80c5b7146ba6 100644
@@ -8,6 +8,7 @@
 #include <linux/mmu_notifier.h>
 #include <linux/dma-mapping.h>
 #include <linux/memremap.h>
+#include <linux/migrate.h>
 #include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_svm.h>
 #include <linux/swap.h>
@@ -316,7 +317,276 @@ int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, u
 }
 EXPORT_SYMBOL_GPL(drm_svm_hmmptr_populate);
 
-static struct dev_pagemap_ops drm_devm_pagemap_ops;
+static void __drm_svm_free_pages(unsigned long *mpfn, unsigned long npages)
+{
+	struct page *page;
+	int j;
+
+	for (j = 0; j < npages; j++) {
+		page = migrate_pfn_to_page(mpfn[j]);
+		mpfn[j] = 0;
+		if (page) {
+			unlock_page(page);
+			put_page(page);
+		}
+	}
+}
+
+/**
+ * __drm_svm_alloc_host_pages() - allocate host pages for the fault vma
+ *
+ * @vma: the fault vma that we need allocate page for
+ * @addr: the virtual address that we allocate pages for
+ * @mpfn: used to output the migration pfns of the allocated pages
+ * @npages: number of pages to allocate
+ *
+ * This function allocate host pages for a specified vma.
+ *
+ * When this function returns, the pages are locked.
+ *
+ * Return 0 on success
+ * error code otherwise
+ */
+static int __drm_svm_alloc_host_pages(struct vm_area_struct *vma,
+					unsigned long addr,
+					u64 npages,
+					unsigned long *mpfn)
+{
+	struct page *page;
+	int i;
+
+	for (i = 0; i < npages; i++) {
+		page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
+		if (unlikely(!page))
+			goto free_allocated;
+
+		/**Lock page per hmm requirement, see hmm.rst*/
+		lock_page(page);
+
+		mpfn[i] = migrate_pfn(page_to_pfn(page));
+		addr += PAGE_SIZE;
+	}
+	return 0;
+
+free_allocated:
+	__drm_svm_free_pages(mpfn, i);
+	return -ENOMEM;
+}
+
+static struct migrate_vec *__generate_migrate_vec_vram(unsigned long *mpfn, bool is_migrate_src, unsigned long npages)
+{
+	struct migrate_vec *vec;
+	int size = sizeof(*vec) + sizeof(vec->addr_vec[1]) * (npages - 1);
+	struct drm_mem_region *mr;
+	struct page *page;
+	u64 dpa;
+	int i, j;
+
+	BUG_ON(is_migrate_src && npages != 1);
+	vec = kzalloc(size, GFP_KERNEL);
+	if (!vec)
+		return NULL;
+
+	mr = drm_page_to_mem_region(page);
+	for(i = 0, j = 0; i < npages; i++) {
+		/**
+		 * We only migrate one page from vram to sram on CPU page fault today.
+		 * If this source page is not marked with _MIGRATE flag, something is
+		 * wrong. We need to report error to core mm.
+		 *
+		 * If we move to multiple pages migration, below logic need a revisit,
+		 * as it is fine to only migrate some pages (but not all) as indicated
+		 * by hmm.
+		 */
+		if (is_migrate_src && !(mpfn[i] & MIGRATE_PFN_MIGRATE)) {
+			pr_err("Migrate from vram to sram: MIGRATE_PFN_MIGRATE flag is not set\n");
+			kfree(vec);
+			return NULL;
+		}
+
+		page = migrate_pfn_to_page(mpfn[i]);
+		if (!page || !is_device_private_page(page)) {
+			pr_err("No page or wrong page zone in %s\n", __func__);
+			kfree(vec);
+			return NULL;
+		}
+
+		dpa = drm_mem_region_page_to_dpa(mr, page);
+		vec->addr_vec[j++].dpa = dpa;
+	}
+	vec->mr = mr;
+	vec->npages = j;
+	return vec;
+}
+
+static struct migrate_vec *__generate_migrate_vec_sram(struct device *dev, unsigned long *mpfn,
+		bool is_migrate_src, unsigned long npages)
+{
+	enum dma_data_direction dir = is_migrate_src ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	struct migrate_vec *vec;
+	int size = sizeof(*vec) + sizeof(vec->addr_vec[1]) * (npages - 1);
+	dma_addr_t dma_addr;
+	struct page *page;
+	int i, j, k;
+
+	page = migrate_pfn_to_page(mpfn[0]);
+	if (unlikely(!page))
+		return NULL;
+
+	vec = kzalloc(size, GFP_KERNEL);
+	if (!vec)
+		return NULL;
+
+	for(i = 0, k =0 ; i < npages; i++) {
+		if (is_migrate_src && !(mpfn[i] & MIGRATE_PFN_MIGRATE))
+			continue;
+
+		page = migrate_pfn_to_page(mpfn[i]);
+		if (!page || is_device_private_page(page)) {
+			pr_err("No page or wrong page zone in %s\n", __func__);
+			goto undo_dma_mapping;
+		}
+
+		dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, dir);
+		if (unlikely(dma_mapping_error(dev, dma_addr)))
+			goto undo_dma_mapping;
+
+		vec->addr_vec[k++].dma_addr = dma_addr;
+	}
+
+	vec->mr = NULL;
+	vec->npages = k;
+	return vec;
+
+undo_dma_mapping:
+	for (j = 0; j < k; j++) {
+		if (vec->addr_vec[j].dma_addr)
+			dma_unmap_page(dev, vec->addr_vec[j].dma_addr, PAGE_SIZE, dir);
+	}
+	kfree(vec);
+	return NULL;
+}
+
+static void __free_migrate_vec_sram(struct device *dev, struct migrate_vec *vec, bool is_migrate_src)
+{
+	enum dma_data_direction dir = is_migrate_src ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	int i;
+
+	BUG_ON(vec->mr != NULL);
+
+	for (i = 0; i < vec->npages; i++) {
+		dma_unmap_page(dev, vec->addr_vec[i].dma_addr, PAGE_SIZE, dir);
+		/** No need to free host pages. migrate_vma_finalize take care */
+	}
+	kfree(vec);
+}
+
+static void __free_migrate_vec_vram(struct migrate_vec *vec)
+{
+	BUG_ON(vec->mr == NULL);
+
+	kfree(vec);
+}
+/**
+ * drm_svm_migrate_to_sram() - Migrate memory back to sram on CPU page fault
+ *
+ * @vmf: cpu vm fault structure, contains fault information such as vma etc.
+ *
+ * Note, this is in CPU's vm fault handler, caller holds the mmap read lock.
+ *
+ * This function migrate one page at the fault address. This is the normal
+ * core mm page fault scheme. Linux doesn't aggressively prefault at CPU page
+ * fault time. It only fault-in one page to recover the fault address. Even
+ * if we migrate more than one page, core mm still only program one pte entry
+ * (covers one page).See logic in function handle_pte_fault. do_swap_page
+ * eventually calls to drm_svm_migrate_to_sram in our case.
+ *
+ * We call migrate_vma_setup to set up the migration. During migrate_vma_setup,
+ * device page table is invalidated before migration (by calling the driver registered
+ * mmu notifier)
+ *
+ * We call migrate_vma_finalize to finalize the migration. During migrate_vma_finalize,
+ * device pages of the source buffer is freed (by calling memory region's
+ * drm_mem_region_free_page callback function)
+ *
+ * Return:
+ * 0 on success
+ * VM_FAULT_SIGBUS: failed to migrate page to system memory, application
+ * will be signaled a SIGBUG
+ */
+static vm_fault_t drm_svm_migrate_to_sram(struct vm_fault *vmf)
+{
+	struct drm_mem_region *mr = drm_page_to_mem_region(vmf->page);
+	struct drm_device *drm = mr->mr_ops.drm_mem_region_get_device(mr);
+	unsigned long src_pfn = 0, dst_pfn = 0;
+	struct device *dev = drm->dev;
+	struct vm_area_struct *vma = vmf->vma;
+	struct migrate_vec *src;
+	struct migrate_vec *dst;
+	struct dma_fence *fence;
+	vm_fault_t ret = 0, r;
+
+	struct migrate_vma migrate_vma = {
+		.vma		= vma,
+		.start		= ALIGN_DOWN(vmf->address, PAGE_SIZE),
+		.end		= ALIGN_DOWN(vmf->address, PAGE_SIZE) + PAGE_SIZE,
+		.src		= &src_pfn,
+		.dst		= &dst_pfn,
+		.pgmap_owner	= mr->mr_ops.drm_mem_region_pagemap_owner(mr),
+		.flags		= MIGRATE_VMA_SELECT_DEVICE_PRIVATE,
+		.fault_page = vmf->page,
+	};
+
+	if (migrate_vma_setup(&migrate_vma) < 0)
+		return VM_FAULT_SIGBUS;
+
+	if (!migrate_vma.cpages)
+		return 0;
+
+	r = __drm_svm_alloc_host_pages(vma, migrate_vma.start, 1, migrate_vma.dst);
+	if (r) {
+		ret = VM_FAULT_OOM;
+		goto migrate_finalize;
+	}
+
+	src = __generate_migrate_vec_vram(migrate_vma.src, true, 1);
+	if (!src) {
+		ret = VM_FAULT_SIGBUS;
+		goto free_host_pages;
+	}
+
+	dst = __generate_migrate_vec_sram(dev, migrate_vma.dst, false, 1);
+	if (!dst) {
+		ret = VM_FAULT_SIGBUS;
+		goto free_migrate_src;
+	}
+
+	fence = mr->mr_ops.drm_mem_region_migrate(src, dst);
+	if (IS_ERR(fence)) {
+		ret = VM_FAULT_SIGBUS;
+		goto free_migrate_dst;
+	}
+	dma_fence_wait(fence, false);
+	dma_fence_put(fence);
+
+	migrate_vma_pages(&migrate_vma);
+
+free_migrate_dst:
+	__free_migrate_vec_sram(dev, dst, false);
+free_migrate_src:
+	__free_migrate_vec_vram(src);
+free_host_pages:
+	if (ret)
+		__drm_svm_free_pages(migrate_vma.dst, 1);
+migrate_finalize:
+	if (ret)
+		memset(migrate_vma.dst, 0, sizeof(*migrate_vma.dst));
+	migrate_vma_finalize(&migrate_vma);
+	return ret;
+}
+static struct dev_pagemap_ops drm_devm_pagemap_ops = {
+	.migrate_to_ram = drm_svm_migrate_to_sram,
+};
 
 /**
  * drm_svm_register_mem_region: Remap and provide memmap backing for device memory
Signature · 1 line
2.26.3
author
#12 lore raw

PATCH 13/42 drm/svm: Migrate a range of hmmptr to vram

In reply to Oak Zeng

Introduce a helper function drm_svm_migrate_hmmptr_to_vram to migrate any sub-range of a hmmptr to vram. The range has to be at page boundary. This is supposed to be called by the driver to migrate a hmmptr to vram.

Cc Daniel Vetter <[email protected]>
Cc Dave Airlie <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Christian König <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/drm_svm.c | 121 ++++++++++++++++++++++++++++++++++++++
 include/drm/drm_svm.h     |   3 +
 2 files changed, 124 insertions(+)
drivers/gpu/drm/drm_svm.c +121 -0
index 80c5b7146ba6..d34809b2bad6 100644
@@ -640,3 +640,124 @@ int drm_svm_register_mem_region(const struct drm_device *drm,
 	return 0;
 }
 EXPORT_SYMBOL_GPL(drm_svm_register_mem_region);
+
+static void __drm_svm_init_device_pages(unsigned long *pfn, unsigned long npages)
+{
+	struct page *page;
+	int i;
+
+	for(i = 0; i < npages; i++) {
+		page = pfn_to_page(pfn[i]);
+		zone_device_page_init(page);
+		pfn[i] = migrate_pfn(pfn[i]);
+	}
+}
+
+/**
+ * drm_svm_migrate_hmmptr_to_vram() - migrate a sub-range of a hmmptr to vram
+ * Must be called with mmap_read_lock held.
+ *
+ * @vm: the vm that the hmmptr belongs to
+ * @mr: the destination memory region we want to migrate to
+ * @hmmptr: the hmmptr to migrate.
+ * @start: start(CPU virtual address, inclusive) of the range to migrate
+ * @end: end(CPU virtual address, exclusive) of the range to migrate
+ *
+ * Returns: negative errno on faiure, 0 on success
+ */
+int drm_svm_migrate_hmmptr_to_vram(struct drm_gpuvm *vm,
+		struct drm_mem_region *mr,
+		struct drm_hmmptr *hmmptr, unsigned long start, unsigned long end)
+{
+	struct drm_device *drm = mr->mr_ops.drm_mem_region_get_device(mr);
+	struct mm_struct *mm = vm->mm;
+	unsigned long npages = __npages_in_range(start, end);
+	struct vm_area_struct *vas;
+	struct migrate_vma migrate = {
+		.start		= ALIGN_DOWN(start, PAGE_SIZE),
+		.end		= ALIGN(end, PAGE_SIZE),
+		.pgmap_owner	= mr->mr_ops.drm_mem_region_pagemap_owner(mr),
+		.flags          = MIGRATE_VMA_SELECT_SYSTEM,
+	};
+	struct device *dev = drm->dev;
+	struct dma_fence *fence;
+	struct migrate_vec *src;
+	struct migrate_vec *dst;
+	int ret = 0;
+	void *buf;
+
+	mmap_assert_locked(mm);
+
+	BUG_ON(start < __hmmptr_cpu_start(hmmptr));
+	BUG_ON(end > __hmmptr_cpu_end(hmmptr));
+
+	vas = find_vma_intersection(mm, start, end);
+	if (!vas)
+		return -ENOENT;
+
+	migrate.vma = vas;
+	buf = kvcalloc(npages, 2* sizeof(*migrate.src), GFP_KERNEL);
+	if(!buf)
+		return -ENOMEM;
+
+	migrate.src = buf;
+	migrate.dst = migrate.src + npages;
+	ret = migrate_vma_setup(&migrate);
+	if (ret) {
+		drm_warn(drm, "vma setup returned %d for range [0x%lx - 0x%lx]\n",
+				ret, start, end);
+		goto free_buf;
+	}
+
+	/**
+	 * Partial migration is just normal. Print a message for now.
+	 * Once this behavior is verified, delete this warning.
+	 */
+	if (migrate.cpages != npages)
+		drm_warn(drm, "Partial migration for range [0x%lx - 0x%lx], range is %ld pages, migrate only %ld pages\n",
+				start, end, npages, migrate.cpages);
+
+	ret = mr->mr_ops.drm_mem_region_alloc_pages(mr, migrate.cpages, migrate.dst);
+	if (ret)
+		goto migrate_finalize;
+
+	__drm_svm_init_device_pages(migrate.dst, migrate.cpages);
+
+	src = __generate_migrate_vec_sram(dev, migrate.src, true, npages);
+	if (!src) {
+		ret = -EFAULT;
+		goto free_device_pages;
+	}
+
+	dst = __generate_migrate_vec_vram(migrate.dst, false, migrate.cpages);
+	if (!dst) {
+		ret = -EFAULT;
+		goto free_migrate_src;
+	}
+
+	fence = mr->mr_ops.drm_mem_region_migrate(src, dst);
+	if (IS_ERR(fence)) {
+		ret = -EIO;
+		goto free_migrate_dst;
+	}
+	dma_fence_wait(fence, false);
+	dma_fence_put(fence);
+
+	migrate_vma_pages(&migrate);
+
+free_migrate_dst:
+	__free_migrate_vec_vram(dst);
+free_migrate_src:
+	__free_migrate_vec_sram(dev, src, true);
+free_device_pages:
+	if (ret)
+		__drm_svm_free_pages(migrate.dst, migrate.cpages);
+migrate_finalize:
+	if (ret)
+		memset(migrate.dst, 0, sizeof(*migrate.dst)*migrate.cpages);
+	migrate_vma_finalize(&migrate);
+free_buf:
+	kvfree(buf);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_svm_migrate_hmmptr_to_vram);
include/drm/drm_svm.h +3 -0
index 04552cc1c67f..1cbf83427f2a 100644
@@ -224,4 +224,7 @@ void drm_svm_hmmptr_map_dma_pages(struct drm_hmmptr *hmmptr, u64 page_idx, u64 n
 void drm_svm_hmmptr_unmap_dma_pages(struct drm_hmmptr *hmmptr, u64 page_idx, u64 npages);
 int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, u64 end,
             bool write, bool is_mmap_locked);
+int drm_svm_migrate_hmmptr_to_vram(struct drm_gpuvm *vm,
+		struct drm_mem_region *mr,
+		struct drm_hmmptr *hmmptr, unsigned long start, unsigned long end);
 #endif
Signature · 1 line
2.26.3
author
#13 lore raw

PATCH 14/42 drm/svm: Add DRM SVM documentation

In reply to Oak Zeng

The purpose of DRM SVM design is to provide helpers and infrastructure to facilitate driver implementation of SVM (shared virtual memory) function.

The overview part described the helpers that DRM layer provdes and the virtual function interfaces that driver has to implement if driver use drm SVM layer to implement SVM functionality.

Unlike legacy DRM BO (buffer object) based physical memory management, DRM SVM use a complete page centric memory management. This has fundamental impact to the memory allocate, free, eviction and migration logic. It also requires the driver to support range (which is required to be page boundary) based GPU page table update and invalidation. The page centric design section described this design.

Lock design is one of the key of the design, which is described in the lock design section.

There are other sections described the memory attributes API design and the memory eviction design.

Create a DRM SVM section in drm-mm.rst and generate SVM documents under this document section.

Cc Daniel Vetter <[email protected]>
Cc Dave Airlie <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Christian König <[email protected]>
Cc Felix Kuehling <[email protected]>
Cc Jason Gunthorpe <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 Documentation/gpu/drm-mm.rst |  42 +++++
 drivers/gpu/drm/drm_svm.c    | 309 +++++++++++++++++++++++++++++++++++
 2 files changed, 351 insertions(+)
Documentation/gpu/drm-mm.rst +42 -0
index d55751cad67c..fa968abb9387 100644
@@ -573,3 +573,45 @@ Scheduler Function References
 
 .. kernel-doc:: drivers/gpu/drm/scheduler/sched_entity.c
    :export:
+
+DRM SVM
+======
+
+Overview
+-------
+
+.. kernel-doc:: drivers/gpu/drm/drm_svm.c
+   :doc: Overview
+
+Page centric design
+------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_svm.c
+   :doc: page centric design
+
+Lock design
+-----------
+
+.. kernel-doc:: drivers/gpu/drm/drm_svm.c
+   :doc: lock design
+
+Memory hints
+-----------
+
+.. kernel-doc:: drivers/gpu/drm/drm_svm.c
+   :doc: Memory hints
+
+Memory eviction
+-----------
+
+.. kernel-doc:: drivers/gpu/drm/drm_svm.c
+   :doc: pag granularity eviction
+
+Function References
+-----------------------------
+
+.. kernel-doc:: include/drm/drm_svm.h
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/drm_svm.c
+   :export:
drivers/gpu/drm/drm_svm.c +309 -0

315 lines not on this page · raw message

Signature · 1 line
2.26.3
author
#14 lore raw

PATCH 15/42 drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

Engine is old nomenclature, replace with exec queue.

---
 drivers/gpu/drm/xe/xe_migrate.c | 9 ++++-----
 drivers/gpu/drm/xe/xe_migrate.h | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)
drivers/gpu/drm/xe/xe_migrate.c +4 -5
index ddd50c3f7208..a140c46ef969 100644
@@ -84,15 +84,14 @@ struct xe_migrate {
 #define MAX_PTE_PER_SDI 0x1FE
 
 /**
- * xe_tile_migrate_engine() - Get this tile's migrate engine.
+ * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue.
  * @tile: The tile.
  *
- * Returns the default migrate engine of this tile.
- * TODO: Perhaps this function is slightly misplaced, and even unneeded?
+ * Returns the default migrate exec queue of this tile.
  *
- * Return: The default migrate engine
+ * Return: The default migrate exec queue
  */
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile)
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)
 {
 	return tile->migrate->q;
 }
drivers/gpu/drm/xe/xe_migrate.h +1 -1
index 951f19318ea4..a5bcaafe4a99 100644
@@ -106,5 +106,5 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
 
 void xe_migrate_wait(struct xe_migrate *m);
 
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile);
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
 #endif
Signature · 1 line
2.26.3
author
#15 lore raw

PATCH 16/42 drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

Each xe_vma_op resolves to 0-3 pt_ops. Add storage for the pt_ops to xe_vma_ops which is dynamically allocated based the number and types of xe_vma_op in the xe_vma_ops list. Allocation only implemented in this patch.

This will help with converting xe_vma_ops (multiple xe_vma_op) in a atomic update unit.

Cc Oak Zeng <[email protected]>
Cc Thomas Hellström <[email protected]>
---
 drivers/gpu/drm/xe/xe_pt_types.h | 12 ++++++
 drivers/gpu/drm/xe/xe_vm.c       | 66 +++++++++++++++++++++++++++++++-
 drivers/gpu/drm/xe/xe_vm_types.h |  8 ++++
 3 files changed, 84 insertions(+), 2 deletions(-)
drivers/gpu/drm/xe/xe_pt_types.h +12 -0
index cee70cb0f014..2093150f461e 100644
@@ -74,4 +74,16 @@ struct xe_vm_pgtable_update {
 	u32 flags;
 };
 
+/** struct xe_vm_pgtable_update_op - Page table update operation */
+struct xe_vm_pgtable_update_op {
+	/** @entries: entries to update for this operation */
+	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
+	/** @num_entries: number of entries for this update operation */
+	u32 num_entries;
+	/** @bind: is a bind */
+	bool bind;
+	/** @rebind: is a rebind */
+	bool rebind;
+};
+
 #endif
drivers/gpu/drm/xe/xe_vm.c +64 -2

143 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm_types.h +8 -0
index ce1a63a5e3e7..211c88801182 100644
@@ -21,6 +21,7 @@ struct xe_bo;
 struct xe_sync_entry;
 struct xe_user_fence;
 struct xe_vm;
+struct xe_vm_pgtable_update_op;
 
 #define XE_VMA_READ_ONLY	DRM_GPUVA_USERBITS
 #define XE_VMA_DESTROYED	(DRM_GPUVA_USERBITS << 1)
@@ -368,6 +369,13 @@ struct xe_vma_ops {
 	struct xe_sync_entry *syncs;
 	/** @num_syncs: number of syncs */
 	u32 num_syncs;
+	/** @pt_update_ops: page table update operations */
+	struct {
+		/** @ops: operations */
+		struct xe_vm_pgtable_update_op *ops;
+		/** @num_ops: number of operations */
+		u32 num_ops;
+	} pt_update_ops[XE_MAX_TILES_PER_DEVICE];
 };
 
 #endif
Signature · 1 line
2.26.3
author
#16 lore raw

PATCH 17/42 drm/xe: Convert multiple bind ops into single job

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

This aligns with the uAPI of an array of binds or single bind that results in multiple GPUVA ops to be considered a single atomic operations.

The implemenation is roughly:
- xe_vma_ops is a list of xe_vma_op (GPUVA op)
- each xe_vma_op resolves to 0-3 PT ops
- xe_vma_ops creates a single job
- if at any point during binding a failure occurs, xe_vma_ops contains the information necessary unwind the PT and VMA (GPUVA) state

v2:
- add missing dma-resv slot reservation (CI, testing)

Cc Oak Zeng <[email protected]>
Cc Thomas Hellström <[email protected]>
---
 drivers/gpu/drm/xe/xe_bo_types.h |    2 +
 drivers/gpu/drm/xe/xe_migrate.c  |  301 ++++----
 drivers/gpu/drm/xe/xe_migrate.h  |   32 +-
 drivers/gpu/drm/xe/xe_pt.c       | 1108 +++++++++++++++++++-----------
 drivers/gpu/drm/xe/xe_pt.h       |   14 +-
 drivers/gpu/drm/xe/xe_pt_types.h |   36 +
 drivers/gpu/drm/xe/xe_vm.c       |  519 +++-----------
 drivers/gpu/drm/xe/xe_vm.h       |    2 +
 drivers/gpu/drm/xe/xe_vm_types.h |   45 +-
 9 files changed, 1036 insertions(+), 1023 deletions(-)
drivers/gpu/drm/xe/xe_bo_types.h +2 -0
index 86422e113d39..02d68873558a 100644
@@ -58,6 +58,8 @@ struct xe_bo {
 #endif
 	/** @freed: List node for delayed put. */
 	struct llist_node freed;
+	/** @update_index: Update index if PT BO */
+	int update_index;
 	/** @created: Whether the bo has passed initial creation */
 	bool created;
 
drivers/gpu/drm/xe/xe_migrate.c +137 -164

415 lines not on this page · raw message

drivers/gpu/drm/xe/xe_migrate.h +20 -12
index a5bcaafe4a99..453e0ecf5034 100644
@@ -47,6 +47,24 @@ struct xe_migrate_pt_update_ops {
 			 struct xe_tile *tile, struct iosys_map *map,
 			 void *pos, u32 ofs, u32 num_qwords,
 			 const struct xe_vm_pgtable_update *update);
+	/**
+	 * @clear: Clear a command buffer or page-table with ptes.
+	 * @pt_update: Embeddable callback argument.
+	 * @tile: The tile for the current operation.
+	 * @map: struct iosys_map into the memory to be populated.
+	 * @pos: If @map is NULL, map into the memory to be populated.
+	 * @ofs: qword offset into @map, unused if @map is NULL.
+	 * @num_qwords: Number of qwords to write.
+	 * @update: Information about the PTEs to be inserted.
+	 *
+	 * This interface is intended to be used as a callback into the
+	 * page-table system to populate command buffers or shared
+	 * page-tables with PTEs.
+	 */
+	void (*clear)(struct xe_migrate_pt_update *pt_update,
+		      struct xe_tile *tile, struct iosys_map *map,
+		      void *pos, u32 ofs, u32 num_qwords,
+		      const struct xe_vm_pgtable_update *update);
 
 	/**
 	 * @pre_commit: Callback to be called just before arming the
@@ -67,14 +85,10 @@ struct xe_migrate_pt_update_ops {
 struct xe_migrate_pt_update {
 	/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */
 	const struct xe_migrate_pt_update_ops *ops;
-	/** @vma: The vma we're updating the pagetable for. */
-	struct xe_vma *vma;
+	/** @vops: VMA operations */
+	struct xe_vma_ops *vops;
 	/** @job: The job if a GPU page-table update. NULL otherwise */
 	struct xe_sched_job *job;
-	/** @start: Start of update for the range fence */
-	u64 start;
-	/** @last: Last of update for the range fence */
-	u64 last;
 	/** @tile_id: Tile ID of the update */
 	u8 tile_id;
 };
@@ -96,12 +110,6 @@ struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
 
 struct dma_fence *
 xe_migrate_update_pgtables(struct xe_migrate *m,
-			   struct xe_vm *vm,
-			   struct xe_bo *bo,
-			   struct xe_exec_queue *q,
-			   const struct xe_vm_pgtable_update *updates,
-			   u32 num_updates,
-			   struct xe_sync_entry *syncs, u32 num_syncs,
 			   struct xe_migrate_pt_update *pt_update);
 
 void xe_migrate_wait(struct xe_migrate *m);
drivers/gpu/drm/xe/xe_pt.c +706 -402

1,260 lines not on this page · raw message

drivers/gpu/drm/xe/xe_pt.h +6 -8
index 71a4fbfcff43..9ab386431cad 100644
@@ -17,6 +17,7 @@ struct xe_sync_entry;
 struct xe_tile;
 struct xe_vm;
 struct xe_vma;
+struct xe_vma_ops;
 
 /* Largest huge pte is currently 1GiB. May become device dependent. */
 #define MAX_HUGEPTE_LEVEL 2
@@ -34,14 +35,11 @@ void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
 
 void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred);
 
-struct dma_fence *
-__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
-		 struct xe_sync_entry *syncs, u32 num_syncs,
-		 bool rebind);
-
-struct dma_fence *
-__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
-		   struct xe_sync_entry *syncs, u32 num_syncs);
+int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops);
+struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
+				       struct xe_vma_ops *vops);
+void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
 
 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);
 
drivers/gpu/drm/xe/xe_pt_types.h +36 -0
index 2093150f461e..384cc04de719 100644
@@ -78,6 +78,8 @@ struct xe_vm_pgtable_update {
 struct xe_vm_pgtable_update_op {
 	/** @entries: entries to update for this operation */
 	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
+	/** @vma: VMA for operation, operation not valid if NULL */
+	struct xe_vma *vma;
 	/** @num_entries: number of entries for this update operation */
 	u32 num_entries;
 	/** @bind: is a bind */
@@ -86,4 +88,38 @@ struct xe_vm_pgtable_update_op {
 	bool rebind;
 };
 
+/** struct xe_vm_pgtable_update_ops: page table update operations */
+struct xe_vm_pgtable_update_ops {
+	/** @ops: operations */
+	struct xe_vm_pgtable_update_op *ops;
+	/** @deferred: deferred list to destroy PT entries */
+	struct llist_head deferred;
+	/** @q: exec queue for PT operations */
+	struct xe_exec_queue *q;
+	/** @start: start address of ops */
+	u64 start;
+	/** @last: last address of ops */
+	u64 last;
+	/** @num_ops: number of operations */
+	u32 num_ops;
+	/** @current_op: current operations */
+	u32 current_op;
+	/** @needs_userptr_lock: Needs userptr lock */
+	bool needs_userptr_lock;
+	/** @needs_invalidation: Needs invalidation */
+	bool needs_invalidation;
+	/**
+	 * @wait_vm_bookkeep: PT operations need to wait until VM is idle
+	 * (bookkeep dma-resv slots are idle) and stage all future VM activity
+	 * behind these operations (install PT operations into VM kernel
+	 * dma-resv slot).
+	 */
+	bool wait_vm_bookkeep;
+	/**
+	 * @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv
+	 * slots are idle.
+	 */
+	bool wait_vm_kernel;
+};
+
 #endif
drivers/gpu/drm/xe/xe_vm.c +113 -406

633 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +2 -0
index b481608b12f1..c864dba35e1d 100644
@@ -259,6 +259,8 @@ static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)
 	return drm_gpuvm_resv(&vm->gpuvm);
 }
 
+void xe_vm_kill(struct xe_vm *vm, bool unlocked);
+
 /**
  * xe_vm_assert_held(vm) - Assert that the vm's reservation object is held.
  * @vm: The vm
drivers/gpu/drm/xe/xe_vm_types.h +14 -31

84 lines not on this page · raw message

Signature · 1 line
2.26.3
author
#17 lore raw

PATCH 18/42 drm/xe: Update VM trace events

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

The trace events have changed moving to a single job per VM bind IOCTL, update the trace events align with old behavior as much as possible.

v1: Rebase (Oak)

Cc Oak Zeng <[email protected]>
Cc Thomas Hellström <[email protected]>
---
 drivers/gpu/drm/xe/xe_trace_bo.h | 10 ++++----
 drivers/gpu/drm/xe/xe_vm.c       | 42 ++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 7 deletions(-)
drivers/gpu/drm/xe/xe_trace_bo.h +5 -5
index f39f09ed3495..9b1a1d4304ae 100644
@@ -117,11 +117,6 @@ DEFINE_EVENT(xe_vma, xe_vma_acc,
 	     TP_ARGS(vma)
 );
 
-DEFINE_EVENT(xe_vma, xe_vma_fail,
-	     TP_PROTO(struct xe_vma *vma),
-	     TP_ARGS(vma)
-);
-
 DEFINE_EVENT(xe_vma, xe_vma_bind,
 	     TP_PROTO(struct xe_vma *vma),
 	     TP_ARGS(vma)
@@ -237,6 +232,11 @@ DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_exit,
 	     TP_ARGS(vm)
 );
 
+DEFINE_EVENT(xe_vm, xe_vm_ops_fail,
+	     TP_PROTO(struct xe_vm *vm),
+	     TP_ARGS(vm)
+);
+
 #endif
 
 /* This part must be outside protection */
drivers/gpu/drm/xe/xe_vm.c +40 -2

66 lines not on this page · raw message

Signature · 1 line
2.26.3
author
#18 lore raw

PATCH 19/42 drm/xe: Update PT layer with better error handling

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

Update PT layer so if a memory allocation for a PTE fails the error can be propagated to the user without requiring to be killed.

---
 drivers/gpu/drm/xe/xe_pt.c | 208 ++++++++++++++++++++++++++++---------
 1 file changed, 160 insertions(+), 48 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +160 -48
index 8c61bb2c034d..62aec1a42249 100644
@@ -845,19 +845,27 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *t
 	}
 }
 
-static void xe_pt_abort_bind(struct xe_vma *vma,
-			     struct xe_vm_pgtable_update *entries,
-			     u32 num_entries)
+static void xe_pt_cancel_bind(struct xe_vma *vma,
+			      struct xe_vm_pgtable_update *entries,
+			      u32 num_entries)
 {
 	u32 i, j;
 
 	for (i = 0; i < num_entries; i++) {
-		if (!entries[i].pt_entries)
+		struct xe_pt *pt = entries[i].pt;
+
+		if (!pt)
 			continue;
 
-		for (j = 0; j < entries[i].qwords; j++)
-			xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL);
+		if (pt->level) {
+			for (j = 0; j < entries[i].qwords; j++)
+				xe_pt_destroy(entries[i].pt_entries[j].pt,
+					      xe_vma_vm(vma)->flags, NULL);
+		}
+
 		kfree(entries[i].pt_entries);
+		entries[i].pt_entries = NULL;
+		entries[i].qwords = 0;
 	}
 }
 
@@ -873,10 +881,61 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma)
 	xe_vm_assert_held(vm);
 }
 
-static void xe_pt_commit_bind(struct xe_vma *vma,
-			      struct xe_vm_pgtable_update *entries,
-			      u32 num_entries, bool rebind,
-			      struct llist_head *deferred)
+static void xe_pt_commit(struct xe_vma *vma,
+			 struct xe_vm_pgtable_update *entries,
+			 u32 num_entries, struct llist_head *deferred)
+{
+	u32 i, j;
+
+	xe_pt_commit_locks_assert(vma);
+
+	for (i = 0; i < num_entries; i++) {
+		struct xe_pt *pt = entries[i].pt;
+
+		if (!pt->level)
+			continue;
+
+		for (j = 0; j < entries[i].qwords; j++) {
+			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
+
+			xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred);
+		}
+	}
+}
+
+static void xe_pt_abort_bind(struct xe_vma *vma,
+			     struct xe_vm_pgtable_update *entries,
+			     u32 num_entries, bool rebind)
+{
+	int i, j;
+
+	xe_pt_commit_locks_assert(vma);
+
+	for (i = num_entries - 1; i >= 0; --i) {
+		struct xe_pt *pt = entries[i].pt;
+		struct xe_pt_dir *pt_dir;
+
+		if (!rebind)
+			pt->num_live -= entries[i].qwords;
+
+		if (!pt->level)
+			continue;
+
+		pt_dir = as_xe_pt_dir(pt);
+		for (j = 0; j < entries[i].qwords; j++) {
+			u32 j_ = j + entries[i].ofs;
+			struct xe_pt *newpte = xe_pt_entry(pt_dir, j_);
+			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
+
+			pt_dir->children[j_] = oldpte ? &oldpte->base : 0;
+			xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
+		}
+	}
+}
+
+static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
+				      struct xe_vm_pgtable_update *entries,
+				      u32 num_entries, bool rebind)
 {
 	u32 i, j;
 
@@ -896,12 +955,13 @@ static void xe_pt_commit_bind(struct xe_vma *vma,
 		for (j = 0; j < entries[i].qwords; j++) {
 			u32 j_ = j + entries[i].ofs;
 			struct xe_pt *newpte = entries[i].pt_entries[j].pt;
+			struct xe_pt *oldpte = NULL;
 
 			if (xe_pt_entry(pt_dir, j_))
-				xe_pt_destroy(xe_pt_entry(pt_dir, j_),
-					      xe_vma_vm(vma)->flags, deferred);
+				oldpte = xe_pt_entry(pt_dir, j_);
 
 			pt_dir->children[j_] = &newpte->base;
+			entries[i].pt_entries[j].pt = oldpte;
 		}
 	}
 }
@@ -925,8 +985,6 @@ xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
 	err = xe_pt_stage_bind(tile, vma, entries, num_entries);
 	if (!err)
 		xe_tile_assert(tile, *num_entries);
-	else /* abort! */
-		xe_pt_abort_bind(vma, entries, *num_entries);
 
 	return err;
 }
@@ -1449,7 +1507,7 @@ xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
 				     &end_offset))
 		return 0;
 
-	(void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false);
+	(void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
 	xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
 
 	return 0;
@@ -1517,32 +1575,57 @@ xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
 		memset64(ptr, empty, num_qwords);
 }
 
+static void xe_pt_abort_unbind(struct xe_vma *vma,
+			       struct xe_vm_pgtable_update *entries,
+			       u32 num_entries)
+{
+	int j, i;
+
+	xe_pt_commit_locks_assert(vma);
+
+	for (j = num_entries - 1; j >= 0; --j) {
+		struct xe_vm_pgtable_update *entry = &entries[j];
+		struct xe_pt *pt = entry->pt;
+		struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
+
+		pt->num_live += entry->qwords;
+
+		if (!pt->level)
+			continue;
+
+		for (i = entry->ofs; i < entry->ofs + entry->qwords; i++)
+			pt_dir->children[i] =
+				entries[j].pt_entries[i - entry->ofs].pt ?
+				&entries[j].pt_entries[i - entry->ofs].pt->base : 0;
+	}
+}
+
 static void
-xe_pt_commit_unbind(struct xe_vma *vma,
-		    struct xe_vm_pgtable_update *entries, u32 num_entries,
-		    struct llist_head *deferred)
+xe_pt_commit_prepare_unbind(struct xe_vma *vma,
+			    struct xe_vm_pgtable_update *entries,
+			    u32 num_entries)
 {
-	u32 j;
+	int j, i;
 
 	xe_pt_commit_locks_assert(vma);
 
 	for (j = 0; j < num_entries; ++j) {
 		struct xe_vm_pgtable_update *entry = &entries[j];
 		struct xe_pt *pt = entry->pt;
+		struct xe_pt_dir *pt_dir;
 
 		pt->num_live -= entry->qwords;
-		if (pt->level) {
-			struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
-			u32 i;
-
-			for (i = entry->ofs; i < entry->ofs + entry->qwords;
-			     i++) {
-				if (xe_pt_entry(pt_dir, i))
-					xe_pt_destroy(xe_pt_entry(pt_dir, i),
-						      xe_vma_vm(vma)->flags, deferred);
+		if (!pt->level)
+			continue;
 
-				pt_dir->children[i] = NULL;
-			}
+		pt_dir = as_xe_pt_dir(pt);
+		for (i = entry->ofs; i < entry->ofs + entry->qwords; i++) {
+			if (xe_pt_entry(pt_dir, i))
+				entries[j].pt_entries[i - entry->ofs].pt =
+					xe_pt_entry(pt_dir, i);
+			else
+				entries[j].pt_entries[i - entry->ofs].pt = NULL;
+			pt_dir->children[i] = NULL;
 		}
 	}
 }
@@ -1588,7 +1671,6 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 {
 	u32 current_op = pt_update_ops->current_op;
 	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
-	struct llist_head *deferred = &pt_update_ops->deferred;
 	int err;
 
 	xe_bo_assert_held(xe_vma_bo(vma));
@@ -1635,11 +1717,12 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 			(!pt_op->rebind && vm->scratch_pt[tile->id] &&
 			 xe_vm_in_preempt_fence_mode(vm));
 
-		/* FIXME: Don't commit right away */
 		vma->tile_staged |= BIT(tile->id);
 		pt_op->vma = vma;
-		xe_pt_commit_bind(vma, pt_op->entries, pt_op->num_entries,
-				  pt_op->rebind, deferred);
+		xe_pt_commit_prepare_bind(vma, pt_op->entries,
+					  pt_op->num_entries, pt_op->rebind);
+	} else {
+		xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
 	}
 
 	return err;
@@ -1651,7 +1734,6 @@ static int unbind_op_prepare(struct xe_tile *tile,
 {
 	u32 current_op = pt_update_ops->current_op;
 	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
-	struct llist_head *deferred = &pt_update_ops->deferred;
 	int err;
 
 	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
@@ -1687,9 +1769,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 	pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma);
 	pt_update_ops->needs_invalidation = true;
 
-	/* FIXME: Don't commit right away */
-	xe_pt_commit_unbind(vma, pt_op->entries, pt_op->num_entries,
-			    deferred);
+	xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
 
 	return 0;
 }
@@ -1910,7 +1990,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	struct invalidation_fence *ifence = NULL;
 	struct xe_range_fence *rfence;
 	struct xe_vma_op *op;
-	int err = 0;
+	int err = 0, i;
 	struct xe_migrate_pt_update update = {
 		.ops = pt_update_ops->needs_userptr_lock ?
 			&userptr_migrate_ops :
@@ -1930,8 +2010,10 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 
 	if (pt_update_ops->needs_invalidation) {
 		ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
-		if (!ifence)
-			return ERR_PTR(-ENOMEM);
+		if (!ifence) {
+			err = -ENOMEM;
+			goto kill_vm_tile1;
+		}
 	}
 
 	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
@@ -1946,6 +2028,15 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 		goto free_rfence;
 	}
 
+	/* Point of no return - VM killed if failure after this */
+	for (i = 0; i < pt_update_ops->current_op; ++i) {
+		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
+
+		xe_pt_commit(pt_op->vma, pt_op->entries,
+			     pt_op->num_entries, &pt_update_ops->deferred);
+		pt_op->vma = NULL;	/* skip in xe_pt_update_ops_abort */
+	}
+
 	err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
 				    &xe_range_fence_kfree_ops,
 				    pt_update_ops->start,
@@ -1981,10 +2072,15 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	if (pt_update_ops->needs_userptr_lock)
 		up_read(&vm->userptr.notifier_lock);
 	dma_fence_put(fence);
+	if (!tile->id)
+		xe_vm_kill(vops->vm, false);
 free_rfence:
 	kfree(rfence);
 free_ifence:
 	kfree(ifence);
+kill_vm_tile1:
+	if (err != -EAGAIN && tile->id)
+		xe_vm_kill(vops->vm, false);
 
 	return ERR_PTR(err);
 }
@@ -2005,12 +2101,10 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
-	/* FIXME: Not 100% correct */
-	for (i = 0; i < pt_update_ops->num_ops; ++i) {
+	for (i = 0; i < pt_update_ops->current_op; ++i) {
 		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
 
-		if (pt_op->bind)
-			xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
 	}
 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
 }
@@ -2024,10 +2118,28 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
  */
 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
 {
+	struct xe_vm_pgtable_update_ops *pt_update_ops =
+		&vops->pt_update_ops[tile->id];
+	int i;
+
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
-	/* FIXME: Just kill VM for now + cleanup PTs */
+	for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
+		struct xe_vm_pgtable_update_op *pt_op =
+			&pt_update_ops->ops[i];
+
+		if (!pt_op->vma || i >= pt_update_ops->current_op)
+			continue;
+
+		if (pt_op->bind)
+			xe_pt_abort_bind(pt_op->vma, pt_op->entries,
+					 pt_op->num_entries,
+					 pt_op->rebind);
+		else
+			xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
+					   pt_op->num_entries);
+	}
+
 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
-	xe_vm_kill(vops->vm, false);
- }
+}
Signature · 1 line
2.26.3
author
#19 lore raw

PATCH 20/42 drm/xe: Retry BO allocation

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

TTM doesn't support fair eviction via WW locking, this mitigated in by using retry loops in exec and preempt rebind worker. Extend this retry loop to BO allocation. Once TTM supports fair eviction this patch can be reverted.

---
 drivers/gpu/drm/xe/xe_bo.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/xe/xe_bo.c +7 -1
index 74294f1b05bc..8fe78f7b000f 100644
@@ -1929,6 +1929,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 	struct xe_file *xef = to_xe_file(file);
 	struct drm_xe_gem_create *args = data;
 	struct xe_vm *vm = NULL;
+	ktime_t end = 0;
 	struct xe_bo *bo;
 	unsigned int bo_flags;
 	u32 handle;
@@ -1994,11 +1995,14 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 		vm = xe_vm_lookup(xef, args->vm_id);
 		if (XE_IOCTL_DBG(xe, !vm))
 			return -ENOENT;
+	}
+
+retry:
+	if (vm) {
 		err = xe_vm_lock(vm, true);
 		if (err)
 			goto out_vm;
 	}
-
 	bo = xe_bo_create_user(xe, NULL, vm, args->size, args->cpu_caching,
 			       ttm_bo_type_device, bo_flags);
 
@@ -2007,6 +2011,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data,
 
 	if (IS_ERR(bo)) {
 		err = PTR_ERR(bo);
+		if (xe_vm_validate_should_retry(NULL, err, &end))
+			goto retry;
 		goto out_vm;
 	}
 
Signature · 1 line
2.26.3
author
#20 lore raw

PATCH 21/42 drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

Add the DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag, which is used to create unpopulated virtual memory areas (VMAs) without memory backing or GPU page tables. These VMAs are referred to as system allocator VMAs. The idea is that upon a page fault, the memory backing and GPU page tables will be populated.

System allocator VMAs only update GPUVM state; they do not have an internal page table (PT) state, nor do they have GPU mappings.

It is expected that system allocator VMAs will be mixed with buffer object (BO) VMAs within a single VM. In other words, system allocations and runtime allocations can be mixed within a single user-mode driver (UMD) program.

Expected usage:

- Bind the entire virtual address (VA) space upon program load using the DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag and DRM_XE_VM_BIND_OP_MAP op.
- If a buffer object (BO) requires GPU mapping, reserve an address range using mmap, unbind the address range from system allocator using DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag and DRM_XE_VM_BIND_OP_UNMAP op, and bind the BO to the reserved address using existing bind IOCTLs (runtime allocation).
- If a BO no longer requires GPU mapping, bind the mapping address with the DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag.
- Any malloc'd address accessed by the GPU will be faulted in via the SVM implementation (system allocation).
- Upon freeing any malloc'd data, the SVM implementation will remove GPU mappings.

v1:
fix commit message (Oak)
rebase with lastest xekmd, fix conflict in xe_analyze_vm (Oak) allow UNMAP operation for system allocator (Krishna)

FIXME: Not enforcing DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR for fault mode VMs

FIXME: Only supporting 1 to 1 mapping between user address space and GPU address space

---
 drivers/gpu/drm/xe/xe_pt.c       |  73 +++++++++++++++++----
 drivers/gpu/drm/xe/xe_vm.c       | 107 ++++++++++++++++++++-----------
 drivers/gpu/drm/xe/xe_vm.h       |   8 ++-
 drivers/gpu/drm/xe/xe_vm_types.h |   3 +
 include/uapi/drm/xe_drm.h        |  11 +++-
 5 files changed, 148 insertions(+), 54 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +59 -14
index 62aec1a42249..5c2043c90556 100644
@@ -1064,6 +1064,11 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
 {
 	int err = 0;
 
+	/*
+	 * No need to check for is_system_allocator here as vma_add_deps is a
+	 * NOP if VMA is_system_allocator
+	 */
+
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
 		if (!op->map.immediate && xe_vm_in_fault_mode(vm))
@@ -1673,6 +1678,7 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
 	int err;
 
+	xe_tile_assert(tile, !xe_vma_is_system_allocator(vma));
 	xe_bo_assert_held(xe_vma_bo(vma));
 
 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
@@ -1739,6 +1745,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 	if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
 		return 0;
 
+	xe_tile_assert(tile, !xe_vma_is_system_allocator(vma));
 	xe_bo_assert_held(xe_vma_bo(vma));
 
 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
@@ -1785,15 +1792,21 @@ static int op_prepare(struct xe_vm *vm,
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
-		if (!op->map.immediate && xe_vm_in_fault_mode(vm))
+		if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) ||
+		    op->map.is_system_allocator)
 			break;
 
 		err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma);
 		pt_update_ops->wait_vm_kernel = true;
 		break;
 	case DRM_GPUVA_OP_REMAP:
-		err = unbind_op_prepare(tile, pt_update_ops,
-					gpuva_to_vma(op->base.remap.unmap->va));
+	{
+		struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
+
+		if (xe_vma_is_system_allocator(old))
+			break;
+
+		err = unbind_op_prepare(tile, pt_update_ops, old);
 
 		if (!err && op->remap.prev) {
 			err = bind_op_prepare(vm, tile, pt_update_ops,
@@ -1806,15 +1819,28 @@ static int op_prepare(struct xe_vm *vm,
 			pt_update_ops->wait_vm_bookkeep = true;
 		}
 		break;
+	}
 	case DRM_GPUVA_OP_UNMAP:
-		err = unbind_op_prepare(tile, pt_update_ops,
-					gpuva_to_vma(op->base.unmap.va));
+	{
+		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
+
+		if (xe_vma_is_system_allocator(vma))
+			break;
+
+		err = unbind_op_prepare(tile, pt_update_ops, vma);
 		break;
+	}
 	case DRM_GPUVA_OP_PREFETCH:
-		err = bind_op_prepare(vm, tile, pt_update_ops,
-				      gpuva_to_vma(op->base.prefetch.va));
+	{
+		struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
+
+		if (xe_vma_is_system_allocator(vma))
+			break;
+
+		err = bind_op_prepare(vm, tile, pt_update_ops, vma);
 		pt_update_ops->wait_vm_kernel = true;
 		break;
+	}
 	default:
 		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
 	}
@@ -1875,6 +1901,8 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
 			   struct xe_vm_pgtable_update_ops *pt_update_ops,
 			   struct xe_vma *vma, struct dma_fence *fence)
 {
+	xe_tile_assert(tile, !xe_vma_is_system_allocator(vma));
+
 	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
 		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
 				   pt_update_ops->wait_vm_bookkeep ?
@@ -1901,6 +1929,8 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile,
 			     struct xe_vm_pgtable_update_ops *pt_update_ops,
 			     struct xe_vma *vma, struct dma_fence *fence)
 {
+	xe_tile_assert(tile, !xe_vma_is_system_allocator(vma));
+
 	if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
 		dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence,
 				   pt_update_ops->wait_vm_bookkeep ?
@@ -1928,14 +1958,20 @@ static void op_commit(struct xe_vm *vm,
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
-		if (!op->map.immediate && xe_vm_in_fault_mode(vm))
+		if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) ||
+		    op->map.is_system_allocator)
 			break;
 
 		bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence);
 		break;
 	case DRM_GPUVA_OP_REMAP:
-		unbind_op_commit(vm, tile, pt_update_ops,
-				 gpuva_to_vma(op->base.remap.unmap->va), fence);
+	{
+		struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
+
+		if (xe_vma_is_system_allocator(old))
+			break;
+
+		unbind_op_commit(vm, tile, pt_update_ops, old, fence);
 
 		if (op->remap.prev)
 			bind_op_commit(vm, tile, pt_update_ops, op->remap.prev,
@@ -1944,14 +1980,23 @@ static void op_commit(struct xe_vm *vm,
 			bind_op_commit(vm, tile, pt_update_ops, op->remap.next,
 				       fence);
 		break;
+	}
 	case DRM_GPUVA_OP_UNMAP:
-		unbind_op_commit(vm, tile, pt_update_ops,
-				 gpuva_to_vma(op->base.unmap.va), fence);
+	{
+		struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
+
+		if (!xe_vma_is_system_allocator(vma))
+			unbind_op_commit(vm, tile, pt_update_ops, vma, fence);
 		break;
+	}
 	case DRM_GPUVA_OP_PREFETCH:
-		bind_op_commit(vm, tile, pt_update_ops,
-			       gpuva_to_vma(op->base.prefetch.va), fence);
+	{
+		struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
+
+		if (!xe_vma_is_system_allocator(vma))
+			bind_op_commit(vm, tile, pt_update_ops, vma, fence);
 		break;
+	}
 	default:
 		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
 	}
drivers/gpu/drm/xe/xe_vm.c +69 -38

241 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +7 -1
index c864dba35e1d..1a5aed678214 100644
@@ -151,6 +151,11 @@ static inline bool xe_vma_is_null(struct xe_vma *vma)
 	return vma->gpuva.flags & DRM_GPUVA_SPARSE;
 }
 
+static inline bool xe_vma_is_system_allocator(struct xe_vma *vma)
+{
+	return vma->gpuva.flags & XE_VMA_SYSTEM_ALLOCATOR;
+}
+
 static inline bool xe_vma_has_no_bo(struct xe_vma *vma)
 {
 	return !xe_vma_bo(vma);
@@ -158,7 +163,8 @@ static inline bool xe_vma_has_no_bo(struct xe_vma *vma)
 
 static inline bool xe_vma_is_userptr(struct xe_vma *vma)
 {
-	return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma);
+	return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma) &&
+		!xe_vma_is_system_allocator(vma);
 }
 
 /**
drivers/gpu/drm/xe/xe_vm_types.h +3 -0
index 27d651093d30..e1d3dd699380 100644
@@ -32,6 +32,7 @@ struct xe_vm_pgtable_update_op;
 #define XE_VMA_PTE_64K		(DRM_GPUVA_USERBITS << 6)
 #define XE_VMA_PTE_COMPACT	(DRM_GPUVA_USERBITS << 7)
 #define XE_VMA_DUMPABLE		(DRM_GPUVA_USERBITS << 8)
+#define XE_VMA_SYSTEM_ALLOCATOR	(DRM_GPUVA_USERBITS << 9)
 
 /** struct xe_userptr - User pointer */
 struct xe_userptr {
@@ -284,6 +285,8 @@ struct xe_vma_op_map {
 	bool read_only;
 	/** @is_null: is NULL binding */
 	bool is_null;
+	/** @is_system_allocator: is system allocator binding */
+	bool is_system_allocator;
 	/** @dumpable: whether BO is dumped on GPU hang */
 	bool dumpable;
 	/** @pat_index: The pat index to use for this operation. */
include/uapi/drm/xe_drm.h +10 -1
index d7b0903c22b2..af8ee6f929d4 100644
@@ -885,6 +885,12 @@ struct drm_xe_vm_destroy {
  *    will only be valid for DRM_XE_VM_BIND_OP_MAP operations, the BO
  *    handle MBZ, and the BO offset MBZ. This flag is intended to
  *    implement VK sparse bindings.
+ *  - %DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR - When the system allocator flag is
+ *    set, no mappings are created rather the range is reserved for system
+ *    allocations which will be populated on GPU page faults. Only valid on VMs
+ *    with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The system allocator flag are
+ *    only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO handle MBZ, and
+ *    the BO offset MBZ.
  */
 struct drm_xe_vm_bind_op {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -937,7 +943,9 @@ struct drm_xe_vm_bind_op {
 	 * on the @pat_index. For such mappings there is no actual memory being
 	 * mapped (the address in the PTE is invalid), so the various PAT memory
 	 * attributes likely do not apply.  Simply leaving as zero is one
-	 * option (still a valid pat_index).
+	 * option (still a valid pat_index). Same applies to
+	 * DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR bindings as for such mapping
+	 * there is no actual memory being mapped.
 	 */
 	__u16 pat_index;
 
@@ -975,6 +983,7 @@ struct drm_xe_vm_bind_op {
 #define DRM_XE_VM_BIND_FLAG_IMMEDIATE	(1 << 1)
 #define DRM_XE_VM_BIND_FLAG_NULL	(1 << 2)
 #define DRM_XE_VM_BIND_FLAG_DUMPABLE	(1 << 3)
+#define DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR	(1 << 4)
 	/** @flags: Bind flags */
 	__u32 flags;
 
Signature · 1 line
2.26.3
author
#21 lore raw

PATCH 22/42 drm/xe: Add a helper to calculate userptr end address

In reply to Oak Zeng

xe_vma_userptr_end is added to calculate a userptr end.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_vm.h | 5 +++++
 1 file changed, 5 insertions(+)
drivers/gpu/drm/xe/xe_vm.h +5 -0
index 1a5aed678214..89f3306561ad 100644
@@ -146,6 +146,11 @@ static inline u64 xe_vma_userptr(struct xe_vma *vma)
 	return vma->gpuva.gem.offset;
 }
 
+static inline u64 xe_vma_userptr_end(struct xe_vma *vma)
+{
+	return vma->gpuva.gem.offset + xe_vma_size(vma);
+}
+
 static inline bool xe_vma_is_null(struct xe_vma *vma)
 {
 	return vma->gpuva.flags & DRM_GPUVA_SPARSE;
Signature · 1 line
2.26.3
author
#22 lore raw

PATCH 23/42 drm/xe: Add dma_addr res cursor

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

---
 drivers/gpu/drm/xe/xe_res_cursor.h | 50 +++++++++++++++++++++++++++++-
 1 file changed, 49 insertions(+), 1 deletion(-)
drivers/gpu/drm/xe/xe_res_cursor.h +49 -1
index 655af89b31a9..b17b3375f6d9 100644
@@ -44,7 +44,9 @@ struct xe_res_cursor {
 	u64 remaining;
 	void *node;
 	u32 mem_type;
+	unsigned int order;
 	struct scatterlist *sgl;
+	const dma_addr_t *dma_addr;
 	struct drm_buddy *mm;
 };
 
@@ -71,6 +73,7 @@ static inline void xe_res_first(struct ttm_resource *res,
 				struct xe_res_cursor *cur)
 {
 	cur->sgl = NULL;
+	cur->dma_addr = NULL;
 	if (!res)
 		goto fallback;
 
@@ -161,11 +164,43 @@ static inline void xe_res_first_sg(const struct sg_table *sg,
 	cur->start = start;
 	cur->remaining = size;
 	cur->size = 0;
+	cur->dma_addr = NULL;
 	cur->sgl = sg->sgl;
 	cur->mem_type = XE_PL_TT;
 	__xe_res_sg_next(cur);
 }
 
+/**
+ * xe_res_first_dma - initialize a xe_res_cursor with dma_addr array
+ *
+ * @dma_addr: dma_addr array to walk
+ * @start: Start of the range
+ * @size: Size of the range
+ * @order: Order of dma mapping. i.e. PAGE_SIZE << order is mapping size
+ * @cur: cursor object to initialize
+ *
+ * Start walking over the range of allocations between @start and @size.
+ */
+static inline void xe_res_first_dma(const dma_addr_t *dma_addr,
+				    u64 start, u64 size,
+				    unsigned int order,
+				    struct xe_res_cursor *cur)
+{
+	XE_WARN_ON(start);
+	XE_WARN_ON(!dma_addr);
+	XE_WARN_ON(!IS_ALIGNED(start, PAGE_SIZE) ||
+		   !IS_ALIGNED(size, PAGE_SIZE));
+
+	cur->node = NULL;
+	cur->start = start;
+	cur->remaining = size;
+	cur->size = PAGE_SIZE << order;
+	cur->dma_addr = dma_addr;
+	cur->order = order;
+	cur->sgl = NULL;
+	cur->mem_type = XE_PL_TT;
+}
+
 /**
  * xe_res_next - advance the cursor
  *
@@ -192,6 +227,13 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
 		return;
 	}
 
+	if (cur->dma_addr) {
+		cur->size = (PAGE_SIZE << cur->order) -
+			(size - cur->size);
+		cur->start += size;
+		return;
+	}
+
 	if (cur->sgl) {
 		cur->start += size;
 		__xe_res_sg_next(cur);
@@ -233,6 +275,12 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
  */
 static inline u64 xe_res_dma(const struct xe_res_cursor *cur)
 {
-	return cur->sgl ? sg_dma_address(cur->sgl) + cur->start : cur->start;
+	if (cur->dma_addr)
+		return cur->dma_addr[cur->start >> (PAGE_SHIFT + cur->order)] +
+			(cur->start & ((PAGE_SIZE << cur->order) - 1));
+	else if (cur->sgl)
+		return sg_dma_address(cur->sgl) + cur->start;
+	else
+		return cur->start;
 }
 #endif
Signature · 1 line
2.26.3
author
#23 lore raw

PATCH 24/42 drm/xe: Use drm_mem_region for xe

In reply to Oak Zeng

drm_mem_region was introduced to move some memory management codes to drm layer so it can be shared b/t different vendor drivers. This patch apply drm_mem_region concept to xekmd driver.

drm_mem_region is the parent class of xe_mem_region. Some xe_mem_region member such as dpa_base is deleted as it is already in the parent class.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/display/xe_fb_pin.c        |  2 +-
 drivers/gpu/drm/xe/display/xe_plane_initial.c |  2 +-
 drivers/gpu/drm/xe/xe_bo.c                    |  6 +++---
 drivers/gpu/drm/xe/xe_device_types.h          | 11 ++---------
 drivers/gpu/drm/xe/xe_migrate.c               |  8 ++++----
 drivers/gpu/drm/xe/xe_query.c                 |  2 +-
 drivers/gpu/drm/xe/xe_tile.c                  |  2 +-
 drivers/gpu/drm/xe/xe_ttm_vram_mgr.c          |  2 +-
 drivers/gpu/drm/xe/xe_vram.c                  | 12 ++++++------
 9 files changed, 20 insertions(+), 27 deletions(-)
drivers/gpu/drm/xe/display/xe_fb_pin.c +1 -1
index a2f417209124..5c4590c62c08 100644
@@ -272,7 +272,7 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
 		 * accessible.  This is important on small-bar systems where
 		 * only some subset of VRAM is CPU accessible.
 		 */
-		if (tile->mem.vram.io_size < tile->mem.vram.usable_size) {
+		if (tile->mem.vram.io_size < tile->mem.vram.drm_mr.usable_size) {
 			ret = -EINVAL;
 			goto err;
 		}
drivers/gpu/drm/xe/display/xe_plane_initial.c +1 -1
index e135b20962d9..c2c079a2b133 100644
@@ -86,7 +86,7 @@ initial_plane_bo(struct xe_device *xe,
 		 * We don't currently expect this to ever be placed in the
 		 * stolen portion.
 		 */
-		if (phys_base >= tile0->mem.vram.usable_size) {
+		if (phys_base >= tile0->mem.vram.drm_mr.usable_size) {
 			drm_err(&xe->drm,
 				"Initial plane programming using invalid range, phys_base=%pa\n",
 				&phys_base);
drivers/gpu/drm/xe/xe_bo.c +3 -3
index 8fe78f7b000f..371ea9a5dd16 100644
@@ -172,7 +172,7 @@ static void add_vram(struct xe_device *xe, struct xe_bo *bo,
 	xe_assert(xe, *c < ARRAY_SIZE(bo->placements));
 
 	vram = to_xe_ttm_vram_mgr(ttm_manager_type(&xe->ttm, mem_type))->vram;
-	xe_assert(xe, vram && vram->usable_size);
+	xe_assert(xe, vram && vram->drm_mr.usable_size);
 	io_size = vram->io_size;
 
 	/*
@@ -183,7 +183,7 @@ static void add_vram(struct xe_device *xe, struct xe_bo *bo,
 			XE_BO_FLAG_GGTT))
 		place.flags |= TTM_PL_FLAG_CONTIGUOUS;
 
-	if (io_size < vram->usable_size) {
+	if (io_size < vram->drm_mr.usable_size) {
 		if (bo_flags & XE_BO_FLAG_NEEDS_CPU_ACCESS) {
 			place.fpfn = 0;
 			place.lpfn = io_size >> PAGE_SHIFT;
@@ -1637,7 +1637,7 @@ uint64_t vram_region_gpu_offset(struct ttm_resource *res)
 	if (res->mem_type == XE_PL_STOLEN)
 		return xe_ttm_stolen_gpu_offset(xe);
 
-	return res_to_mem_region(res)->dpa_base;
+	return res_to_mem_region(res)->drm_mr.dpa_base;
 }
 
 /**
drivers/gpu/drm/xe/xe_device_types.h +2 -9
index f1c09824b145..cf61b52e6d84 100644
@@ -11,6 +11,7 @@
 #include <drm/drm_device.h>
 #include <drm/drm_file.h>
 #include <drm/ttm/ttm_device.h>
+#include <drm/drm_svm.h>
 
 #include "xe_devcoredump_types.h"
 #include "xe_heci_gsc.h"
@@ -69,6 +70,7 @@ struct xe_pat_ops;
  * device, such as HBM memory or CXL extension memory.
  */
 struct xe_mem_region {
+	struct drm_mem_region drm_mr;
 	/** @io_start: IO start address of this VRAM instance */
 	resource_size_t io_start;
 	/**
@@ -81,15 +83,6 @@ struct xe_mem_region {
 	 * configuration is known as small-bar.
 	 */
 	resource_size_t io_size;
-	/** @dpa_base: This memory regions's DPA (device physical address) base */
-	resource_size_t dpa_base;
-	/**
-	 * @usable_size: usable size of VRAM
-	 *
-	 * Usable size of VRAM excluding reserved portions
-	 * (e.g stolen mem)
-	 */
-	resource_size_t usable_size;
 	/**
 	 * @actual_physical_size: Actual VRAM size
 	 *
drivers/gpu/drm/xe/xe_migrate.c +4 -4
index e65aa57c8dc8..15c8973c0495 100644
@@ -126,7 +126,7 @@ static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr)
 	 * Remove the DPA to get a correct offset into identity table for the
 	 * migrate offset
 	 */
-	addr -= xe->mem.vram.dpa_base;
+	addr -= xe->mem.vram.drm_mr.dpa_base;
 	return addr + (256ULL << xe_pt_shift(2));
 }
 
@@ -256,21 +256,21 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 		u64 pos, ofs, flags;
 		/* XXX: Unclear if this should be usable_size? */
 		u64 vram_limit =  xe->mem.vram.actual_physical_size +
-			xe->mem.vram.dpa_base;
+			xe->mem.vram.drm_mr.dpa_base;
 
 		level = 2;
 		ofs = map_ofs + XE_PAGE_SIZE * level + 256 * 8;
 		flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level,
 						    true, 0);
 
-		xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M));
+		xe_assert(xe, IS_ALIGNED(xe->mem.vram.drm_mr.usable_size, SZ_2M));
 
 		/*
 		 * Use 1GB pages when possible, last chunk always use 2M
 		 * pages as mixing reserved memory (stolen, WOCPM) with a single
 		 * mapping is not allowed on certain platforms.
 		 */
-		for (pos = xe->mem.vram.dpa_base; pos < vram_limit;
+		for (pos = xe->mem.vram.drm_mr.dpa_base; pos < vram_limit;
 		     pos += SZ_1G, ofs += 8) {
 			if (pos + SZ_1G >= vram_limit) {
 				u64 pt31_ofs = bo->size - XE_PAGE_SIZE;
drivers/gpu/drm/xe/xe_query.c +1 -1
index 995effcb904b..8b3d63420cef 100644
@@ -334,7 +334,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query)
 	config->num_params = num_params;
 	config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] =
 		xe->info.devid | (xe->info.revid << 16);
-	if (xe_device_get_root_tile(xe)->mem.vram.usable_size)
+	if (xe_device_get_root_tile(xe)->mem.vram.drm_mr.usable_size)
 		config->info[DRM_XE_QUERY_CONFIG_FLAGS] =
 			DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM;
 	config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
drivers/gpu/drm/xe/xe_tile.c +1 -1
index 15ea0a942f67..109f3118e821 100644
@@ -132,7 +132,7 @@ static int tile_ttm_mgr_init(struct xe_tile *tile)
 	struct xe_device *xe = tile_to_xe(tile);
 	int err;
 
-	if (tile->mem.vram.usable_size) {
+	if (tile->mem.vram.drm_mr.usable_size) {
 		err = xe_ttm_vram_mgr_init(tile, tile->mem.vram_mgr);
 		if (err)
 			return err;
drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +1 -1
index fe3779fdba2c..dd31b24fb07d 100644
@@ -364,7 +364,7 @@ int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr)
 
 	mgr->vram = vram;
 	return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + tile->id,
-				      vram->usable_size, vram->io_size,
+				      vram->drm_mr.usable_size, vram->io_size,
 				      PAGE_SIZE);
 }
 
drivers/gpu/drm/xe/xe_vram.c +6 -6
index 5bcd59190353..fff18517a9f9 100644
@@ -150,7 +150,7 @@ static int determine_lmem_bar_size(struct xe_device *xe)
 		return -EIO;
 
 	/* XXX: Need to change when xe link code is ready */
-	xe->mem.vram.dpa_base = 0;
+	xe->mem.vram.drm_mr.dpa_base = 0;
 
 	/* set up a map to the total memory area. */
 	xe->mem.vram.mapping = ioremap_wc(xe->mem.vram.io_start, xe->mem.vram.io_size);
@@ -333,16 +333,16 @@ int xe_vram_probe(struct xe_device *xe)
 			return -ENODEV;
 		}
 
-		tile->mem.vram.dpa_base = xe->mem.vram.dpa_base + tile_offset;
-		tile->mem.vram.usable_size = vram_size;
+		tile->mem.vram.drm_mr.dpa_base = xe->mem.vram.drm_mr.dpa_base + tile_offset;
+		tile->mem.vram.drm_mr.usable_size = vram_size;
 		tile->mem.vram.mapping = xe->mem.vram.mapping + tile_offset;
 
-		if (tile->mem.vram.io_size < tile->mem.vram.usable_size)
+		if (tile->mem.vram.io_size < tile->mem.vram.drm_mr.usable_size)
 			drm_info(&xe->drm, "Small BAR device\n");
 		drm_info(&xe->drm, "VRAM[%u, %u]: Actual physical size %pa, usable size exclude stolen %pa, CPU accessible size %pa\n", id,
-			 tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.usable_size, &tile->mem.vram.io_size);
+			 tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.drm_mr.usable_size, &tile->mem.vram.io_size);
 		drm_info(&xe->drm, "VRAM[%u, %u]: DPA range: [%pa-%llx], io range: [%pa-%llx]\n", id, tile->id,
-			 &tile->mem.vram.dpa_base, tile->mem.vram.dpa_base + (u64)tile->mem.vram.actual_physical_size,
+			 &tile->mem.vram.drm_mr.dpa_base, tile->mem.vram.drm_mr.dpa_base + (u64)tile->mem.vram.actual_physical_size,
 			 &tile->mem.vram.io_start, tile->mem.vram.io_start + (u64)tile->mem.vram.io_size);
 
 		/* calculate total size using tile size to get the correct HW sizing */
Signature · 1 line
2.26.3
author
#24 lore raw

PATCH 25/42 drm/xe: use drm_hmmptr in xe

In reply to Oak Zeng

drm_hmmptr was created to move some userptr and svm logic such as mmu notifier registration, range population etc to drm layer, so those logic can be shared b/t vendor drivers.

This patch apply the drm_hmmptr to xekmd driver. drm_hmmptr is the parent class of xe_userptr.

Most of the changes are straight forward, such as some xe_userptr members are now moved to drm_hmmptr. Since drm_hmmptr doesn't use scatter-gather table for dma address anymore, we changed to the dma-address cursor from the sg cursor during page table building.

xe_hmm.c and xe_hmm.h are removed, as the logic is now moved to drm_svm.c

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/Makefile      |   2 -
 drivers/gpu/drm/xe/xe_hmm.c      | 253 -------------------------------
 drivers/gpu/drm/xe/xe_hmm.h      |  11 --
 drivers/gpu/drm/xe/xe_pt.c       |  10 +-
 drivers/gpu/drm/xe/xe_vm.c       |  89 ++++++++---
 drivers/gpu/drm/xe/xe_vm_types.h |  11 +-
 6 files changed, 71 insertions(+), 305 deletions(-)
 delete mode 100644 drivers/gpu/drm/xe/xe_hmm.c
 delete mode 100644 drivers/gpu/drm/xe/xe_hmm.h
drivers/gpu/drm/xe/Makefile +0 -2
index 2e7c94510ace..6457f1b74cd8 100644
@@ -128,8 +128,6 @@ xe-y += xe_bb.o \
 	xe_wa.o \
 	xe_wopcm.o
 
-xe-$(CONFIG_HMM_MIRROR) += xe_hmm.o
-
 # graphics hardware monitoring (HWMON) support
 xe-$(CONFIG_HWMON) += xe_hwmon.o
 
drivers/gpu/drm/xe/xe_hmm.c deleted +0 -253

253 lines not on this page · raw message

drivers/gpu/drm/xe/xe_hmm.h deleted +0 -11
deleted file mode 100644
index 909dc2bdcd97..000000000000
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright © 2024 Intel Corporation
- */
-
-#include <linux/types.h>
-
-struct xe_userptr_vma;
-
-int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma, bool is_mm_mmap_locked);
-void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma);
drivers/gpu/drm/xe/xe_pt.c +5 -5
index 5c2043c90556..9c1a66e373d2 100644
@@ -669,8 +669,8 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
 
 	if (!xe_vma_is_null(vma)) {
 		if (xe_vma_is_userptr(vma))
-			xe_res_first_sg(to_userptr_vma(vma)->userptr.sg, 0,
-					xe_vma_size(vma), &curs);
+			xe_res_first_dma(to_userptr_vma(vma)->userptr.hmmptr.dma_addr,
+					0, xe_vma_size(vma), 0, &curs);
 		else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
 			xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
 				     xe_vma_size(vma), &curs);
@@ -1221,12 +1221,12 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
 		return 0;
 
 	uvma = to_userptr_vma(vma);
-	notifier_seq = uvma->userptr.notifier_seq;
+	notifier_seq = uvma->userptr.hmmptr.notifier_seq;
 
 	if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))
                return 0;
 
-	if (!mmu_interval_read_retry(&uvma->userptr.notifier,
+	if (!mmu_interval_read_retry(&uvma->userptr.hmmptr.notifier,
 				     notifier_seq) &&
 	    !xe_pt_userptr_inject_eagain(uvma))
 		return 0;
@@ -1757,7 +1757,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 	 * state if an invalidation is running while preparing an unbind.
 	 */
 	if (xe_vma_is_userptr(vma) && xe_vm_in_fault_mode(xe_vma_vm(vma)))
-		mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.notifier);
+		mmu_interval_read_begin(&to_userptr_vma(vma)->userptr.hmmptr.notifier);
 
 	pt_op->vma = vma;
 	pt_op->bind = false;
drivers/gpu/drm/xe/xe_vm.c +65 -24

165 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm_types.h +1 -10
index e1d3dd699380..976982972a06 100644
@@ -36,20 +36,11 @@ struct xe_vm_pgtable_update_op;
 
 /** struct xe_userptr - User pointer */
 struct xe_userptr {
+	struct drm_hmmptr hmmptr;
 	/** @invalidate_link: Link for the vm::userptr.invalidated list */
 	struct list_head invalidate_link;
 	/** @userptr: link into VM repin list if userptr. */
 	struct list_head repin_link;
-	/**
-	 * @notifier: MMU notifier for user pointer (invalidation call back)
-	 */
-	struct mmu_interval_notifier notifier;
-	/** @sgt: storage for a scatter gather table */
-	struct sg_table sgt;
-	/** @sg: allocated scatter gather table */
-	struct sg_table *sg;
-	/** @notifier_seq: notifier sequence number */
-	unsigned long notifier_seq;
 	/**
 	 * @initial_bind: user pointer has been bound at least once.
 	 * write: vm->userptr.notifier_lock in read mode and vm->resv held.
Signature · 1 line
2.26.3
author
#25 lore raw

PATCH 26/42 drm/xe: Moving to range based vma invalidation

In reply to Oak Zeng

New parameters are introduced to the vma invalidation function to allow partially invalidate a vma.

For userptr invalidation, we now only invalidate the mmu notified range instead of the whole vma. This more more proper than the whole vma based invalidation.

For other cases, still keep the whole vma invalidation scheme.

One of the consequence of this change is, we now don't have information whether a vma is fully invalidated or not, because vma can be partially invalidated now. The tile_invalidated member is deleted due to this reason.

This is prepare work for system allocator where we want to apply range based vma invalidation. It is also reasonable to apply the same scheme for userptr invalidation.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_bo.c           |  2 +-
 drivers/gpu/drm/xe/xe_gt_pagefault.c | 11 ---------
 drivers/gpu/drm/xe/xe_pt.c           | 16 +++++++++----
 drivers/gpu/drm/xe/xe_pt.h           |  2 +-
 drivers/gpu/drm/xe/xe_vm.c           | 36 +++++++++++++++++++---------
 drivers/gpu/drm/xe/xe_vm.h           |  2 +-
 drivers/gpu/drm/xe/xe_vm_types.h     |  3 ---
 7 files changed, 40 insertions(+), 32 deletions(-)
drivers/gpu/drm/xe/xe_bo.c +1 -1
index 371ea9a5dd16..71670e7397fc 100644
@@ -514,7 +514,7 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
 			struct xe_vma *vma = gpuva_to_vma(gpuva);
 
 			trace_xe_vma_evict(vma);
-			ret = xe_vm_invalidate_vma(vma);
+			ret = xe_vm_invalidate_vma(vma, xe_vma_start(vma), xe_vma_end(vma));
 			if (XE_WARN_ON(ret))
 				return ret;
 		}
drivers/gpu/drm/xe/xe_gt_pagefault.c +0 -11
index 9292d5468868..9e84cff964b8 100644
@@ -65,12 +65,6 @@ static bool access_is_atomic(enum access_type access_type)
 	return access_type == ACCESS_TYPE_ATOMIC;
 }
 
-static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
-{
-	return BIT(tile->id) & vma->tile_present &&
-		!(BIT(tile->id) & vma->tile_invalidated);
-}
-
 static bool vma_matches(struct xe_vma *vma, u64 page_addr)
 {
 	if (page_addr > xe_vma_end(vma) - 1 ||
@@ -138,10 +132,6 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 	trace_xe_vma_pagefault(vma);
 	atomic = access_is_atomic(pf->access_type);
 
-	/* Check if VMA is valid */
-	if (vma_is_valid(tile, vma) && !atomic)
-		return 0;
-
 retry_userptr:
 	if (xe_vma_is_userptr(vma) &&
 	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
@@ -175,7 +165,6 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 
 	dma_fence_wait(fence, false);
 	dma_fence_put(fence);
-	vma->tile_invalidated &= ~BIT(tile->id);
 
 unlock_dma_resv:
 	drm_exec_fini(&exec);
drivers/gpu/drm/xe/xe_pt.c +12 -4
index 9c1a66e373d2..a379705fe9b4 100644
@@ -792,6 +792,8 @@ static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
  * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range
  * @tile: The tile we're zapping for.
  * @vma: GPU VMA detailing address range.
+ * @start: start of the range.
+ * @end: end of the range.
  *
  * Eviction and Userptr invalidation needs to be able to zap the
  * gpu ptes of a given address range in pagefaulting mode.
@@ -803,8 +805,11 @@ static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
  *
  * Return: Whether ptes were actually updated and a TLB invalidation is
  * required.
+ *
+ * FIXME: double confirm xe_pt_walk_shared support walking of a sub-range of
+ * vma (vs whole vma)
  */
-bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
+bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma, u64 start, u64 end)
 {
 	struct xe_pt_zap_ptes_walk xe_walk = {
 		.base = {
@@ -815,13 +820,16 @@ bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
 		.tile = tile,
 	};
 	struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
-	u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated);
+	u8 pt_mask = vma->tile_present;
+
+	xe_assert(tile_to_xe(tile), start >= xe_vma_start(vma));
+	xe_assert(tile_to_xe(tile), end <= xe_vma_end(vma));
 
 	if (!(pt_mask & BIT(tile->id)))
 		return false;
 
-	(void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
-				xe_vma_end(vma), &xe_walk.base);
+	(void)xe_pt_walk_shared(&pt->base, pt->level, start,
+				end, &xe_walk.base);
 
 	return xe_walk.needs_invalidate;
 }
drivers/gpu/drm/xe/xe_pt.h +1 -1
index 9ab386431cad..aa8ff28e75b0 100644
@@ -41,6 +41,6 @@ struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
 
-bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);
+bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma, u64 start, u64 end);
 
 #endif
drivers/gpu/drm/xe/xe_vm.c +25 -11
index 31d1ab1b376d..f2dbdfa786b3 100644
@@ -617,21 +617,31 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
 {
 	struct xe_userptr *userptr = container_of(mni, typeof(*userptr), hmmptr.notifier);
 	struct xe_userptr_vma *uvma = container_of(userptr, typeof(*uvma), userptr);
+	u64 range_start, range_end, range_size, range_offset;
 	struct xe_vma *vma = &uvma->vma;
 	struct xe_vm *vm = xe_vma_vm(vma);
 	struct dma_resv_iter cursor;
 	struct dma_fence *fence;
+	u64 start, end;
 	long err;
 
+	range_start = max_t(u64, xe_vma_userptr(vma), range->start);
+	range_end = min_t(u64, xe_vma_userptr_end(vma), range->end);
+	range_size = range_end - range_start;
+	range_offset = range_start - xe_vma_userptr(vma);
+
 	xe_assert(vm->xe, xe_vma_is_userptr(vma));
 	trace_xe_vma_userptr_invalidate(vma);
 
+	start = xe_vma_start(vma) + range_offset;
+	end = start + range_size;
+
 	if (!mmu_notifier_range_blockable(range))
 		return false;
 
 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
 	       "NOTIFIER: addr=0x%016llx, range=0x%016llx",
-		xe_vma_start(vma), xe_vma_size(vma));
+		start, range_size);
 
 	down_write(&vm->userptr.notifier_lock);
 	mmu_interval_set_seq(mni, cur_seq);
@@ -674,11 +684,11 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
 	XE_WARN_ON(err <= 0);
 
 	if (xe_vm_in_fault_mode(vm)) {
-		err = xe_vm_invalidate_vma(vma);
+		err = xe_vm_invalidate_vma(vma, start, end);
 		XE_WARN_ON(err);
 	}
 
-	xe_vma_userptr_dma_unmap_pages(uvma, xe_vma_userptr(vma), xe_vma_userptr_end(vma));
+	xe_vma_userptr_dma_unmap_pages(uvma, range_start, range_end);
 
 	trace_xe_vma_userptr_invalidate_complete(vma);
 
@@ -721,7 +731,8 @@ int xe_vm_userptr_pin(struct xe_vm *vm)
 					      DMA_RESV_USAGE_BOOKKEEP,
 					      false, MAX_SCHEDULE_TIMEOUT);
 
-			err = xe_vm_invalidate_vma(&uvma->vma);
+			err = xe_vm_invalidate_vma(&uvma->vma, xe_vma_start(&uvma->vma),
+				     xe_vma_end(&uvma->vma));
 			xe_vm_unlock(vm);
 			if (err)
 				return err;
@@ -3207,8 +3218,10 @@ void xe_vm_unlock(struct xe_vm *vm)
 }
 
 /**
- * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock
+ * xe_vm_invalidate_vma - invalidate GPU mappings for a range of VMA without a lock
  * @vma: VMA to invalidate
+ * @start: start of the range.
+ * @end: end of the range.
  *
  * Walks a list of page tables leaves which it memset the entries owned by this
  * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is
@@ -3216,7 +3229,7 @@ void xe_vm_unlock(struct xe_vm *vm)
  *
  * Returns 0 for success, negative error code otherwise.
  */
-int xe_vm_invalidate_vma(struct xe_vma *vma)
+int xe_vm_invalidate_vma(struct xe_vma *vma, u64 start, u64 end)
 {
 	struct xe_device *xe = xe_vma_vm(vma)->xe;
 	struct xe_tile *tile;
@@ -3227,11 +3240,13 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
 
 	xe_assert(xe, !xe_vma_is_null(vma));
 	xe_assert(xe, !xe_vma_is_system_allocator(vma));
+	xe_assert(xe, start >= xe_vma_start(vma));
+	xe_assert(xe, end <= xe_vma_end(vma));
 	trace_xe_vma_invalidate(vma);
 
 	vm_dbg(&xe_vma_vm(vma)->xe->drm,
 	       "INVALIDATE: addr=0x%016llx, range=0x%016llx",
-		xe_vma_start(vma), xe_vma_size(vma));
+		start, end - start);
 
 	/* Check that we don't race with page-table updates */
 	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
@@ -3248,14 +3263,15 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
 	}
 
 	for_each_tile(tile, xe, id) {
-		if (xe_pt_zap_ptes(tile, vma)) {
+		if (xe_pt_zap_ptes(tile, vma, start, end)) {
 			tile_needs_invalidate |= BIT(id);
 			xe_device_wmb(xe);
 			/*
 			 * FIXME: We potentially need to invalidate multiple
 			 * GTs within the tile
 			 */
-			seqno[id] = xe_gt_tlb_invalidation_vma(tile->primary_gt, NULL, vma);
+			seqno[id] = xe_gt_tlb_invalidation_range(tile->primary_gt, NULL,
+					     start, end, xe_vma_vm(vma)->usm.asid);
 			if (seqno[id] < 0)
 				return seqno[id];
 		}
@@ -3269,8 +3285,6 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
 		}
 	}
 
-	vma->tile_invalidated = vma->tile_mask;
-
 	return 0;
 }
 
drivers/gpu/drm/xe/xe_vm.h +1 -1
index 89f3306561ad..7c10f6c60b63 100644
@@ -223,7 +223,7 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker);
 struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma,
 				u8 tile_mask);
 
-int xe_vm_invalidate_vma(struct xe_vma *vma);
+int xe_vm_invalidate_vma(struct xe_vma *vma, u64 start, u64 end);
 
 static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm)
 {
drivers/gpu/drm/xe/xe_vm_types.h +0 -3
index 976982972a06..4d9707c19031 100644
@@ -75,9 +75,6 @@ struct xe_vma {
 		struct work_struct destroy_work;
 	};
 
-	/** @tile_invalidated: VMA has been invalidated */
-	u8 tile_invalidated;
-
 	/** @tile_mask: Tile mask of where to create binding for this VMA */
 	u8 tile_mask;
 
Signature · 1 line
2.26.3
author
#26 lore raw

PATCH 27/42 drm/xe: Support range based page table update

In reply to Oak Zeng

Currently the page table update interface only supports whole xe_vma bind. This works fine for BO based driver. But for system allocator, we need to partially bind a vma to GPU page table.

GPU page table update interface such as xe_vma_rebind is modified to support partial vma bind. Binding range (start, end) parameters are added to a few vma bind and page table update functions.

VMA unbind is still whole xe_vma based as there is no requirement to unbind a vma partiallly.

There is no function change in this patch. It is only a interface change as a preparation of the coming system allocator codes.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_gt_pagefault.c |  3 +-
 drivers/gpu/drm/xe/xe_pt.c           | 60 ++++++++++++++++------------
 drivers/gpu/drm/xe/xe_vm.c           | 23 ++++++-----
 drivers/gpu/drm/xe/xe_vm.h           |  2 +-
 4 files changed, 51 insertions(+), 37 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +2 -1
index 9e84cff964b8..847d3fbb38fe 100644
@@ -154,7 +154,8 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 
 		/* Bind VMA only to the GT that has faulted */
 		trace_xe_vma_pf_bind(vma);
-		fence = xe_vma_rebind(vm, vma, BIT(tile->id));
+		fence = xe_vma_rebind(vm, vma, xe_vma_start(vma),
+			    xe_vma_end(vma), BIT(tile->id));
 		if (IS_ERR(fence)) {
 			err = PTR_ERR(fence);
 			if (xe_vm_validate_should_retry(&exec, err, &end))
drivers/gpu/drm/xe/xe_pt.c +35 -25

175 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.c +13 -10
index f2dbdfa786b3..b11579a3fdc2 100644
@@ -803,15 +803,15 @@ static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask)
 }
 
 static void xe_vm_populate_rebind(struct xe_vma_op *op, struct xe_vma *vma,
-				  u8 tile_mask)
+				  u64 start, u64 end, u8 tile_mask)
 {
 	INIT_LIST_HEAD(&op->link);
 	op->tile_mask = tile_mask;
 	op->base.op = DRM_GPUVA_OP_MAP;
-	op->base.map.va.addr = vma->gpuva.va.addr;
-	op->base.map.va.range = vma->gpuva.va.range;
+	op->base.map.va.addr = start;
+	op->base.map.va.range = end - start;
 	op->base.map.gem.obj = vma->gpuva.gem.obj;
-	op->base.map.gem.offset = vma->gpuva.gem.offset;
+	op->base.map.gem.offset = vma->gpuva.gem.offset + (start - xe_vma_start(vma));
 	op->map.vma = vma;
 	op->map.immediate = true;
 	op->map.dumpable = vma->gpuva.flags & XE_VMA_DUMPABLE;
@@ -819,7 +819,7 @@ static void xe_vm_populate_rebind(struct xe_vma_op *op, struct xe_vma *vma,
 }
 
 static int xe_vm_ops_add_rebind(struct xe_vma_ops *vops, struct xe_vma *vma,
-				u8 tile_mask)
+				u64 start, u64 end, u8 tile_mask)
 {
 	struct xe_vma_op *op;
 
@@ -827,7 +827,7 @@ static int xe_vm_ops_add_rebind(struct xe_vma_ops *vops, struct xe_vma *vma,
 	if (!op)
 		return -ENOMEM;
 
-	xe_vm_populate_rebind(op, vma, tile_mask);
+	xe_vm_populate_rebind(op, vma, start, end, tile_mask);
 	list_add_tail(&op->link, &vops->list);
 	xe_vma_ops_incr_pt_update_ops(vops, tile_mask);
 
@@ -866,8 +866,8 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 		else
 			trace_xe_vma_rebind_exec(vma);
 
-		err = xe_vm_ops_add_rebind(&vops, vma,
-					   vma->tile_present);
+		err = xe_vm_ops_add_rebind(&vops, vma, xe_vma_start(vma),
+					   xe_vma_end(vma), vma->tile_present);
 		if (err)
 			goto free_ops;
 	}
@@ -895,7 +895,8 @@ int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 	return err;
 }
 
-struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_mask)
+struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma,
+					    u64 start, u64 end, u8 tile_mask)
 {
 	struct dma_fence *fence = NULL;
 	struct xe_vma_ops vops;
@@ -907,6 +908,8 @@ struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_ma
 	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 	xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
+	xe_assert(vm->xe, start >= xe_vma_start(vma));
+	xe_assert(vm->xe, end <= xe_vma_end(vma));
 
 	xe_vma_ops_init(&vops, vm, NULL, NULL, 0);
 	for_each_tile(tile, vm->xe, id) {
@@ -915,7 +918,7 @@ struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, u8 tile_ma
 			xe_tile_migrate_exec_queue(tile);
 	}
 
-	err = xe_vm_ops_add_rebind(&vops, vma, tile_mask);
+	err = xe_vm_ops_add_rebind(&vops, vma, start, end, tile_mask);
 	if (err)
 		return ERR_PTR(err);
 
drivers/gpu/drm/xe/xe_vm.h +1 -1
index 7c10f6c60b63..680c0c49b2f4 100644
@@ -221,7 +221,7 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm);
 
 int xe_vm_rebind(struct xe_vm *vm, bool rebind_worker);
 struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma,
-				u8 tile_mask);
+				u64 start, u64 end, u8 tile_mask);
 
 int xe_vm_invalidate_vma(struct xe_vma *vma, u64 start, u64 end);
 
Signature · 1 line
2.26.3
author
#27 lore raw

PATCH 28/42 drm/xe/uapi: Add DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM flag

In reply to Oak Zeng

When a xe_vm is created with DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM, xe_vm shares the same process address space with the CPU process who create this xe_vm.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_vm.c       | 2 ++
 drivers/gpu/drm/xe/xe_vm_types.h | 1 +
 include/uapi/drm/xe_drm.h        | 4 ++++
 3 files changed, 7 insertions(+)
drivers/gpu/drm/xe/xe_vm.c +2 -0
index b11579a3fdc2..39765e4b6b78 100644
@@ -1832,6 +1832,8 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data,
 		flags |= XE_VM_FLAG_LR_MODE;
 	if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)
 		flags |= XE_VM_FLAG_FAULT_MODE;
+	if (args->flags & DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM)
+		flags |= XE_VM_FLAG_PARTICIPATE_SVM;
 
 	vm = xe_vm_create(xe, flags);
 	if (IS_ERR(vm))
drivers/gpu/drm/xe/xe_vm_types.h +1 -0
index 4d9707c19031..165c2ca258de 100644
@@ -143,6 +143,7 @@ struct xe_vm {
 #define XE_VM_FLAG_BANNED		BIT(5)
 #define XE_VM_FLAG_TILE_ID(flags)	FIELD_GET(GENMASK(7, 6), flags)
 #define XE_VM_FLAG_SET_TILE_ID(tile)	FIELD_PREP(GENMASK(7, 6), (tile)->id)
+#define XE_VM_FLAG_PARTICIPATE_SVM		BIT(8)
 	unsigned long flags;
 
 	/** @composite_fence_ctx: context composite fence */
include/uapi/drm/xe_drm.h +4 -0
index af8ee6f929d4..ee09ed1ea3c2 100644
@@ -830,6 +830,9 @@ struct drm_xe_gem_mmap_offset {
  *    demand when accessed, and also allows per-VM overcommit of memory.
  *    The xe driver internally uses recoverable pagefaults to implement
  *    this.
+ *  - %DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM - This vm participates
+ *    SVM (shared virtual memory), which means this vm share one process
+ *    address space with CPU process who create this xe_vm.
  */
 struct drm_xe_vm_create {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -838,6 +841,7 @@ struct drm_xe_vm_create {
 #define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE	(1 << 0)
 #define DRM_XE_VM_CREATE_FLAG_LR_MODE	        (1 << 1)
 #define DRM_XE_VM_CREATE_FLAG_FAULT_MODE	(1 << 2)
+#define DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM	(1 << 3)
 	/** @flags: Flags */
 	__u32 flags;
 
Signature · 1 line
2.26.3
author
#28 lore raw

PATCH 29/42 drm/xe/svm: Create userptr if page fault occurs on system_allocator VMA

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

If a page fault occurs on system_allocator VMA, create a userptr VMA to replaced fault region and map to GPU.

v1: Pass userptr to the req_offset of sm_map_ops_create function. This fix malloc'd memory failure (Oak)
drop xe_vma_is_userptr(vma) condition. XE_VMA_FAULT_USERPTR is enough to determine a fault userptr (Matt)

v2: don't pin fault userptr during fault userptr creation. Fault userptr is created during gpu page fault. After creation, we decide whether we need to migrate it to GPU vram. So pin fault userptr after migration is enough. A pin before migration is redundant. (Oak)

v3: remove mm field from xe_vm as we add mm in drm_gpuvm now. rebase (Oak)

Cc Thomas Hellström <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_gt_pagefault.c |  12 +++
 drivers/gpu/drm/xe/xe_vm.c           | 118 +++++++++++++++++++++++++--
 drivers/gpu/drm/xe/xe_vm.h           |   7 ++
 drivers/gpu/drm/xe/xe_vm_types.h     |   1 +
 4 files changed, 133 insertions(+), 5 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +12 -0
index 847d3fbb38fe..018fa19ff80e 100644
@@ -132,6 +132,18 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 	trace_xe_vma_pagefault(vma);
 	atomic = access_is_atomic(pf->access_type);
 
+	/*
+	 * Create userptr VMA if fault occurs in a range reserved for system
+	 * allocator.
+	 */
+	if (xe_vma_is_system_allocator(vma)) {
+		vma = xe_vm_fault_userptr(vm, pf->page_addr);
+		if (IS_ERR(vma)) {
+			xe_vm_kill(vm, true);
+			return PTR_ERR(vma);
+		}
+	}
+
 retry_userptr:
 	if (xe_vma_is_userptr(vma) &&
 	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
drivers/gpu/drm/xe/xe_vm.c +113 -5

145 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +7 -0
index 680c0c49b2f4..a31409b87b8a 100644
@@ -172,6 +172,11 @@ static inline bool xe_vma_is_userptr(struct xe_vma *vma)
 		!xe_vma_is_system_allocator(vma);
 }
 
+static inline bool xe_vma_is_fault_userptr(struct xe_vma *vma)
+{
+	return vma->gpuva.flags & XE_VMA_FAULT_USERPTR;
+}
+
 /**
  * to_userptr_vma() - Return a pointer to an embedding userptr vma
  * @vma: Pointer to the embedded struct xe_vma
@@ -252,6 +257,8 @@ int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma);
 
 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma);
 
+struct xe_vma *xe_vm_fault_userptr(struct xe_vm *vm, u64 fault_addr);
+
 bool xe_vm_validate_should_retry(struct drm_exec *exec, int err, ktime_t *end);
 
 int xe_vm_lock_vma(struct drm_exec *exec, struct xe_vma *vma);
drivers/gpu/drm/xe/xe_vm_types.h +1 -0
index 165c2ca258de..c1bffa60cefc 100644
@@ -33,6 +33,7 @@ struct xe_vm_pgtable_update_op;
 #define XE_VMA_PTE_COMPACT	(DRM_GPUVA_USERBITS << 7)
 #define XE_VMA_DUMPABLE		(DRM_GPUVA_USERBITS << 8)
 #define XE_VMA_SYSTEM_ALLOCATOR	(DRM_GPUVA_USERBITS << 9)
+#define XE_VMA_FAULT_USERPTR	(DRM_GPUVA_USERBITS << 10)
 
 /** struct xe_userptr - User pointer */
 struct xe_userptr {
Signature · 1 line
2.26.3
author
#29 lore raw

PATCH 30/42 drm/xe/svm: Add faulted userptr VMA garbage collector

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

When a faulted userptr VMA (allocated by page handler) is invalidated add to list which a garbage collector will unmap from GPU, destroy faulted userptr VMA, and replace with system_allocator VMA.

v1: Run gargabe collector only on MMU_NOTIFY_UNMAP event. For other
events, we just invalidate GPU page table but keep the vma because the userptr is still exist. On next GPU access, we will revalidate and rebind this userptr to GPU(Oak)
v2: rebase
support range based userptr invalidation in garbage collector. Allow partial of a userptr to be invalidated (such as trigger by partial munmap of a userptr) (Oak)
Fix vm->lock recursive lock issue (Oak)

Cc Thomas Hellström <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_gt_pagefault.c |   1 +
 drivers/gpu/drm/xe/xe_vm.c           | 162 +++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_vm.h           |   8 ++
 drivers/gpu/drm/xe/xe_vm_types.h     |  15 +++
 4 files changed, 186 insertions(+)
drivers/gpu/drm/xe/xe_gt_pagefault.c +1 -0
index 018fa19ff80e..2adc7d774f3d 100644
@@ -145,6 +145,7 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 	}
 
 retry_userptr:
+	xe_vm_userptr_garbage_collector(vm);
 	if (xe_vma_is_userptr(vma) &&
 	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
 		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
drivers/gpu/drm/xe/xe_vm.c +162 -0

191 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +8 -0
index a31409b87b8a..b3b6ceec39ba 100644
@@ -285,6 +285,14 @@ void xe_vm_kill(struct xe_vm *vm, bool unlocked);
  */
 #define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))
 
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+				u8 tile_mask);
+void xe_vma_ops_free(struct xe_vma_ops *vops);
+struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
+
+void xe_vm_kill(struct xe_vm *vm, bool unlocked);
+void xe_vm_userptr_garbage_collector(struct xe_vm *vm);
+
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
 #define vm_dbg drm_dbg
 #else
drivers/gpu/drm/xe/xe_vm_types.h +15 -0
index c1bffa60cefc..6ebe05242997 100644
@@ -40,6 +40,10 @@ struct xe_userptr {
 	struct drm_hmmptr hmmptr;
 	/** @invalidate_link: Link for the vm::userptr.invalidated list */
 	struct list_head invalidate_link;
+	/** invalidation start address */
+	u64 invalidate_start;
+	/** invalidation range */
+	u64 invalidate_range;
 	/** @userptr: link into VM repin list if userptr. */
 	struct list_head repin_link;
 	/**
@@ -212,6 +216,17 @@ struct xe_vm {
 		 * write mode.
 		 */
 		struct list_head invalidated;
+		/**
+		 * @userptr.fault_invalidated: List of invalidated userptrs,
+		 * craeted by page fault, which will be destroy by the garbage
+		 * collector. Protected from access with the @invalidated_lock.
+		 */
+		struct list_head fault_invalidated;
+		/**
+		 * @userptr.garbage_collector: worker to implement destroying of
+		 * userptrs on @userptr.fault_invalidated list.
+		 */
+		struct work_struct garbage_collector;
 	} userptr;
 
 	/** @preempt: preempt state */
Signature · 1 line
2.26.3
author
#30 lore raw

PATCH 31/42 drm/xe: Introduce helper to get tile from memory region

In reply to Oak Zeng

Introduce a simple helper to retrieve tile from memory region

v1: move the function to xe_device.h (Matt)
improve commit message, add kerneldoc (Thomas)
v2: move the function to xe_tile.h (Matt)

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_tile.h | 8 ++++++++
 1 file changed, 8 insertions(+)
drivers/gpu/drm/xe/xe_tile.h +8 -0
index 1c9e42ade6b0..091c5a98110a 100644
@@ -15,4 +15,12 @@ int xe_tile_init_noalloc(struct xe_tile *tile);
 
 void xe_tile_migrate_wait(struct xe_tile *tile);
 
+/**
+ * xe_mem_region_to_tile() - retrieve tile from memory region
+ * @mr: the memory region we retrieve tile from
+ */
+static inline struct xe_tile *xe_mem_region_to_tile(struct xe_mem_region *mr)
+{
+	return container_of(mr, struct xe_tile, mem.vram);
+}
 #endif
Signature · 1 line
2.26.3
author
#31 lore raw

PATCH 32/42 drm/xe/svm: implement functions to allocate and free device memory

In reply to Oak Zeng

Function xe_svm_alloc_pages allocate pages from drm buddy and perform house keeping work for all the pages allocated, such as get a page refcount, set page's zone_device_data to point to the drm buddy block that the page belongs to.

Function xe_svm_free_page free one page back to drm buddy allocator.

v1: Drop buddy block meta data (Matt)
Take vram_mgr->lock during buddy operation (Matt) Moving to page granularity memory free (Oak)

Cc Matthew Brost <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
Cc Matthew Auld <[email protected]>
---
 drivers/gpu/drm/drm_svm.c   |   1 -
 drivers/gpu/drm/xe/Makefile |   1 +
 drivers/gpu/drm/xe/xe_svm.c | 148 ++++++++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_svm.h |  16 ++++
 4 files changed, 165 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/xe/xe_svm.c
 create mode 100644 drivers/gpu/drm/xe/xe_svm.h
drivers/gpu/drm/drm_svm.c +0 -1
index 967e7addae54..a751cc2c9c91 100644
@@ -3,7 +3,6 @@
  * Copyright © 2024 Intel Corporation
  */
 
-
 #include <linux/scatterlist.h>
 #include <linux/mmu_notifier.h>
 #include <linux/dma-mapping.h>
drivers/gpu/drm/xe/Makefile +1 -0
index 6457f1b74cd8..cf5ce447648c 100644
@@ -108,6 +108,7 @@ xe-y += xe_bb.o \
 	xe_sa.o \
 	xe_sched_job.o \
 	xe_step.o \
+	xe_svm.o \
 	xe_sync.o \
 	xe_tile.o \
 	xe_tile_sysfs.o \
drivers/gpu/drm/xe/xe_svm.c new file +148 -0

148 lines not on this page · raw message

drivers/gpu/drm/xe/xe_svm.h new file +16 -0
new file mode 100644
index 000000000000..09fef8ea97f3
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2024 Intel Corporation
+ */
+
+#ifndef _XE_SVM_
+#define _XE_SVM_
+
+struct drm_mem_region;
+struct page;
+
+int xe_svm_alloc_pages(struct drm_mem_region *mr,
+            unsigned long npages, unsigned long *pfn);
+void xe_svm_free_page(struct page *page);
+
+#endif
Signature · 1 line
2.26.3
author
#32 lore raw

PATCH 33/42 drm/xe/svm: Get drm device from drm memory region

In reply to Oak Zeng

Implement a function xe_mem_region_to_device to get the drm device which has a specific mem region

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_device.h | 8 ++++++++
 1 file changed, 8 insertions(+)
drivers/gpu/drm/xe/xe_device.h +8 -0
index bb07f5669dbb..bae8ebeda20c 100644
@@ -9,6 +9,7 @@
 #include <drm/drm_util.h>
 
 #include "xe_device_types.h"
+#include "xe_tile.h"
 
 static inline struct xe_device *to_xe_device(const struct drm_device *dev)
 {
@@ -170,4 +171,11 @@ static inline bool xe_device_wedged(struct xe_device *xe)
 
 void xe_device_declare_wedged(struct xe_device *xe);
 
+static inline struct drm_device *xe_mem_region_to_device(struct drm_mem_region *mr)
+{
+	struct xe_mem_region *xe_mr = container_of(mr, struct xe_mem_region, drm_mr);
+	struct xe_tile *tile = xe_mem_region_to_tile(xe_mr);
+
+	return &tile->xe->drm;
+}
 #endif
Signature · 1 line
2.26.3
author
#33 lore raw

PATCH 34/42 drm/xe/svm: Get page map owner of a memory region

In reply to Oak Zeng

Introduce function xe_svm_mem_region_to_pagemap_owner to get a memory region's page map owner.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_svm.c | 8 ++++++++
 drivers/gpu/drm/xe/xe_svm.h | 1 +
 2 files changed, 9 insertions(+)
drivers/gpu/drm/xe/xe_svm.c +8 -0
index a2c52f205ea0..d2838ce46eaf 100644
@@ -146,3 +146,11 @@ int xe_svm_alloc_pages(struct drm_mem_region *mr,
 
 	return ret;
 }
+
+void* xe_svm_mem_region_to_pagemap_owner(struct drm_mem_region *mr)
+{
+	struct xe_mem_region *xe_mr = container_of(mr, struct xe_mem_region, drm_mr);
+	struct xe_tile *tile = xe_mem_region_to_tile(xe_mr);
+
+	return tile->xe;
+}
drivers/gpu/drm/xe/xe_svm.h +1 -0
index 09fef8ea97f3..b70551d12578 100644
@@ -12,5 +12,6 @@ struct page;
 int xe_svm_alloc_pages(struct drm_mem_region *mr,
             unsigned long npages, unsigned long *pfn);
 void xe_svm_free_page(struct page *page);
+void* xe_svm_mem_region_to_pagemap_owner(struct drm_mem_region *mr);
 
 #endif
Signature · 1 line
2.26.3
author
#34 lore raw

PATCH 35/42 drm/xe/svm: Add migrate layer functions for SVM support

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

Add functions which migrate to / from VRAM accepting a single DPA argument (VRAM) and array of dma addresses (SRAM).

FIXME: Support non-contiguous VRAM DPA. The VRAM DPA can be an array and we can dynamically map DPAs into contiguous device virtual address space like what we did for SRAM, and still use one single blitter command for migration

Cc Thomas Hellström <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_migrate.c | 126 ++++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_migrate.h |   5 ++
 2 files changed, 131 insertions(+)
drivers/gpu/drm/xe/xe_migrate.c +126 -0
index 15c8973c0495..40867d0de20c 100644
@@ -1457,6 +1457,132 @@ void xe_migrate_wait(struct xe_migrate *m)
 		dma_fence_wait(m->fence, false);
 }
 
+static u32 pte_update_cmd_size(u64 size)
+{
+	u32 dword;
+	u64 entries = DIV_ROUND_UP(size, XE_PAGE_SIZE);
+
+	XE_WARN_ON(size > MAX_PREEMPTDISABLE_TRANSFER);
+	/*
+	 * MI_STORE_DATA_IMM command is used to update page table. Each
+	 * instruction can update maximumly 0x1ff pte entries. To update
+	 * n (n <= 0x1ff) pte entries, we need:
+	 * 1 dword for the MI_STORE_DATA_IMM command header (opcode etc)
+	 * 2 dword for the page table's physical location
+	 * 2*n dword for value of pte to fill (each pte entry is 2 dwords)
+	 */
+	dword = (1 + 2) * DIV_ROUND_UP(entries, 0x1ff);
+	dword += entries * 2;
+
+	return dword;
+}
+
+static void build_pt_update_batch_sram(struct xe_migrate *m,
+				       struct xe_bb *bb, u32 pt_offset,
+				       dma_addr_t *sram_addr, u32 size)
+{
+	u16 pat_index = tile_to_xe(m->tile)->pat.idx[XE_CACHE_WB];
+	u32 ptes;
+	int i = 0;
+
+	ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE);
+	while (ptes) {
+		u32 chunk = min(0x1ffU, ptes);
+
+		bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk);
+		bb->cs[bb->len++] = pt_offset;
+		bb->cs[bb->len++] = 0;
+
+		pt_offset += chunk * 8;
+		ptes -= chunk;
+
+		while (chunk--) {
+			u64 addr = sram_addr[i++] & PAGE_MASK;
+
+			xe_tile_assert(m->tile, addr);
+			addr = m->q->vm->pt_ops->pte_encode_addr(m->tile->xe,
+								 addr, pat_index,
+								 0, false, 0);
+			bb->cs[bb->len++] = lower_32_bits(addr);
+			bb->cs[bb->len++] = upper_32_bits(addr);
+		}
+	}
+}
+
+struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
+					 unsigned long npages,
+					 dma_addr_t *sram_addr, u64 vram_addr,
+					 bool dst_vram)
+{
+	struct xe_gt *gt = m->tile->primary_gt;
+	struct xe_device *xe = gt_to_xe(gt);
+	struct dma_fence *fence = NULL;
+	u32 batch_size = 2;
+	u64 src_L0_ofs, dst_L0_ofs;
+	u64 round_update_size;
+	struct xe_sched_job *job;
+	struct xe_bb *bb;
+	u32 update_idx, pt_slot = 0;
+	int err;
+
+	round_update_size = min_t(u64, npages * PAGE_SIZE,
+				  MAX_PREEMPTDISABLE_TRANSFER);
+	batch_size += pte_update_cmd_size(round_update_size);
+	batch_size += EMIT_COPY_DW;
+
+	bb = xe_bb_new(gt, batch_size, true);
+	if (IS_ERR(bb)) {
+		err = PTR_ERR(bb);
+		return ERR_PTR(err);
+	}
+
+	build_pt_update_batch_sram(m, bb, pt_slot * XE_PAGE_SIZE,
+				   sram_addr, round_update_size);
+
+	if (dst_vram) {
+		src_L0_ofs = xe_migrate_vm_addr(pt_slot, 0);
+		dst_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr);
+
+	} else {
+		src_L0_ofs = xe_migrate_vram_ofs(xe, vram_addr);
+		dst_L0_ofs = xe_migrate_vm_addr(pt_slot, 0);
+	}
+
+	bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
+	update_idx = bb->len;
+
+	emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, round_update_size,
+		  XE_PAGE_SIZE);
+
+	mutex_lock(&m->job_mutex);
+	job = xe_bb_create_migration_job(m->q, bb,
+					 xe_migrate_batch_base(m, true),
+					 update_idx);
+	if (IS_ERR(job)) {
+		err = PTR_ERR(job);
+		goto err;
+	}
+
+	xe_sched_job_add_migrate_flush(job, 0);
+	xe_sched_job_arm(job);
+	fence = dma_fence_get(&job->drm.s_fence->finished);
+	xe_sched_job_push(job);
+
+	dma_fence_put(m->fence);
+	m->fence = dma_fence_get(fence);
+	mutex_unlock(&m->job_mutex);
+
+	xe_bb_free(bb, fence);
+
+	return fence;
+
+err:
+	mutex_unlock(&m->job_mutex);
+	xe_bb_free(bb, NULL);
+
+	return ERR_PTR(err);
+}
+
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 #include "tests/xe_migrate.c"
 #endif
drivers/gpu/drm/xe/xe_migrate.h +5 -0
index 453e0ecf5034..c6a18be1373f 100644
@@ -115,4 +115,9 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
 void xe_migrate_wait(struct xe_migrate *m);
 
 struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
+
+struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
+					 unsigned long npages,
+					 dma_addr_t *sram_addr, u64 vram_addr,
+					 bool dst_vram);
 #endif
Signature · 1 line
2.26.3
author
#35 lore raw

PATCH 36/42 drm/xe/svm: introduce svm migration function

In reply to Oak Zeng

xe_svm_migrate is introduced to migrate b/t two migration vectors. This will be a callback function registered to drm layer for svm memory migration.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_svm.c | 46 +++++++++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_svm.h |  3 +++
 2 files changed, 49 insertions(+)
drivers/gpu/drm/xe/xe_svm.c +46 -0
index d2838ce46eaf..014b70250abd 100644
@@ -7,8 +7,10 @@
 #include <linux/sched/mm.h>
 #include <linux/gfp.h>
 #include <drm/drm_buddy.h>
+#include <drm/drm_svm.h>
 #include "xe_ttm_vram_mgr_types.h"
 #include "xe_device_types.h"
+#include "xe_migrate.h"
 #include "xe_device.h"
 #include "xe_tile.h"
 #include "xe_svm.h"
@@ -154,3 +156,47 @@ void* xe_svm_mem_region_to_pagemap_owner(struct drm_mem_region *mr)
 
 	return tile->xe;
 }
+
+/**
+ * xe_svm_migrate() - migrate b/t two migration vectors.
+ * @src_vec: source migration vector
+ * @dst_vec: dst migration vector
+ *
+ * For now this function only support migration b/t system memory
+ * and device memory. It can be extended to support migration b/t
+ * two device memory regions.
+ *
+ * FIXME: right now it requires vram dpa to be physically contiguous.
+ * This is guaranteed by the DRM_BUDDY_CONTIGUOUS_ALLOCATION in
+ * xe_svm_alloc_pages. In the future we can support migration with
+ * non-contiguous vram. This requires change to xe_migrate_vram
+ * to map non-contiguous vram to contiguous device virtual address
+ * space (aka not using identity vram migrate vm mapping)
+ */
+struct dma_fence* xe_svm_migrate(struct migrate_vec *src_vec,
+            struct migrate_vec *dst_vec)
+{
+	unsigned long npages = src_vec->npages;
+	struct xe_mem_region *xe_mr;
+	struct drm_mem_region *mr;
+	bool dst_vram = false;
+	struct xe_tile *tile;
+	dma_addr_t *sram_addr;
+	u64 vram_addr;
+
+	if (dst_vec->mr) {
+		mr = dst_vec->mr;
+		dst_vram = true;
+		vram_addr = dst_vec->addr_vec[0].dpa;
+		sram_addr = &src_vec->addr_vec[0].dma_addr;
+	} else {
+		mr = src_vec->mr;
+		vram_addr = src_vec->addr_vec[0].dpa;
+		sram_addr = &dst_vec->addr_vec[0].dma_addr;
+	}
+
+	xe_mr = container_of(mr, struct xe_mem_region, drm_mr);
+	tile = xe_mem_region_to_tile(xe_mr);
+
+	return xe_migrate_vram(tile->migrate, npages, sram_addr, vram_addr, dst_vram);
+}
drivers/gpu/drm/xe/xe_svm.h +3 -0
index b70551d12578..1b891b2a7587 100644
@@ -8,10 +8,13 @@
 
 struct drm_mem_region;
 struct page;
+struct migrate_vec;
 
 int xe_svm_alloc_pages(struct drm_mem_region *mr,
             unsigned long npages, unsigned long *pfn);
 void xe_svm_free_page(struct page *page);
 void* xe_svm_mem_region_to_pagemap_owner(struct drm_mem_region *mr);
+struct dma_fence* xe_svm_migrate(struct migrate_vec *src_vec,
+            struct migrate_vec *dst_vec);
 
 #endif
Signature · 1 line
2.26.3
author
#36 lore raw

PATCH 37/42 drm/xe/svm: Register xe memory region to drm layer

In reply to Oak Zeng

Register xe memory region to drm layer for SVM purpose. This will essentially create struct page backing for xe device memory, which will be used by SVM functions.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_tile.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
drivers/gpu/drm/xe/xe_tile.c +19 -0
index 109f3118e821..6c50df281b11 100644
@@ -3,7 +3,9 @@
  * Copyright © 2023 Intel Corporation
  */
 
+#include <linux/memremap.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_svm.h>
 
 #include "xe_device.h"
 #include "xe_ggtt.h"
@@ -14,6 +16,7 @@
 #include "xe_tile_sysfs.h"
 #include "xe_ttm_vram_mgr.h"
 #include "xe_wa.h"
+#include "xe_svm.h"
 
 /**
  * DOC: Multi-tile Design
@@ -142,6 +145,15 @@ static int tile_ttm_mgr_init(struct xe_tile *tile)
 	return 0;
 }
 
+static void __init_drm_mem_region_ops(struct drm_mem_region_ops *ops)
+{
+	ops->drm_mem_region_alloc_pages = &xe_svm_alloc_pages;
+	ops->drm_mem_region_free_page = &xe_svm_free_page;
+	ops->drm_mem_region_migrate = &xe_svm_migrate;
+	ops->drm_mem_region_pagemap_owner = &xe_svm_mem_region_to_pagemap_owner;
+	ops->drm_mem_region_get_device = &xe_mem_region_to_device;
+}
+
 /**
  * xe_tile_init_noalloc - Init tile up to the point where allocations can happen.
  * @tile: The tile to initialize.
@@ -159,6 +171,8 @@ static int tile_ttm_mgr_init(struct xe_tile *tile)
 int xe_tile_init_noalloc(struct xe_tile *tile)
 {
 	int err;
+	struct xe_device *xe = tile_to_xe(tile);
+	struct drm_device *drm = &xe->drm;
 
 	err = tile_ttm_mgr_init(tile);
 	if (err)
@@ -172,6 +186,11 @@ int xe_tile_init_noalloc(struct xe_tile *tile)
 
 	err = xe_tile_sysfs_init(tile);
 
+	if (xe->info.has_usm) {
+		__init_drm_mem_region_ops(&tile->mem.vram.drm_mr.mr_ops);
+		drm_svm_register_mem_region(drm, &tile->mem.vram.drm_mr, MEMORY_DEVICE_PRIVATE);
+	}
+
 	return 0;
 }
 
Signature · 1 line
2.26.3
author
#37 lore raw

PATCH 38/42 drm/xe/svm: Introduce DRM_XE_SVM kernel config

In reply to Oak Zeng

Introduce a DRM_XE_SVM kernel config entry for xe svm feature. xe svm feature allows share virtual address space between CPU and GPU program.

v1: Improve commit message (Thomas)
Avoid using #if directive (Thomas)

Cc Matthew Brost <[email protected]>
Cc Thomas Hellström <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/Kconfig   | 21 +++++++++++++++++++++
 drivers/gpu/drm/xe/xe_tile.c | 11 ++++++++---
 2 files changed, 29 insertions(+), 3 deletions(-)
drivers/gpu/drm/xe/Kconfig +21 -0
index 7bbe46a98ff1..27d6145cefa7 100644
@@ -84,6 +84,27 @@ config DRM_XE_FORCE_PROBE
 	  4571.
 
 	  Use "!*" to block the probe of the driver for all known devices.
+config DRM_XE_SVM
+	bool "Enable Shared Virtual Memory support in xe"
+	depends on DRM_XE
+	depends on ARCH_ENABLE_MEMORY_HOTPLUG
+	depends on ARCH_ENABLE_MEMORY_HOTREMOVE
+	depends on MEMORY_HOTPLUG
+	depends on MEMORY_HOTREMOVE
+	depends on ARCH_HAS_PTE_DEVMAP
+	depends on SPARSEMEM_VMEMMAP
+	depends on ZONE_DEVICE
+	depends on DEVICE_PRIVATE
+	depends on MMU
+	select HMM_MIRROR
+	select MMU_NOTIFIER
+	default y
+	help
+	  Choose this option if you want Shared Virtual Memory (SVM)
+	  support in xe. With SVM, virtual address space is shared
+	  between CPU and GPU. This means any virtual address such
+	  as malloc or mmap returns, variables on stack, or global
+	  memory pointers, can be used for GPU transparently.
 
 menu "drm/Xe Debugging"
 depends on DRM_XE
drivers/gpu/drm/xe/xe_tile.c +8 -3
index 6c50df281b11..1f938d001094 100644
@@ -171,8 +171,13 @@ static void __init_drm_mem_region_ops(struct drm_mem_region_ops *ops)
 int xe_tile_init_noalloc(struct xe_tile *tile)
 {
 	int err;
-	struct xe_device *xe = tile_to_xe(tile);
-	struct drm_device *drm = &xe->drm;
+	struct xe_device __maybe_unused *xe;
+	struct drm_device __maybe_unused *drm;
+
+	if (IS_ENABLED(CONFIG_DRM_XE_SVM)) {
+		xe = tile_to_xe(tile);
+		drm = &xe->drm;
+	}
 
 	err = tile_ttm_mgr_init(tile);
 	if (err)
@@ -186,7 +191,7 @@ int xe_tile_init_noalloc(struct xe_tile *tile)
 
 	err = xe_tile_sysfs_init(tile);
 
-	if (xe->info.has_usm) {
+	if (IS_ENABLED(CONFIG_DRM_XE_SVM) && xe->info.has_usm) {
 		__init_drm_mem_region_ops(&tile->mem.vram.drm_mr.mr_ops);
 		drm_svm_register_mem_region(drm, &tile->mem.vram.drm_mr, MEMORY_DEVICE_PRIVATE);
 	}
Signature · 1 line
2.26.3
author
#38 lore raw

PATCH 39/42 drm/xe/svm: Migration from sram to vram for system allocator

In reply to Oak Zeng

If applicable, migrate a range of hmmptr from sram to vram for system allocator. Traditional userptr is not migrated. Only userptr created during fault (aka userptr splitted from system allocator vma, aka fault userptr in codes) can be migrated.

Instead of the whole userptr, only a sub-range of userptr is populated and bind to GPU page table. Right now the range granularity is 2MiB, which will be overwritten by memory attribute API.

FIXME: The migration should be conditional on user memory attributes setting. Add this logic when memory attributes are supported

v1: Use none NULL owner when call drm_svm_hmmptr_populate (Himal)

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
---
 drivers/gpu/drm/xe/xe_gt_pagefault.c | 41 ++++++++++++++++++++++++++--
 drivers/gpu/drm/xe/xe_vm.c           | 18 +++++++-----
 drivers/gpu/drm/xe/xe_vm.h           |  2 +-
 3 files changed, 50 insertions(+), 11 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +38 -3
index 2adc7d774f3d..ada11feb0c7f 100644
@@ -128,6 +128,12 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 	ktime_t end = 0;
 	int err;
 	bool atomic;
+	/**FIXME: use migration granularity from memory attributes*/
+	u64 migrate_granularity = SZ_2M;
+	u64 fault_addr = pf->page_addr;
+	u64 fault_offset, fault_cpu_addr;
+	u64 aligned_cpu_fault_start, aligned_cpu_fault_end;
+	u64 cpu_va_start, cpu_va_end, gpu_va_start, gpu_va_end;
 
 	trace_xe_vma_pagefault(vma);
 	atomic = access_is_atomic(pf->access_type);
@@ -144,13 +150,42 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 		}
 	}
 
+	fault_offset = fault_addr - xe_vma_start(vma);
+	fault_cpu_addr = xe_vma_userptr(vma) + fault_offset;
+	aligned_cpu_fault_start = ALIGN_DOWN(fault_cpu_addr, migrate_granularity);
+	aligned_cpu_fault_end  = aligned_cpu_fault_start + migrate_granularity;
+
+	if (xe_vma_is_userptr(vma)) {
+		cpu_va_start = max_t(u64, xe_vma_userptr(vma), aligned_cpu_fault_start);
+		cpu_va_end = min_t(u64, xe_vma_userptr_end(vma), aligned_cpu_fault_end);
+		gpu_va_start = xe_vma_start(vma) + (cpu_va_start - xe_vma_userptr(vma));
+		gpu_va_end = xe_vma_end(vma) - (xe_vma_userptr_end(vma) - cpu_va_end);
+	} else {
+		gpu_va_start = xe_vma_start(vma);
+		gpu_va_end = xe_vma_end(vma);
+	}
+
 retry_userptr:
 	xe_vm_userptr_garbage_collector(vm);
 	if (xe_vma_is_userptr(vma) &&
 	    xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
 		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
+		struct xe_userptr *userptr = &uvma->userptr;
+		struct drm_hmmptr *hmmptr = &userptr->hmmptr;
+
+		mmap_read_lock(hmmptr->notifier.mm);
+		if (xe_vma_is_fault_userptr(vma)) {
+			/**FIXME: add migration policy here*/
+			err = drm_svm_migrate_hmmptr_to_vram(&vm->gpuvm, &tile->mem.vram.drm_mr,
+                hmmptr, cpu_va_start, cpu_va_end);
+			if (err) {
+				mmap_read_unlock(hmmptr->notifier.mm);
+				return err;
+			}
 
-		err = xe_vma_userptr_pin_pages(uvma);
+		}
+		err = xe_vma_userptr_pin_pages(uvma, cpu_va_start, cpu_va_end, true);
+		mmap_read_unlock(hmmptr->notifier.mm);
 		if (err)
 			return err;
 	}
@@ -167,8 +202,8 @@ static int handle_vma_pagefault(struct xe_tile *tile, struct pagefault *pf,
 
 		/* Bind VMA only to the GT that has faulted */
 		trace_xe_vma_pf_bind(vma);
-		fence = xe_vma_rebind(vm, vma, xe_vma_start(vma),
-			    xe_vma_end(vma), BIT(tile->id));
+		fence = xe_vma_rebind(vm, vma, gpu_va_start,
+			    gpu_va_end, BIT(tile->id));
 		if (IS_ERR(fence)) {
 			err = PTR_ERR(fence);
 			if (xe_vm_validate_should_retry(&exec, err, &end))
drivers/gpu/drm/xe/xe_vm.c +11 -7
index af316502ce6f..afb7d78a2260 100644
@@ -76,6 +76,7 @@ static void xe_vma_userptr_dma_map_pages(struct xe_userptr_vma *uvma,
 	xe_assert(xe, xe_vma_is_userptr(vma));
 	xe_assert(xe, start >= xe_vma_userptr(vma));
 	xe_assert(xe, end <= xe_vma_userptr_end(vma));
+
 	drm_svm_hmmptr_map_dma_pages(hmmptr, page_idx, npages);
 }
 
@@ -96,7 +97,7 @@ static void xe_vma_userptr_dma_unmap_pages(struct xe_userptr_vma *uvma,
 	drm_svm_hmmptr_unmap_dma_pages(hmmptr, page_idx, npages);
 }
 
-int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
+int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma, u64 start, u64 end, bool mmap_locked)
 {
 	struct drm_hmmptr *hmmptr = &uvma->userptr.hmmptr;
 	struct xe_vma *vma = &uvma->vma;
@@ -106,14 +107,15 @@ int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma)
 
 	lockdep_assert_held(&vm->lock);
 	xe_assert(xe, xe_vma_is_userptr(vma));
+	xe_assert(xe, start >= xe_vma_userptr(vma));
+	xe_assert(xe, end <= xe_vma_userptr_end(vma));
 
-	ret = drm_svm_hmmptr_populate(hmmptr, NULL, xe_vma_userptr(vma),
-			 xe_vma_userptr(vma) + xe_vma_size(vma),
-			 !xe_vma_read_only(vma), false);
+	ret = drm_svm_hmmptr_populate(hmmptr, xe, start, end,
+			 !xe_vma_read_only(vma), mmap_locked);
 	if (ret)
 		return ret;
 
-	xe_vma_userptr_dma_map_pages(uvma, xe_vma_userptr(vma), xe_vma_userptr_end(vma));
+	xe_vma_userptr_dma_map_pages(uvma, start, end);
 	return 0;
 }
 
@@ -736,7 +738,8 @@ int xe_vm_userptr_pin(struct xe_vm *vm)
 	/* Pin and move to temporary list */
 	list_for_each_entry_safe(uvma, next, &vm->userptr.repin_list,
 				 userptr.repin_link) {
-		err = xe_vma_userptr_pin_pages(uvma);
+		err = xe_vma_userptr_pin_pages(uvma, xe_vma_userptr(&uvma->vma),
+			xe_vma_userptr_end(&uvma->vma), false);
 		if (err == -EFAULT) {
 			list_del_init(&uvma->userptr.repin_link);
 
@@ -2116,7 +2119,8 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op,
 		drm_exec_fini(&exec);
 
 	if (xe_vma_is_userptr(vma) && !xe_vma_is_fault_userptr(vma)) {
-		err = xe_vma_userptr_pin_pages(to_userptr_vma(vma));
+		err = xe_vma_userptr_pin_pages(to_userptr_vma(vma), xe_vma_userptr(vma),
+				xe_vma_userptr_end(vma), false);
 		if (err) {
 			prep_vma_destroy(vm, vma, false);
 			xe_vma_destroy_unlocked(vma);
drivers/gpu/drm/xe/xe_vm.h +1 -1
index b3b6ceec39ba..f24891cb1fcb 100644
@@ -253,7 +253,7 @@ static inline void xe_vm_reactivate_rebind(struct xe_vm *vm)
 	}
 }
 
-int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma);
+int xe_vma_userptr_pin_pages(struct xe_userptr_vma *uvma, u64 start, u64 end, bool mmap_locked);
 
 int xe_vma_userptr_check_repin(struct xe_userptr_vma *uvma);
 
Signature · 1 line
2.26.3
author
#39 lore raw

PATCH 40/42 drm/xe/svm: Determine a vma is backed by device memory

In reply to Oak Zeng

With system allocator, a userptr can now be back by device memory also. Introduce a helper function xe_vma_is_devmem to determine whether a range of vma is backed by device memory.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_pt.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +24 -2
index f6fc2dcb3767..c68ff02a931e 100644
@@ -3,6 +3,7 @@
  * Copyright © 2022 Intel Corporation
  */
 
+#include <linux/hmm.h>
 #include "xe_pt.h"
 
 #include "regs/xe_gtt_defs.h"
@@ -578,6 +579,28 @@ static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
 	.pt_entry = xe_pt_stage_bind_entry,
 };
 
+static bool xe_vma_is_devmem(struct xe_vma *vma, u64 start)
+{
+	if (xe_vma_is_userptr(vma)) {
+		struct xe_userptr_vma *uvma = to_userptr_vma(vma);
+		struct drm_hmmptr *hmmptr = &uvma->userptr.hmmptr;
+		u64 offset = start - xe_vma_start(vma);
+		u64 page_idx = offset >> PAGE_SHIFT;
+		u64 hmm_pfn = hmmptr->pfn[page_idx];
+		struct page *page = hmm_pfn_to_page(hmm_pfn);
+
+		/**
+		 * FIXME: Assume there is no mixture system memory and device
+		 * memory placement in the [start, end) range of vma. We might
+		 * need to relook at this in the future.
+		 */
+		return is_device_private_page(page);
+	} else {
+		struct xe_bo *bo = xe_vma_bo(vma);
+		return bo && (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo));
+	}
+}
+
 /**
  * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address
  * range.
@@ -604,8 +627,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, u64 start, u64 end,
 {
 	struct xe_device *xe = tile_to_xe(tile);
 	struct xe_bo *bo = xe_vma_bo(vma);
-	bool is_devmem = !xe_vma_is_userptr(vma) && bo &&
-		(xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo));
+	bool is_devmem = xe_vma_is_devmem(vma, start);
 	struct xe_res_cursor curs;
 	struct xe_pt_stage_bind_walk xe_walk = {
 		.base = {
Signature · 1 line
2.26.3
author
#40 lore raw

PATCH 41/42 drm/xe/svm: Introduce hmm_pfn array based resource cursor

In reply to Oak Zeng

This type of resource cursor will be used by system allocator or userptr. With system allocator, all the backing resource are backed by struct page. The resource could be in system memory or in gpu device memory. For userptr, the backing resource is always in system memory.

For system memory, the page is already dma-mapped. The dma-mapped address is in dma_addr array.

For gpu device memory, we will calculate the device physical address using hmm_pfn.

Note we support a mixture placement of system memory and device memory. In the resource range, some of the page could be backed by system mem, and some by device memory.

Cc Thomas Hellström <[email protected]>
Cc Matthew Brost <[email protected]>
Cc Brian Welty <[email protected]>
Cc Himal Prasad Ghimiray <[email protected]>
---
 drivers/gpu/drm/xe/xe_pt.c         |  5 +-
 drivers/gpu/drm/xe/xe_res_cursor.h | 80 ++++++++++++++++++++----------
 2 files changed, 57 insertions(+), 28 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +3 -2
index c68ff02a931e..4a7dab64f9b6 100644
@@ -698,8 +698,9 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, u64 start, u64 end,
 		u64 offset = start - xe_vma_start(vma);
 		u64 page_idx = offset >> PAGE_SHIFT;
 		if (xe_vma_is_userptr(vma))
-			xe_res_first_dma(to_userptr_vma(vma)->userptr.hmmptr.dma_addr + page_idx,
-					0, xe_vma_size(vma), 0, &curs);
+			xe_res_first_hmmptr(to_userptr_vma(vma)->userptr.hmmptr.dma_addr + page_idx,
+					to_userptr_vma(vma)->userptr.hmmptr.pfn + page_idx,
+					&tile->mem.vram.drm_mr, size, &curs);
 		else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
 			xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma) + offset,
 				     size, &curs);
drivers/gpu/drm/xe/xe_res_cursor.h +54 -26

146 lines not on this page · raw message

Signature · 1 line
2.26.3
author
#41 lore raw

PATCH 42/42 drm/xe: Enable system allocator uAPI

In reply to Oak Zeng

From: Matthew Brost <[email protected]>

---
 drivers/gpu/drm/xe/xe_vm.c | 6 ------
 1 file changed, 6 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +0 -6
index afb7d78a2260..9f2384cccaa8 100644
@@ -2871,12 +2871,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe,
 		u16 pat_index = (*bind_ops)[i].pat_index;
 		u16 coh_mode;
 
-		/* FIXME: Disabling system allocator for now */
-		if (XE_IOCTL_DBG(xe, is_system_allocator)) {
-			err = -EOPNOTSUPP;
-			goto free_bind_ops;
-		}
-
 		if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) {
 			err = -EINVAL;
 			goto free_bind_ops;
Signature · 1 line
2.26.3
#42 lore raw

✓ CI.Patch_applied: success for series starting with [CI,01/42] mm/hmm: let users to tag specific PFNs

In reply to Oak Zeng
== Series Details ==

== Series Details ==

Series: series starting with [CI,01/42] mm/hmm: let users to tag specific PFNs
URL : https://patchwork.freedesktop.org/series/134827/
State : success

== Summary ==

=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: 5123ff6f6fa8 drm-tip: 2024y-06m-13d-15h-38m-17s UTC integration manifest === git am output follows ===
Applying: mm/hmm: let users to tag specific PFNs
Applying: dma-mapping: provide an interface to allocate IOVA
Applying: dma-mapping: provide callbacks to link/unlink pages to specific IOVA
Applying: iommu/dma: Provide an interface to allow preallocate IOVA
Applying: iommu/dma: Prepare map/unmap page functions to receive IOVA
Applying: iommu/dma: Implement link/unlink page callbacks
Applying: drm: Move GPUVA_START/LAST to drm_gpuvm.h
Applying: drm/svm: Mark drm_gpuvm to participate SVM
Applying: drm/svm: introduce drm_mem_region concept
Applying: drm/svm: introduce hmmptr and helper functions
Applying: drm/svm: Introduce helper to remap drm memory region
Applying: drm/svm: handle CPU page fault
Applying: drm/svm: Migrate a range of hmmptr to vram
Applying: drm/svm: Add DRM SVM documentation
Applying: drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
Applying: drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
Applying: drm/xe: Convert multiple bind ops into single job
Applying: drm/xe: Update VM trace events
Applying: drm/xe: Update PT layer with better error handling
Applying: drm/xe: Retry BO allocation
Applying: drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag
Applying: drm/xe: Add a helper to calculate userptr end address
Applying: drm/xe: Add dma_addr res cursor
Applying: drm/xe: Use drm_mem_region for xe
Applying: drm/xe: use drm_hmmptr in xe
Applying: drm/xe: Moving to range based vma invalidation
Applying: drm/xe: Support range based page table update
Applying: drm/xe/uapi: Add DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM flag
Applying: drm/xe/svm: Create userptr if page fault occurs on system_allocator VMA
Applying: drm/xe/svm: Add faulted userptr VMA garbage collector
Applying: drm/xe: Introduce helper to get tile from memory region
Applying: drm/xe/svm: implement functions to allocate and free device memory
Applying: drm/xe/svm: Get drm device from drm memory region
Applying: drm/xe/svm: Get page map owner of a memory region
Applying: drm/xe/svm: Add migrate layer functions for SVM support
Applying: drm/xe/svm: introduce svm migration function
Applying: drm/xe/svm: Register xe memory region to drm layer
Applying: drm/xe/svm: Introduce DRM_XE_SVM kernel config
Applying: drm/xe/svm: Migration from sram to vram for system allocator
Applying: drm/xe/svm: Determine a vma is backed by device memory
Applying: drm/xe/svm: Introduce hmm_pfn array based resource cursor
Applying: drm/xe: Enable system allocator uAPI

#43 lore raw

✗ CI.checkpatch: warning for series starting with [CI,01/42] mm/hmm: let users to tag specific PFNs

In reply to Oak Zeng
== Series Details ==

== Series Details ==

Series: series starting with [CI,01/42] mm/hmm: let users to tag specific PFNs
URL : https://patchwork.freedesktop.org/series/134827/
State : warning

== Summary ==

+ KERNEL=/kernel
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools mt Cloning into 'mt'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ git -C mt rev-list -n1 origin/master 51ce9f6cd981d42d7467409d7dbc559a450abc1e
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ git log -n1
commit 28ce9a77c37b1b30615dacc125fe66c34a0c7f73

Author: Matthew Brost <[email protected]>
Date:   Thu Jun 13 11:31:28 2024 -0400

drm/xe: Enable system allocator uAPI

+ /mt/dim checkpatch 5123ff6f6fa8412bfba48083f334cbdbb8606cfc drm-intel a4d780325e9c mm/hmm: let users to tag specific PFNs
a8a2b4de2931 dma-mapping: provide an interface to allocate IOVA
-:73: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #73: FILE: include/linux/dma-mapping.h:170:
+}
+static inline void dma_free_iova(struct dma_iova_attrs *iova)

total: 0 errors, 0 warnings, 1 checks, 83 lines checked
d113d4e1ff2f dma-mapping: provide callbacks to link/unlink pages to specific IOVA -:59: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #59: FILE: include/linux/dma-mapping.h:176:
}
+static inline dma_addr_t dma_link_range(struct page *page, unsigned long offset,

-:65: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #65: FILE: include/linux/dma-mapping.h:182:
+}
+static inline void dma_unlink_range(struct dma_iova_attrs *iova,

total: 0 errors, 0 warnings, 2 checks, 123 lines checked a0387f44ee49 iommu/dma: Provide an interface to allow preallocate IOVA -:31: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #31: FILE: drivers/iommu/dma-iommu.c:762:
+static dma_addr_t __iommu_dma_alloc_iova(struct iommu_domain *domain,
size_t size, u64 dma_limit, struct device *dev)

-:40: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #40: FILE: drivers/iommu/dma-iommu.c:808:
+static void __iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)

total: 0 errors, 0 warnings, 2 checks, 132 lines checked 7bef42751853 iommu/dma: Prepare map/unmap page functions to receive IOVA -:197: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #197: FILE: drivers/iommu/dma-iommu.c:1658:
+ *handle = __iommu_dma_map(dev, page_to_phys(page), 0, size, ioprot,
dev->coherent_dma_mask);

total: 0 errors, 0 warnings, 1 checks, 171 lines checked 03edd554cea2 iommu/dma: Implement link/unlink page callbacks 7569e7e8fb21 drm: Move GPUVA_START/LAST to drm_gpuvm.h
-:41: CHECK:MACRO_ARG_REUSE: Macro argument reuse 'node' - possible side-effects? #41: FILE: include/drm/drm_gpuvm.h:42:
+#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)

-:42: CHECK:MACRO_ARG_REUSE: Macro argument reuse 'node' - possible side-effects? #42: FILE: include/drm/drm_gpuvm.h:43:
+#define GPUVA_END(node) ((node)->va.addr + (node)->va.range)

total: 0 errors, 0 warnings, 2 checks, 19 lines checked
28afa6fd6682 drm/svm: Mark drm_gpuvm to participate SVM
bc7fc6d26fb8 drm/svm: introduce drm_mem_region concept
-:45: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #45:
new file mode 100644

-:50: WARNING:SPDX_LICENSE_TAG: Improper SPDX comment style for 'include/drm/drm_svm.h', please use '/*' instead #50: FILE: include/drm/drm_svm.h:1:
+// SPDX-License-Identifier: MIT

-:50: WARNING:SPDX_LICENSE_TAG: Missing or malformed SPDX-License-Identifier tag in line 1 #50: FILE: include/drm/drm_svm.h:1:
+// SPDX-License-Identifier: MIT

-:89: ERROR:FLEXIBLE_ARRAY: Use C99 flexible arrays - see https://docs.kernel.org/process/deprecated.html#zero-length-and-one-element-arrays
#89: FILE: include/drm/drm_svm.h:40:
+ } addr_vec[1];
+};

-:103: WARNING:SPACING: Unnecessary space before function pointer arguments #103: FILE: include/drm/drm_svm.h:54:
+ int (*drm_mem_region_alloc_pages) (struct drm_mem_region *mr,

-:104: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #104: FILE: include/drm/drm_svm.h:55:
+ int (*drm_mem_region_alloc_pages) (struct drm_mem_region *mr,
+ unsigned long npags, unsigned long *pfns);

-:109: WARNING:SPACING: Unnecessary space before function pointer arguments #109: FILE: include/drm/drm_svm.h:60:
+ void (*drm_mem_region_free_page) (struct page *page);

-:120: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #120: FILE: include/drm/drm_svm.h:71:
+ struct dma_fence* (*drm_mem_region_migrate)(struct migrate_vec *src_vec,
+ struct migrate_vec *dst_vec);

-:203: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #203: FILE: include/drm/drm_svm.h:154:
+ BUG_ON(mr != drm_page_to_mem_region(page));

-:204: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #204: FILE: include/drm/drm_svm.h:155:
+ BUG_ON((pfn << PAGE_SHIFT) < mr->hpa_base );

-:204: ERROR:SPACING: space prohibited before that close parenthesis ')' #204: FILE: include/drm/drm_svm.h:155:
+ BUG_ON((pfn << PAGE_SHIFT) < mr->hpa_base );

-:205: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #205: FILE: include/drm/drm_svm.h:156:
+ BUG_ON((pfn << PAGE_SHIFT) >= mr->hpa_base + mr->usable_size);

total: 2 errors, 8 warnings, 2 checks, 162 lines checked
8e96ad0e06bf drm/svm: introduce hmmptr and helper functions
Traceback (most recent call last):
File "scripts/spdxcheck.py", line 6, in <module>
from ply import lex, yacc
ModuleNotFoundError: No module named 'ply'
-:68: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #68:
new file mode 100644

-:78: CHECK:LINE_SPACING: Please don't use multiple blank lines #78: FILE: drivers/gpu/drm/drm_svm.c:6:
+
+

-:167: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #167: FILE: drivers/gpu/drm/drm_svm.c:95:
+ DRM_MM_BUG_ON(page_idx + npages > tpages);

-:198: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #198: FILE: drivers/gpu/drm/drm_svm.c:126:
+ DRM_MM_BUG_ON(page_idx + npages > tpages);

-:201: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #201: FILE: drivers/gpu/drm/drm_svm.c:129:
+ DRM_MM_BUG_ON(!page);

-:209: WARNING:LONG_LINE: line length of 102 exceeds 100 columns #209: FILE: drivers/gpu/drm/drm_svm.c:137:
+ hmmptr->dma_addr[i] = dma_link_range(page, 0, &hmmptr->iova, i << PAGE_SHIFT);

-:221: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #221: FILE: drivers/gpu/drm/drm_svm.c:149:
+int drm_svm_hmmptr_init(struct drm_hmmptr *hmmptr,
+ const struct mmu_interval_notifier_ops *ops)

-:254: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #254: FILE: drivers/gpu/drm/drm_svm.c:182:
+ ret = mmu_interval_notifier_insert(&hmmptr->notifier, current->mm,
+ cpu_va_start, end - start, ops);

-:327: ERROR:CODE_INDENT: code indent should use tabs where possible #327: FILE: drivers/gpu/drm/drm_svm.c:255:
+ bool write, bool is_mmap_locked)$

-:327: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #327: FILE: drivers/gpu/drm/drm_svm.c:255:
+int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, u64 end,
+ bool write, bool is_mmap_locked)

-:327: WARNING:LEADING_SPACE: please, no spaces at the start of a line #327: FILE: drivers/gpu/drm/drm_svm.c:255:
+ bool write, bool is_mmap_locked)$

-:336: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #336: FILE: drivers/gpu/drm/drm_svm.c:264:
+ DRM_MM_BUG_ON(start < __hmmptr_cpu_start(hmmptr));

-:337: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #337: FILE: drivers/gpu/drm/drm_svm.c:265:
+ DRM_MM_BUG_ON(end > __hmmptr_cpu_end(hmmptr));

-:403: CHECK:LINE_SPACING: Please don't use multiple blank lines #403: FILE: include/drm/drm_svm.h:16:

+

-:459: WARNING:SPACING: Unnecessary space before function pointer arguments #459: FILE: include/drm/drm_svm.h:214:
+ struct drm_gpuva * (*get_gpuva) (struct drm_hmmptr *hmmptr);

-:463: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #463: FILE: include/drm/drm_svm.h:218:
+int drm_svm_hmmptr_init(struct drm_hmmptr *hmmptr,
+ const struct mmu_interval_notifier_ops *ops);

-:468: ERROR:CODE_INDENT: code indent should use tabs where possible #468: FILE: include/drm/drm_svm.h:223:
+ bool write, bool is_mmap_locked);$

-:468: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #468: FILE: include/drm/drm_svm.h:223:
+int drm_svm_hmmptr_populate(struct drm_hmmptr *hmmptr, void *owner, u64 start, u64 end,
+ bool write, bool is_mmap_locked);

-:468: WARNING:LEADING_SPACE: please, no spaces at the start of a line #468: FILE: include/drm/drm_svm.h:223:
+ bool write, bool is_mmap_locked);$

total: 2 errors, 10 warnings, 7 checks, 405 lines checked 19ba94de0e8a drm/svm: Introduce helper to remap drm memory region -:77: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #77: FILE: drivers/gpu/drm/drm_svm.c:333:
+int drm_svm_register_mem_region(const struct drm_device *drm,
+ struct drm_mem_region *mr, enum memory_type type)

-:107: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #107: FILE: drivers/gpu/drm/drm_svm.c:363:
+ drm_err(drm, "Failed to remap memory region %p, errno %d\n",
+ mr, ret);

-:113: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #113: FILE: drivers/gpu/drm/drm_svm.c:369:
+ drm_info(drm, "Registered device memory [%llx-%llx] to devm, remapped to %pr\n",
+ mr->dpa_base, mr->dpa_base + mr->usable_size, res);

-:126: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #126: FILE: include/drm/drm_svm.h:168:
+int drm_svm_register_mem_region(const struct drm_device *drm,
+ struct drm_mem_region *mr, enum memory_type type);

total: 0 errors, 0 warnings, 4 checks, 74 lines checked b28543b64bc9 drm/svm: handle CPU page fault
-:78: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #78: FILE: drivers/gpu/drm/drm_svm.c:351:
+static int __drm_svm_alloc_host_pages(struct vm_area_struct *vma,
+ unsigned long addr,

-:103: WARNING:LONG_LINE: line length of 118 exceeds 100 columns
#103: FILE: drivers/gpu/drm/drm_svm.c:376:
+static struct migrate_vec *__generate_migrate_vec_vram(unsigned long *mpfn, bool is_migrate_src, unsigned long npages)

-:112: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #112: FILE: drivers/gpu/drm/drm_svm.c:385:
+ BUG_ON(is_migrate_src && npages != 1);

-:118: ERROR:SPACING: space required before the open parenthesis '(' #118: FILE: drivers/gpu/drm/drm_svm.c:391:
+ for(i = 0, j = 0; i < npages; i++) {

-:150: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #150: FILE: drivers/gpu/drm/drm_svm.c:423:
+static struct migrate_vec *__generate_migrate_vec_sram(struct device *dev, unsigned long *mpfn,
+ bool is_migrate_src, unsigned long npages)

-:167: ERROR:SPACING: spaces required around that '=' (ctx:WxV) #167: FILE: drivers/gpu/drm/drm_svm.c:440:
+ for(i = 0, k =0 ; i < npages; i++) {
^

-:167: ERROR:SPACING: space required before the open parenthesis '(' #167: FILE: drivers/gpu/drm/drm_svm.c:440:
+ for(i = 0, k =0 ; i < npages; i++) {

-:197: WARNING:LONG_LINE: line length of 101 exceeds 100 columns
#197: FILE: drivers/gpu/drm/drm_svm.c:470:
+static void __free_migrate_vec_sram(struct device *dev, struct migrate_vec *vec, bool is_migrate_src)

-:202: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #202: FILE: drivers/gpu/drm/drm_svm.c:475:
+ BUG_ON(vec->mr != NULL);

-:202: CHECK:COMPARISON_TO_NULL: Comparison to NULL could be written "vec->mr" #202: FILE: drivers/gpu/drm/drm_svm.c:475:
+ BUG_ON(vec->mr != NULL);

-:213: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #213: FILE: drivers/gpu/drm/drm_svm.c:486:
+ BUG_ON(vec->mr == NULL);

-:213: CHECK:COMPARISON_TO_NULL: Comparison to NULL could be written "!vec->mr" #213: FILE: drivers/gpu/drm/drm_svm.c:486:
+ BUG_ON(vec->mr == NULL);

-:217: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #217: FILE: drivers/gpu/drm/drm_svm.c:490:
+}
+/**

-:314: CHECK:LINE_SPACING: Please use a blank line after function/struct/union/enum declarations #314: FILE: drivers/gpu/drm/drm_svm.c:587:
+}
+static struct dev_pagemap_ops drm_devm_pagemap_ops = {

total: 3 errors, 5 warnings, 6 checks, 284 lines checked a05b3df36749 drm/svm: Migrate a range of hmmptr to vram -:39: ERROR:SPACING: space required before the open parenthesis '(' #39: FILE: drivers/gpu/drm/drm_svm.c:649:
+ for(i = 0; i < npages; i++) {

-:59: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #59: FILE: drivers/gpu/drm/drm_svm.c:669:
+int drm_svm_migrate_hmmptr_to_vram(struct drm_gpuvm *vm,
+ struct drm_mem_region *mr,

-:81: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #81: FILE: drivers/gpu/drm/drm_svm.c:691:
+ BUG_ON(start < __hmmptr_cpu_start(hmmptr));

-:82: WARNING:AVOID_BUG: Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants #82: FILE: drivers/gpu/drm/drm_svm.c:692:
+ BUG_ON(end > __hmmptr_cpu_end(hmmptr));

-:89: CHECK:SPACING: spaces preferred around that '*' (ctx:VxW) #89: FILE: drivers/gpu/drm/drm_svm.c:699:
+ buf = kvcalloc(npages, 2* sizeof(*migrate.src), GFP_KERNEL);
^

-:90: ERROR:SPACING: space required before the open parenthesis '(' #90: FILE: drivers/gpu/drm/drm_svm.c:700:
+ if(!buf)

-:98: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #98: FILE: drivers/gpu/drm/drm_svm.c:708:
+ drm_warn(drm, "vma setup returned %d for range [0x%lx - 0x%lx]\n",
+ ret, start, end);

-:108: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #108: FILE: drivers/gpu/drm/drm_svm.c:718:
+ drm_warn(drm, "Partial migration for range [0x%lx - 0x%lx], range is %ld pages, migrate only %ld pages\n",
+ start, end, npages, migrate.cpages);

-:147: CHECK:SPACING: spaces preferred around that '*' (ctx:VxV) #147: FILE: drivers/gpu/drm/drm_svm.c:757:
+ memset(migrate.dst, 0, sizeof(*migrate.dst)*migrate.cpages);
^

-:163: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #163: FILE: include/drm/drm_svm.h:228:
+int drm_svm_migrate_hmmptr_to_vram(struct drm_gpuvm *vm,
+ struct drm_mem_region *mr,

total: 2 errors, 2 warnings, 6 checks, 131 lines checked
91dd00878f8f drm/svm: Add DRM SVM documentation
-:230: WARNING:TYPO_SPELLING: 'delibrately' may be misspelled - perhaps 'deliberately'? #230: FILE: drivers/gpu/drm/drm_svm.c:147:
+ * in DRM SVM design. We delibrately choose this implementation option to achieve true

                          ^^^^^^^^^^^

-:328: WARNING:LONG_LINE_COMMENT: line length of 121 exceeds 100 columns #328: FILE: drivers/gpu/drm/drm_svm.c:245:
+ * In GPU page fault handler: munmap from user:

-:331: WARNING:LONG_LINE_COMMENT: line length of 121 exceeds 100 columns #331: FILE: drivers/gpu/drm/drm_svm.c:248:
+ * Mmap_read_lock() mmap_write_lock()

-:332: WARNING:LONG_LINE_COMMENT: line length of 168 exceeds 100 columns #332: FILE: drivers/gpu/drm/drm_svm.c:249:
+ * call drm_svm_migrate_hmmptr_to_vram to migrate hmmptr if needed mmu_notifier_invalidate_range_start(), call hmmptr->invalidate()

-:333: WARNING:LONG_LINE_COMMENT: line length of 136 exceeds 100 columns #333: FILE: drivers/gpu/drm/drm_svm.c:250:
+ * hmmptr.notifier_seq = mmu_interval_read_begin(&hmmptr.notifier) down_write(page_table_lock);

-:334: WARNING:LONG_LINE_COMMENT: line length of 131 exceeds 100 columns #334: FILE: drivers/gpu/drm/drm_svm.c:251:
+ * call drm_svm_populate_hmmptr to populate hmmptr mmu_interval_set_seq();

-:335: WARNING:LONG_LINE_COMMENT: line length of 145 exceeds 100 columns #335: FILE: drivers/gpu/drm/drm_svm.c:252:
+ * Mmap_read_unlock invalidate hmmptr from GPU page table

-:336: WARNING:LONG_LINE_COMMENT: line length of 134 exceeds 100 columns #336: FILE: drivers/gpu/drm/drm_svm.c:253:
+ * up_write(page_table_lock);

-:337: WARNING:LONG_LINE_COMMENT: line length of 139 exceeds 100 columns #337: FILE: drivers/gpu/drm/drm_svm.c:254:
+ * dma_resv_lock() mmu_notifier_invalidate_range_end()

-:338: WARNING:LONG_LINE_COMMENT: line length of 126 exceeds 100 columns #338: FILE: drivers/gpu/drm/drm_svm.c:255:
+ * down_read(page_table_lock); mmap_write_unlock();

total: 0 errors, 10 warnings, 0 checks, 360 lines checked
887afa1becac drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue f3a443042d9d drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
5e0824b6daab drm/xe: Convert multiple bind ops into single job
-:13: WARNING:TYPO_SPELLING: 'implemenation' may be misspelled - perhaps 'implementation'? #13:
The implemenation is roughly:

    ^^^^^^^^^^^^^

-:885: WARNING:SUSPECT_CODE_INDENT: suspect code indent for conditional statements (8, 15) #885: FILE: drivers/gpu/drm/xe/xe_pt.c:1163:
+ if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))
+ return 0;

-:886: ERROR:CODE_INDENT: code indent should use tabs where possible #886: FILE: drivers/gpu/drm/xe/xe_pt.c:1164:
+ return 0;$

-:886: WARNING:LEADING_SPACE: please, no spaces at the start of a line #886: FILE: drivers/gpu/drm/xe/xe_pt.c:1164:
+ return 0;$

-:1667: WARNING:BLOCK_COMMENT_STYLE: Block comments should align the * on each line #1667: FILE: drivers/gpu/drm/xe/xe_pt.c:1900:
+ * installing job fence in various places.
+ *

-:1668: WARNING:BLOCK_COMMENT_STYLE: Block comments should align the * on each line #1668: FILE: drivers/gpu/drm/xe/xe_pt.c:1901:
+ *
+ * Return: fence on success, negative ERR_PTR on error.

-:1800: WARNING:LEADING_SPACE: please, no spaces at the start of a line #1800: FILE: drivers/gpu/drm/xe/xe_pt.c:2033:
+ }$

total: 1 errors, 6 warnings, 0 checks, 2532 lines checked
f5626f039d13 drm/xe: Update VM trace events
d175a9d167a8 drm/xe: Update PT layer with better error handling
1027d80af60e drm/xe: Retry BO allocation
408a6fdce1bd drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR flag
-:41: WARNING:COMMIT_LOG_LONG_LINE: Prefer a maximum 75 chars per line (possible unwrapped commit description?) #41:
FIXME: Not enforcing DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR for fault mode VMs

total: 0 errors, 1 warnings, 0 checks, 452 lines checked a319b7ff5216 drm/xe: Add a helper to calculate userptr end address d7a5c8de3cf8 drm/xe: Add dma_addr res cursor
-:7: WARNING:COMMIT_MESSAGE: Missing commit description - Add an appropriate one

total: 0 errors, 1 warnings, 0 checks, 85 lines checked 9f484e8bb0b9 drm/xe: Use drm_mem_region for xe
-:222: WARNING:LONG_LINE: line length of 134 exceeds 100 columns #222: FILE: drivers/gpu/drm/xe/xe_vram.c:343:
+ tile->id, &tile->mem.vram.actual_physical_size, &tile->mem.vram.drm_mr.usable_size, &tile->mem.vram.io_size);

-:225: WARNING:LONG_LINE: line length of 132 exceeds 100 columns #225: FILE: drivers/gpu/drm/xe/xe_vram.c:345:
+ &tile->mem.vram.drm_mr.dpa_base, tile->mem.vram.drm_mr.dpa_base + (u64)tile->mem.vram.actual_physical_size,

total: 0 errors, 2 warnings, 0 checks, 154 lines checked
7d46b962e5f1 drm/xe: use drm_hmmptr in xe
-:47: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #47:
deleted file mode 100644

-:333: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #333: FILE: drivers/gpu/drm/xe/xe_pt.c:673:
+ xe_res_first_dma(to_userptr_vma(vma)->userptr.hmmptr.dma_addr,
+ 0, xe_vma_size(vma), 0, &curs);

-:385: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #385: FILE: drivers/gpu/drm/xe/xe_vm.c:67:
+static void xe_vma_userptr_dma_map_pages(struct xe_userptr_vma *uvma,
+ u64 start, u64 end)

-:401: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #401: FILE: drivers/gpu/drm/xe/xe_vm.c:83:
+static void xe_vma_userptr_dma_unmap_pages(struct xe_userptr_vma *uvma,
+ u64 start, u64 end)

-:430: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #430: FILE: drivers/gpu/drm/xe/xe_vm.c:111:
+ ret = drm_svm_hmmptr_populate(hmmptr, NULL, xe_vma_userptr(vma),
+ xe_vma_userptr(vma) + xe_vma_size(vma),

-:492: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #492: FILE: drivers/gpu/drm/xe/xe_vm.c:1040:
+ err = drm_svm_hmmptr_init(&userptr->hmmptr,
+ &vma_userptr_notifier_ops);

total: 0 errors, 1 warnings, 5 checks, 226 lines checked 24570b775152 drm/xe: Moving to range based vma invalidation -:13: WARNING:REPEATED_WORD: Possible repeated word: 'more' #13:
instead of the whole vma. This more more proper than the whole vma based

-:197: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #197: FILE: drivers/gpu/drm/xe/xe_vm.c:735:
+ err = xe_vm_invalidate_vma(&uvma->vma, xe_vma_start(&uvma->vma),
+ xe_vma_end(&uvma->vma));

-:251: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #251: FILE: drivers/gpu/drm/xe/xe_vm.c:3274:
+ seqno[id] = xe_gt_tlb_invalidation_range(tile->primary_gt, NULL,
+ start, end, xe_vma_vm(vma)->usm.asid);

total: 0 errors, 1 warnings, 2 checks, 212 lines checked 500a167fc8bd drm/xe: Support range based page table update -:39: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #39: FILE: drivers/gpu/drm/xe/xe_gt_pagefault.c:158:
+ fence = xe_vma_rebind(vm, vma, xe_vma_start(vma),
+ xe_vma_end(vma), BIT(tile->id));

-:93: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #93: FILE: drivers/gpu/drm/xe/xe_pt.c:680:
+ xe_res_first_dma(to_userptr_vma(vma)->userptr.hmmptr.dma_addr + page_idx,
0, xe_vma_size(vma), 0, &curs);

-:207: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #207: FILE: drivers/gpu/drm/xe/xe_pt.c:1817:
+ err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma,
+ op->base.map.va.addr, op->base.map.va.addr + op->base.map.va.range);

-:218: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #218: FILE: drivers/gpu/drm/xe/xe_pt.c:1831:
+ err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.prev,
+ xe_vma_start(op->remap.prev), xe_vma_end(op->remap.prev));

-:225: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #225: FILE: drivers/gpu/drm/xe/xe_pt.c:1836:
+ err = bind_op_prepare(vm, tile, pt_update_ops, op->remap.next,
+ xe_vma_start(op->remap.next), xe_vma_end(op->remap.next));

-:234: WARNING:LONG_LINE: line length of 104 exceeds 100 columns #234: FILE: drivers/gpu/drm/xe/xe_pt.c:1858:
+ err = bind_op_prepare(vm, tile, pt_update_ops, vma, xe_vma_start(vma), xe_vma_end(vma));

-:297: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #297: FILE: drivers/gpu/drm/xe/xe_vm.c:899:
+struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma,
+ u64 start, u64 end, u8 tile_mask)

total: 0 errors, 1 warnings, 6 checks, 262 lines checked
6ccf6d98bd2b drm/xe/uapi: Add DRM_XE_VM_CREATE_FLAG_PARTICIPATE_SVM flag 168dd82b81c5 drm/xe/svm: Create userptr if page fault occurs on system_allocator VMA -:63: ERROR:CODE_INDENT: code indent should use tabs where possible #63: FILE: drivers/gpu/drm/xe/xe_vm.c:1485:
+^I vm_resv_obj, 0, vm->size, 0, 0, &gpuvm_ops,$

-:64: ERROR:CODE_INDENT: code indent should use tabs where possible #64: FILE: drivers/gpu/drm/xe/xe_vm.c:1486:
+^I flags & XE_VM_FLAG_PARTICIPATE_SVM);$

total: 2 errors, 0 warnings, 0 checks, 189 lines checked
14c9f70c0f2c drm/xe/svm: Add faulted userptr VMA garbage collector
8569229e452d drm/xe: Introduce helper to get tile from memory region
20a936859d18 drm/xe/svm: implement functions to allocate and free device memory Traceback (most recent call last):
File "scripts/spdxcheck.py", line 6, in <module>
from ply import lex, yacc
ModuleNotFoundError: No module named 'ply'
-:55: WARNING:FILE_PATH_CHANGES: added, moved or deleted file(s), does MAINTAINERS need updating? #55:
new file mode 100644

-:78: ERROR:CODE_INDENT: code indent should use tabs where possible #78: FILE: drivers/gpu/drm/xe/xe_svm.c:19:
+ u64 end, struct list_head *block_list)$

-:78: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #78: FILE: drivers/gpu/drm/xe/xe_svm.c:19:
+static void drm_buddy_block_free_subrange(struct drm_buddy_block *block, u64 start,
+ u64 end, struct list_head *block_list)

-:78: WARNING:LEADING_SPACE: please, no spaces at the start of a line #78: FILE: drivers/gpu/drm/xe/xe_svm.c:19:
+ u64 end, struct list_head *block_list)$

-:110: ERROR:CODE_INDENT: code indent should use tabs where possible #110: FILE: drivers/gpu/drm/xe/xe_svm.c:51:
+ (page_idx << PAGE_SHIFT) + PAGE_SIZE, &blocks);$

-:110: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #110: FILE: drivers/gpu/drm/xe/xe_svm.c:51:
+ drm_buddy_block_free_subrange(block, page_idx << PAGE_SHIFT,
+ (page_idx << PAGE_SHIFT) + PAGE_SIZE, &blocks);

-:110: WARNING:LEADING_SPACE: please, no spaces at the start of a line #110: FILE: drivers/gpu/drm/xe/xe_svm.c:51:
+ (page_idx << PAGE_SHIFT) + PAGE_SIZE, &blocks);$

-:121: ERROR:SPACING: space required before the open parenthesis '(' #121: FILE: drivers/gpu/drm/xe/xe_svm.c:62:
+ for(i = 0; i < pages_per_block; i++) {

-:161: ERROR:CODE_INDENT: code indent should use tabs where possible #161: FILE: drivers/gpu/drm/xe/xe_svm.c:102:
+ unsigned long npages, unsigned long *pfn)$

-:161: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #161: FILE: drivers/gpu/drm/xe/xe_svm.c:102:
+int xe_svm_alloc_pages(struct drm_mem_region *mr,
+ unsigned long npages, unsigned long *pfn)

-:161: WARNING:LEADING_SPACE: please, no spaces at the start of a line #161: FILE: drivers/gpu/drm/xe/xe_svm.c:102:
+ unsigned long npages, unsigned long *pfn)$

-:173: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #173: FILE: drivers/gpu/drm/xe/xe_svm.c:114:
+ ret = drm_buddy_alloc_blocks(mm, 0, mm->size, size, PAGE_SIZE,
+ &blocks, DRM_BUDDY_CONTIGUOUS_ALLOCATION);

-:195: ERROR:SPACING: space required before the open parenthesis '(' #195: FILE: drivers/gpu/drm/xe/xe_svm.c:136:
+ for(i = 0; i < pages_per_block; i++) {

-:214: WARNING:SPDX_LICENSE_TAG: Improper SPDX comment style for 'drivers/gpu/drm/xe/xe_svm.h', please use '/*' instead #214: FILE: drivers/gpu/drm/xe/xe_svm.h:1:
+// SPDX-License-Identifier: MIT

-:214: WARNING:SPDX_LICENSE_TAG: Missing or malformed SPDX-License-Identifier tag in line 1 #214: FILE: drivers/gpu/drm/xe/xe_svm.h:1:
+// SPDX-License-Identifier: MIT

-:226: ERROR:CODE_INDENT: code indent should use tabs where possible #226: FILE: drivers/gpu/drm/xe/xe_svm.h:13:
+ unsigned long npages, unsigned long *pfn);$

-:226: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #226: FILE: drivers/gpu/drm/xe/xe_svm.h:13:
+int xe_svm_alloc_pages(struct drm_mem_region *mr,
+ unsigned long npages, unsigned long *pfn);

-:226: WARNING:LEADING_SPACE: please, no spaces at the start of a line #226: FILE: drivers/gpu/drm/xe/xe_svm.h:13:
+ unsigned long npages, unsigned long *pfn);$

total: 6 errors, 7 warnings, 5 checks, 178 lines checked a455a192bf00 drm/xe/svm: Get drm device from drm memory region 1f18df852352 drm/xe/svm: Get page map owner of a memory region -:27: ERROR:POINTER_LOCATION: "foo* bar" should be "foo *bar" #27: FILE: drivers/gpu/drm/xe/xe_svm.c:150:
+void* xe_svm_mem_region_to_pagemap_owner(struct drm_mem_region *mr)

-:42: ERROR:POINTER_LOCATION: "foo* bar" should be "foo *bar" #42: FILE: drivers/gpu/drm/xe/xe_svm.h:15:
+void* xe_svm_mem_region_to_pagemap_owner(struct drm_mem_region *mr);

total: 2 errors, 0 warnings, 0 checks, 17 lines checked 8329f276f707 drm/xe/svm: Add migrate layer functions for SVM support -:84: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #84: FILE: drivers/gpu/drm/xe/xe_migrate.c:1513:
+struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
+ unsigned long npages,

-:170: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #170: FILE: drivers/gpu/drm/xe/xe_migrate.h:120:
+struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
+ unsigned long npages,

total: 0 errors, 0 warnings, 2 checks, 141 lines checked 3f166ec25fe8 drm/xe/svm: introduce svm migration function -:55: ERROR:POINTER_LOCATION: "foo* bar" should be "foo *bar" #55: FILE: drivers/gpu/drm/xe/xe_svm.c:176:
+struct dma_fence* xe_svm_migrate(struct migrate_vec *src_vec,

-:56: ERROR:CODE_INDENT: code indent should use tabs where possible #56: FILE: drivers/gpu/drm/xe/xe_svm.c:177:
+ struct migrate_vec *dst_vec)$

-:56: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #56: FILE: drivers/gpu/drm/xe/xe_svm.c:177:
+struct dma_fence* xe_svm_migrate(struct migrate_vec *src_vec,
+ struct migrate_vec *dst_vec)

-:56: WARNING:LEADING_SPACE: please, no spaces at the start of a line #56: FILE: drivers/gpu/drm/xe/xe_svm.c:177:
+ struct migrate_vec *dst_vec)$

-:96: ERROR:POINTER_LOCATION: "foo* bar" should be "foo *bar" #96: FILE: drivers/gpu/drm/xe/xe_svm.h:17:
+struct dma_fence* xe_svm_migrate(struct migrate_vec *src_vec,

-:97: ERROR:CODE_INDENT: code indent should use tabs where possible #97: FILE: drivers/gpu/drm/xe/xe_svm.h:18:
+ struct migrate_vec *dst_vec);$

-:97: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #97: FILE: drivers/gpu/drm/xe/xe_svm.h:18:
+struct dma_fence* xe_svm_migrate(struct migrate_vec *src_vec,
+ struct migrate_vec *dst_vec);

-:97: WARNING:LEADING_SPACE: please, no spaces at the start of a line #97: FILE: drivers/gpu/drm/xe/xe_svm.h:18:
+ struct migrate_vec *dst_vec);$

total: 4 errors, 2 warnings, 2 checks, 70 lines checked 3c1c3ee38334 drm/xe/svm: Register xe memory region to drm layer ebd84ad329fe drm/xe/svm: Introduce DRM_XE_SVM kernel config e909cca629fe drm/xe/svm: Migration from sram to vram for system allocator -:11: WARNING:TYPO_SPELLING: 'splitted' may be misspelled - perhaps 'split'? #11:
created during fault (aka userptr splitted from system allocator vma,

                                  ^^^^^^^^

-:77: ERROR:CODE_INDENT: code indent should use tabs where possible #77: FILE: drivers/gpu/drm/xe/xe_gt_pagefault.c:180:
+ hmmptr, cpu_va_start, cpu_va_end);$

-:77: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #77: FILE: drivers/gpu/drm/xe/xe_gt_pagefault.c:180:
+ err = drm_svm_migrate_hmmptr_to_vram(&vm->gpuvm, &tile->mem.vram.drm_mr,
+ hmmptr, cpu_va_start, cpu_va_end);

-:77: WARNING:LEADING_SPACE: please, no spaces at the start of a line #77: FILE: drivers/gpu/drm/xe/xe_gt_pagefault.c:180:
+ hmmptr, cpu_va_start, cpu_va_end);$

-:84: CHECK:BRACES: Blank lines aren't necessary before a close brace '}' #84: FILE: drivers/gpu/drm/xe/xe_gt_pagefault.c:186:

+ }

-:97: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #97: FILE: drivers/gpu/drm/xe/xe_gt_pagefault.c:206:
+ fence = xe_vma_rebind(vm, vma, gpu_va_start,
+ gpu_va_end, BIT(tile->id));

-:133: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #133: FILE: drivers/gpu/drm/xe/xe_vm.c:114:
+ ret = drm_svm_hmmptr_populate(hmmptr, xe, start, end,
+ !xe_vma_read_only(vma), mmap_locked);

-:148: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #148: FILE: drivers/gpu/drm/xe/xe_vm.c:742:
+ err = xe_vma_userptr_pin_pages(uvma, xe_vma_userptr(&uvma->vma),
+ xe_vma_userptr_end(&uvma->vma), false);

-:158: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #158: FILE: drivers/gpu/drm/xe/xe_vm.c:2123:
+ err = xe_vma_userptr_pin_pages(to_userptr_vma(vma), xe_vma_userptr(vma),
+ xe_vma_userptr_end(vma), false);

total: 1 errors, 2 warnings, 6 checks, 125 lines checked
9d5b44ff08f3 drm/xe/svm: Determine a vma is backed by device memory -:51: WARNING:UNNECESSARY_ELSE: else is not generally useful after a break or return #51: FILE: drivers/gpu/drm/xe/xe_pt.c:598:
+ return is_device_private_page(page);
+ } else {

-:53: WARNING:LINE_SPACING: Missing a blank line after declarations #53: FILE: drivers/gpu/drm/xe/xe_pt.c:600:
+ struct xe_bo *bo = xe_vma_bo(vma);
+ return bo && (xe_bo_is_vram(bo) || xe_bo_is_stolen_devmem(bo));

total: 0 errors, 2 warnings, 0 checks, 44 lines checked 08fbe5f47573 drm/xe/svm: Introduce hmm_pfn array based resource cursor -:41: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #41: FILE: drivers/gpu/drm/xe/xe_pt.c:702:
+ xe_res_first_hmmptr(to_userptr_vma(vma)->userptr.hmmptr.dma_addr + page_idx,
+ to_userptr_vma(vma)->userptr.hmmptr.pfn + page_idx,

-:135: ERROR:CODE_INDENT: code indent should use tabs where possible #135: FILE: drivers/gpu/drm/xe/xe_res_cursor.h:206:
+ unsigned long *hmm_pfn, struct drm_mem_region *mr,$

-:135: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #135: FILE: drivers/gpu/drm/xe/xe_res_cursor.h:206:
+static inline void xe_res_first_hmmptr(dma_addr_t *dma_addr,
+ unsigned long *hmm_pfn, struct drm_mem_region *mr,

-:135: WARNING:LEADING_SPACE: please, no spaces at the start of a line #135: FILE: drivers/gpu/drm/xe/xe_res_cursor.h:206:
+ unsigned long *hmm_pfn, struct drm_mem_region *mr,$

-:191: CHECK:BRACES: braces {} should be used on all arms of this statement #191: FILE: drivers/gpu/drm/xe/xe_res_cursor.h:300:
+ if (cur->mr) {
[...]
else if (cur->sgl)
[...]

total: 1 errors, 1 warnings, 3 checks, 157 lines checked 28ce9a77c37b drm/xe: Enable system allocator uAPI
-:7: WARNING:COMMIT_MESSAGE: Missing commit description - Add an appropriate one

total: 0 errors, 1 warnings, 0 checks, 12 lines checked

#44 lore raw

✗ CI.KUnit: failure for series starting with [CI,01/42] mm/hmm: let users to tag specific PFNs

In reply to Oak Zeng
== Series Details ==

== Series Details ==

Series: series starting with [CI,01/42] mm/hmm: let users to tag specific PFNs
URL : https://patchwork.freedesktop.org/series/134827/
State : failure

== Summary ==

+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig ERROR:root:../drivers/gpu/drm/./drm_svm.c: In function ‘__drm_svm_init_device_pages’:
../drivers/gpu/drm/./drm_svm.c:959:3: error: implicit declaration of function ‘zone_device_page_init’ [-Werror=implicit-function-declaration]

  959 |   zone_device_page_init(page);
      |   ^~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors
make[6]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/./drm_svm.o] Error 1 make[6]: *** Waiting for unfinished jobs....
../lib/iomap.c:156:5: warning: no previous prototype for ‘ioread64_lo_hi’ [-Wmissing-prototypes]
156 | u64 ioread64_lo_hi(const void __iomem *addr)

      |     ^~~~~~~~~~~~~~
../lib/iomap.c:163:5: warning: no previous prototype for ‘ioread64_hi_lo’ [-Wmissing-prototypes]
  163 | u64 ioread64_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~
../lib/iomap.c:170:5: warning: no previous prototype for ‘ioread64be_lo_hi’ [-Wmissing-prototypes]
  170 | u64 ioread64be_lo_hi(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:178:5: warning: no previous prototype for ‘ioread64be_hi_lo’ [-Wmissing-prototypes]
  178 | u64 ioread64be_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:264:6: warning: no previous prototype for ‘iowrite64_lo_hi’ [-Wmissing-prototypes]
  264 | void iowrite64_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:272:6: warning: no previous prototype for ‘iowrite64_hi_lo’ [-Wmissing-prototypes]
  272 | void iowrite64_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:280:6: warning: no previous prototype for ‘iowrite64be_lo_hi’ [-Wmissing-prototypes]
  280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~
../lib/iomap.c:288:6: warning: no previous prototype for ‘iowrite64be_hi_lo’ [-Wmissing-prototypes]
  288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_svm.c: In function ‘xe_svm_alloc_pages’:
../drivers/gpu/drm/xe/xe_svm.c:144:4: error: implicit declaration of function ‘zone_device_page_init’ [-Werror=implicit-function-declaration]
  144 |    zone_device_page_init(page);
      |    ^~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors
make[7]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/xe/xe_svm.o] Error 1 make[7]: *** Waiting for unfinished jobs....
make[6]: *** [../scripts/Makefile.build:485: drivers/gpu/drm/xe] Error 2 make[5]: *** [../scripts/Makefile.build:485: drivers/gpu/drm] Error 2 make[4]: *** [../scripts/Makefile.build:485: drivers/gpu] Error 2 make[3]: *** [../scripts/Makefile.build:485: drivers] Error 2 make[2]: *** [/kernel/Makefile:1934: .] Error 2
make[1]: *** [/kernel/Makefile:240: __sub-make] Error 2
make: *** [Makefile:240: __sub-make] Error 2

[15:54:21] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:

$ make ARCH=um O=.kunit olddefconfig
[15:54:25] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make ARCH=um O=.kunit --jobs=48
+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel