Dynamic drm_pagemaps and Initial multi-device SVM

With the end goal of being able to free unused pagemaps and allocate them on demand, add a refcount to struct drm_pagemap, remove the xe embedded drm_pagemap, allocating and freeing it explicitly.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c      | 51 ++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_svm.c        | 26 ++++++++++-----
 drivers/gpu/drm/xe/xe_vram_types.h |  2 +-
 include/drm/drm_pagemap.h          | 36 +++++++++++++++++++++
 4 files changed, 106 insertions(+), 9 deletions(-)

drivers/gpu/drm/drm_pagemap.c +51 -0

index 22c44807e3fe..4b8692f0b2a2 100644
@@ -538,6 +538,57 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
 	return -ENOMEM;
 }
 
+static void drm_pagemap_release(struct kref *ref)
+{
+	struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref);
+
+	kfree(dpagemap);
+}
+
+/**
+ * drm_pagemap_create() - Create a struct drm_pagemap.
+ * @dev: Pointer to a struct device providing the device-private memory.
+ * @pagemap: Pointer to a pre-setup struct dev_pagemap providing the struct pages.
+ * @ops: Pointer to the struct drm_pagemap_ops.
+ *
+ * Allocate and initialize a struct drm_pagemap.
+ *
+ * Return: A refcounted pointer to a struct drm_pagemap on success.
+ * Error pointer on error.
+ */
+struct drm_pagemap *
+drm_pagemap_create(struct device *dev,
+		   struct dev_pagemap *pagemap,
+		   const struct drm_pagemap_ops *ops)
+{
+	struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap), GFP_KERNEL);
+
+	if (!dpagemap)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&dpagemap->ref);
+	dpagemap->dev = dev;
+	dpagemap->ops = ops;
+	dpagemap->pagemap = pagemap;
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_create);
+
+/**
+ * drm_pagemap_put() - Put a struct drm_pagemap reference
+ * @dpagemap: Pointer to a struct drm_pagemap object.
+ *
+ * Puts a struct drm_pagemap reference and frees the drm_pagemap object
+ * if the refount reaches zero.
+ */
+void drm_pagemap_put(struct drm_pagemap *dpagemap)
+{
+	if (likely(dpagemap))
+		kref_put(&dpagemap->ref, drm_pagemap_release);
+}
+EXPORT_SYMBOL(drm_pagemap_put);
+
 /**
  * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM
  * @devmem_allocation: Pointer to the device memory allocation

drivers/gpu/drm/xe/xe_svm.c +18 -8

index 129e7818565c..6d2c6c144315 100644
@@ -861,7 +861,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 				      struct mm_struct *mm,
 				      unsigned long timeslice_ms)
 {
-	struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap);
+	struct xe_vram_region *vr = container_of(dpagemap->pagemap, typeof(*vr), pagemap);
 	struct xe_device *xe = vr->xe;
 	struct device *dev = xe->drm.dev;
 	struct drm_buddy_block *block;

@@ -1372,7 +1372,7 @@ u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end)
 
 static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile)
 {
-	return &tile->mem.vram->dpagemap;
+	return tile->mem.vram->dpagemap;
 }
 
 /**

@@ -1482,6 +1482,15 @@ int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr)
 		return ret;
 	}
 
+	vr->dpagemap = drm_pagemap_create(dev, &vr->pagemap,
+					  &xe_drm_pagemap_ops);
+	if (IS_ERR(vr->dpagemap)) {
+		drm_err(&xe->drm, "Failed to create drm_pagemap tile %d memory: %pe\n",
+			tile->id, vr->dpagemap);
+		ret = PTR_ERR(vr->dpagemap);
+		goto out_no_dpagemap;
+	}
+
 	vr->pagemap.type = MEMORY_DEVICE_PRIVATE;
 	vr->pagemap.range.start = res->start;
 	vr->pagemap.range.end = res->end;

@@ -1489,22 +1498,23 @@ int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr)
 	vr->pagemap.ops = drm_pagemap_pagemap_ops_get();
 	vr->pagemap.owner = xe_svm_devm_owner(xe);
 	addr = devm_memremap_pages(dev, &vr->pagemap);
-
-	vr->dpagemap.dev = dev;
-	vr->dpagemap.ops = &xe_drm_pagemap_ops;
-
 	if (IS_ERR(addr)) {
-		devm_release_mem_region(dev, res->start, resource_size(res));
 		ret = PTR_ERR(addr);
 		drm_err(&xe->drm, "Failed to remap tile %d memory, errno %pe\n",
 			tile->id, ERR_PTR(ret));
-		return ret;
+		goto out_failed_memremap;
 	}
 	vr->hpa_base = res->start;
 
 	drm_dbg(&xe->drm, "Added tile %d memory [%llx-%llx] to devm, remapped to %pr\n",
 		tile->id, vr->io_start, vr->io_start + vr->usable_size, res);
 	return 0;
+
+out_failed_memremap:
+	drm_pagemap_put(vr->dpagemap);
+out_no_dpagemap:
+	devm_release_mem_region(dev, res->start, resource_size(res));
+	return ret;
 }
 #else
 int xe_svm_alloc_vram(struct xe_tile *tile,

drivers/gpu/drm/xe/xe_vram_types.h +1 -1

index 83772dcbf1af..c0d2c5ee8c10 100644
@@ -72,7 +72,7 @@ struct xe_vram_region {
 	 * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE memory
 	 * pages of this tile.
 	 */
-	struct drm_pagemap dpagemap;
+	struct drm_pagemap *dpagemap;
 	/**
 	 * @hpa_base: base host physical address
 	 *

include/drm/drm_pagemap.h +36 -0

index f6e7e234c089..2c7de928865b 100644
@@ -129,11 +129,15 @@ struct drm_pagemap_ops {
  * struct drm_pagemap: Additional information for a struct dev_pagemap
  * used for device p2p handshaking.
  * @ops: The struct drm_pagemap_ops.
+ * @ref: Reference count.
  * @dev: The struct drevice owning the device-private memory.
+ * @pagemap: Pointer to the underlying dev_pagemap.
  */
 struct drm_pagemap {
 	const struct drm_pagemap_ops *ops;
+	struct kref ref;
 	struct device *dev;
+	struct dev_pagemap *pagemap;
 };
 
 struct drm_pagemap_devmem;

@@ -202,6 +206,37 @@ struct drm_pagemap_devmem_ops {
 			   unsigned long npages);
 };
 
+struct drm_pagemap *drm_pagemap_create(struct device *dev,
+				       struct dev_pagemap *pagemap,
+				       const struct drm_pagemap_ops *ops);
+
+#if IS_ENABLED(CONFIG_DRM_GPUSVM)
+
+void drm_pagemap_put(struct drm_pagemap *dpagemap);
+
+#else
+
+static inline void drm_pagemap_put(struct drm_pagemap *dpagemap)
+{
+}
+
+#endif /* IS_ENABLED(CONFIG_DRM_GPUSVM) */
+
+/**
+ * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap
+ * @dpagemap: Pointer to the struct drm_pagemap.
+ *
+ * Return: Pointer to the struct drm_pagemap.
+ */
+static inline struct drm_pagemap *
+drm_pagemap_get(struct drm_pagemap *dpagemap)
+{
+	if (likely(dpagemap))
+		kref_get(&dpagemap->ref);
+
+	return dpagemap;
+}
+
 /**
  * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation
  *

@@ -246,3 +281,4 @@ int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 			    unsigned long timeslice_ms);
 
 #endif
+

Signature · 1 line

2.51.0

With the end goal of being able to free unused pagemaps and allocate them on demand, add a refcount to struct drm_pagemap, remove the xe embedded drm_pagemap, allocating and freeing it explicitly.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c      | 51 ++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_svm.c        | 26 ++++++++++-----
 drivers/gpu/drm/xe/xe_vram_types.h |  2 +-
 include/drm/drm_pagemap.h          | 36 +++++++++++++++++++++
 4 files changed, 106 insertions(+), 9 deletions(-)

drivers/gpu/drm/drm_pagemap.c +51 -0

index 22c44807e3fe..4b8692f0b2a2 100644
@@ -538,6 +538,57 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
 	return -ENOMEM;
 }
 
+static void drm_pagemap_release(struct kref *ref)
+{
+	struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref);
+
+	kfree(dpagemap);
+}
+
+/**
+ * drm_pagemap_create() - Create a struct drm_pagemap.
+ * @dev: Pointer to a struct device providing the device-private memory.
+ * @pagemap: Pointer to a pre-setup struct dev_pagemap providing the struct pages.
+ * @ops: Pointer to the struct drm_pagemap_ops.
+ *
+ * Allocate and initialize a struct drm_pagemap.
+ *
+ * Return: A refcounted pointer to a struct drm_pagemap on success.
+ * Error pointer on error.
+ */
+struct drm_pagemap *
+drm_pagemap_create(struct device *dev,
+		   struct dev_pagemap *pagemap,
+		   const struct drm_pagemap_ops *ops)
+{
+	struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap), GFP_KERNEL);
+
+	if (!dpagemap)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&dpagemap->ref);
+	dpagemap->dev = dev;
+	dpagemap->ops = ops;
+	dpagemap->pagemap = pagemap;
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_create);
+
+/**
+ * drm_pagemap_put() - Put a struct drm_pagemap reference
+ * @dpagemap: Pointer to a struct drm_pagemap object.
+ *
+ * Puts a struct drm_pagemap reference and frees the drm_pagemap object
+ * if the refount reaches zero.
+ */
+void drm_pagemap_put(struct drm_pagemap *dpagemap)
+{
+	if (likely(dpagemap))
+		kref_put(&dpagemap->ref, drm_pagemap_release);
+}
+EXPORT_SYMBOL(drm_pagemap_put);
+
 /**
  * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM
  * @devmem_allocation: Pointer to the device memory allocation

drivers/gpu/drm/xe/xe_svm.c +18 -8

index 129e7818565c..6d2c6c144315 100644
@@ -861,7 +861,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 				      struct mm_struct *mm,
 				      unsigned long timeslice_ms)
 {
-	struct xe_vram_region *vr = container_of(dpagemap, typeof(*vr), dpagemap);
+	struct xe_vram_region *vr = container_of(dpagemap->pagemap, typeof(*vr), pagemap);
 	struct xe_device *xe = vr->xe;
 	struct device *dev = xe->drm.dev;
 	struct drm_buddy_block *block;

@@ -1372,7 +1372,7 @@ u8 xe_svm_ranges_zap_ptes_in_range(struct xe_vm *vm, u64 start, u64 end)
 
 static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile)
 {
-	return &tile->mem.vram->dpagemap;
+	return tile->mem.vram->dpagemap;
 }
 
 /**

@@ -1482,6 +1482,15 @@ int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr)
 		return ret;
 	}
 
+	vr->dpagemap = drm_pagemap_create(dev, &vr->pagemap,
+					  &xe_drm_pagemap_ops);
+	if (IS_ERR(vr->dpagemap)) {
+		drm_err(&xe->drm, "Failed to create drm_pagemap tile %d memory: %pe\n",
+			tile->id, vr->dpagemap);
+		ret = PTR_ERR(vr->dpagemap);
+		goto out_no_dpagemap;
+	}
+
 	vr->pagemap.type = MEMORY_DEVICE_PRIVATE;
 	vr->pagemap.range.start = res->start;
 	vr->pagemap.range.end = res->end;

@@ -1489,22 +1498,23 @@ int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr)
 	vr->pagemap.ops = drm_pagemap_pagemap_ops_get();
 	vr->pagemap.owner = xe_svm_devm_owner(xe);
 	addr = devm_memremap_pages(dev, &vr->pagemap);
-
-	vr->dpagemap.dev = dev;
-	vr->dpagemap.ops = &xe_drm_pagemap_ops;
-
 	if (IS_ERR(addr)) {
-		devm_release_mem_region(dev, res->start, resource_size(res));
 		ret = PTR_ERR(addr);
 		drm_err(&xe->drm, "Failed to remap tile %d memory, errno %pe\n",
 			tile->id, ERR_PTR(ret));
-		return ret;
+		goto out_failed_memremap;
 	}
 	vr->hpa_base = res->start;
 
 	drm_dbg(&xe->drm, "Added tile %d memory [%llx-%llx] to devm, remapped to %pr\n",
 		tile->id, vr->io_start, vr->io_start + vr->usable_size, res);
 	return 0;
+
+out_failed_memremap:
+	drm_pagemap_put(vr->dpagemap);
+out_no_dpagemap:
+	devm_release_mem_region(dev, res->start, resource_size(res));
+	return ret;
 }
 #else
 int xe_svm_alloc_vram(struct xe_tile *tile,

drivers/gpu/drm/xe/xe_vram_types.h +1 -1

index 83772dcbf1af..c0d2c5ee8c10 100644
@@ -72,7 +72,7 @@ struct xe_vram_region {
 	 * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE memory
 	 * pages of this tile.
 	 */
-	struct drm_pagemap dpagemap;
+	struct drm_pagemap *dpagemap;
 	/**
 	 * @hpa_base: base host physical address
 	 *

include/drm/drm_pagemap.h +36 -0

index f6e7e234c089..2c7de928865b 100644
@@ -129,11 +129,15 @@ struct drm_pagemap_ops {
  * struct drm_pagemap: Additional information for a struct dev_pagemap
  * used for device p2p handshaking.
  * @ops: The struct drm_pagemap_ops.
+ * @ref: Reference count.
  * @dev: The struct drevice owning the device-private memory.
+ * @pagemap: Pointer to the underlying dev_pagemap.
  */
 struct drm_pagemap {
 	const struct drm_pagemap_ops *ops;
+	struct kref ref;
 	struct device *dev;
+	struct dev_pagemap *pagemap;
 };
 
 struct drm_pagemap_devmem;

@@ -202,6 +206,37 @@ struct drm_pagemap_devmem_ops {
 			   unsigned long npages);
 };
 
+struct drm_pagemap *drm_pagemap_create(struct device *dev,
+				       struct dev_pagemap *pagemap,
+				       const struct drm_pagemap_ops *ops);
+
+#if IS_ENABLED(CONFIG_DRM_GPUSVM)
+
+void drm_pagemap_put(struct drm_pagemap *dpagemap);
+
+#else
+
+static inline void drm_pagemap_put(struct drm_pagemap *dpagemap)
+{
+}
+
+#endif /* IS_ENABLED(CONFIG_DRM_GPUSVM) */
+
+/**
+ * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap
+ * @dpagemap: Pointer to the struct drm_pagemap.
+ *
+ * Return: Pointer to the struct drm_pagemap.
+ */
+static inline struct drm_pagemap *
+drm_pagemap_get(struct drm_pagemap *dpagemap)
+{
+	if (likely(dpagemap))
+		kref_get(&dpagemap->ref);
+
+	return dpagemap;
+}
+
 /**
  * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation
  *

@@ -246,3 +281,4 @@ int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 			    unsigned long timeslice_ms);
 
 #endif
+

Signature · 1 line

2.51.0

Quoted 5 lines · Thomas Hellström

With the end goal of being able to free unused pagemaps and allocate them on demand, add a refcount to struct drm_pagemap, remove the xe embedded drm_pagemap, allocating and freeing it explicitly.

General commit for the series — could we add some kernel documentation,
ideally in xe_svm.c, that explains the reference counting scheme used for drm_pagemap?

For example:

- An SVM VM holds a drm_pagemap reference to local pagemaps.
- madvise VMAs hold a reference to the preferred location pagemap.
- Allocated device pages hold a reference to the pagemap.
- The pagemap itself holds a reference to the device/module.

Reference counting schemes can be difficult to reverse-engineer and easy
to forget, so it would be best to document them clearly.

Matt

Quoted 247 lines

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c      | 51
++++++++++++++++++++++++++++++
 drivers/gpu/drm/xe/xe_svm.c        | 26 ++++++++++----- drivers/gpu/drm/xe/xe_vram_types.h |  2 +- include/drm/drm_pagemap.h          | 36 +++++++++++++++++++++

 4 files changed, 106 insertions(+), 9 deletions(-)

drivers/gpu/drm/drm_pagemap.c +0 -0

index 22c44807e3fe..4b8692f0b2a2 100644
@@ -538,6 +538,57 @@ static int
 drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
  	return -ENOMEM;
  }

drivers/gpu/drm/drm_pagemap.c +3 -0

+static void drm_pagemap_release(struct kref *ref)
+{
+	struct drm_pagemap *dpagemap = container_of(ref,

typeof(*dpagemap), ref);

drivers/gpu/drm/drm_pagemap.c +7 -0

+
+	kfree(dpagemap);
+}
+
+/**
+ * drm_pagemap_create() - Create a struct drm_pagemap.
+ * @dev: Pointer to a struct device providing the device-private

memory.
+ * @pagemap: Pointer to a pre-setup struct dev_pagemap providing the struct pages.
+ * @ops: Pointer to the struct drm_pagemap_ops.
+ *
+ * Allocate and initialize a struct drm_pagemap.
+ *
+ * Return: A refcounted pointer to a struct drm_pagemap on success.
+ * Error pointer on error.
+ */

drivers/gpu/drm/drm_pagemap.c +6 -0

+struct drm_pagemap *
+drm_pagemap_create(struct device *dev,
+		   struct dev_pagemap *pagemap,
+		   const struct drm_pagemap_ops *ops)
+{
+	struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap),

GFP_KERNEL);

drivers/gpu/drm/drm_pagemap.c +18 -0

+
+	if (!dpagemap)
+		return ERR_PTR(-ENOMEM);
+
+	kref_init(&dpagemap->ref);
+	dpagemap->dev = dev;
+	dpagemap->ops = ops;
+	dpagemap->pagemap = pagemap;
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_create);
+
+/**
+ * drm_pagemap_put() - Put a struct drm_pagemap reference
+ * @dpagemap: Pointer to a struct drm_pagemap object.
+ *
+ * Puts a struct drm_pagemap reference and frees the drm_pagemap

object
+ * if the refount reaches zero.
+ */

drivers/gpu/drm/drm_pagemap.c +7 -0

+void drm_pagemap_put(struct drm_pagemap *dpagemap)
+{
+	if (likely(dpagemap))
+		kref_put(&dpagemap->ref, drm_pagemap_release);
+}
+EXPORT_SYMBOL(drm_pagemap_put);
+

 /**
  * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM
  * @devmem_allocation: Pointer to the device memory allocation

drivers/gpu/drm/xe/xe_svm.c +1 -1

index 129e7818565c..6d2c6c144315 100644
@@ -861,7 +861,7 @@ static int xe_drm_pagemap_populate_mm(struct
 drm_pagemap *dpagemap,
  				      struct mm_struct *mm,
  				      unsigned long timeslice_ms)
  {
-	struct xe_vram_region *vr = container_of(dpagemap,
 typeof(*vr), dpagemap);
+	struct xe_vram_region *vr = container_of(dpagemap-
 >pagemap, typeof(*vr), pagemap);

  struct xe_device *xe = vr->xe;
  struct device *dev = xe->drm.dev;
  struct drm_buddy_block *block;

drivers/gpu/drm/xe/xe_svm.c +0 -0

@@ -1372,7 +1372,7 @@ u8 xe_svm_ranges_zap_ptes_in_range(struct
 xe_vm *vm, u64 start, u64 end)

 static struct drm_pagemap *tile_local_pagemap(struct xe_tile *tile)
 {

drivers/gpu/drm/xe/xe_svm.c +1 -1

-	return &tile->mem.vram->dpagemap;
+	return tile->mem.vram->dpagemap;

 }

 /**

drivers/gpu/drm/xe/xe_svm.c +0 -0

@@ -1482,6 +1482,15 @@ int xe_devm_add(struct xe_tile *tile, struct
 xe_vram_region *vr)
  		return ret;
  	}

drivers/gpu/drm/xe/xe_svm.c +4 -0

+	vr->dpagemap = drm_pagemap_create(dev, &vr->pagemap,
+					  &xe_drm_pagemap_ops);
+	if (IS_ERR(vr->dpagemap)) {
+		drm_err(&xe->drm, "Failed to create drm_pagemap

tile %d memory: %pe\n",

drivers/gpu/drm/xe/xe_svm.c +5 -0

+			tile->id, vr->dpagemap);
+		ret = PTR_ERR(vr->dpagemap);
+		goto out_no_dpagemap;
+	}
+

  vr->pagemap.type = MEMORY_DEVICE_PRIVATE;
  vr->pagemap.range.start = res->start;
  vr->pagemap.range.end = res->end;

drivers/gpu/drm/xe/xe_svm.c +1 -6

@@ -1489,22 +1498,23 @@ int xe_devm_add(struct xe_tile *tile,
 struct xe_vram_region *vr)
  	vr->pagemap.ops = drm_pagemap_pagemap_ops_get();
  	vr->pagemap.owner = xe_svm_devm_owner(xe);
  	addr = devm_memremap_pages(dev, &vr->pagemap);
-
-	vr->dpagemap.dev = dev;
-	vr->dpagemap.ops = &xe_drm_pagemap_ops;
-
  	if (IS_ERR(addr)) {
-		devm_release_mem_region(dev, res->start,
 resource_size(res));
  		ret = PTR_ERR(addr);
  		drm_err(&xe->drm, "Failed to remap tile %d memory,
 errno %pe\n",
  			tile->id, ERR_PTR(ret));
-		return ret;
+		goto out_failed_memremap;
  	}
  	vr->hpa_base = res->start;

  drm_dbg(&xe->drm, "Added tile %d memory [%llx-%llx] to devm, remapped to %pr\n",
  tile->id, vr->io_start, vr->io_start + vr-

Quoted 1 line

usable_size, res);

  return 0;

drivers/gpu/drm/xe/xe_svm.c +5 -0

+
+out_failed_memremap:
+	drm_pagemap_put(vr->dpagemap);
+out_no_dpagemap:
+	devm_release_mem_region(dev, res->start,

resource_size(res));

drivers/gpu/drm/xe/xe_svm.c +1 -0

+	return ret;

 }
 #else
 int xe_svm_alloc_vram(struct xe_tile *tile,

drivers/gpu/drm/xe/xe_vram_types.h +0 -0

index 83772dcbf1af..c0d2c5ee8c10 100644
@@ -72,7 +72,7 @@ struct xe_vram_region {
  	 * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE

memory
  * pages of this tile.
  */

drivers/gpu/drm/xe/xe_vram_types.h +1 -1

-	struct drm_pagemap dpagemap;
+	struct drm_pagemap *dpagemap;

  /**
  * @hpa_base: base host physical address
  *

include/drm/drm_pagemap.h +0 -0

index f6e7e234c089..2c7de928865b 100644
@@ -129,11 +129,15 @@ struct drm_pagemap_ops {
   * struct drm_pagemap: Additional information for a struct

dev_pagemap
  * used for device p2p handshaking.
  * @ops: The struct drm_pagemap_ops.
+ * @ref: Reference count.
  * @dev: The struct drevice owning the device-private memory.
+ * @pagemap: Pointer to the underlying dev_pagemap. */
 struct drm_pagemap {
  const struct drm_pagemap_ops *ops;

include/drm/drm_pagemap.h +1 -0

+	struct kref ref;

  struct device *dev;

include/drm/drm_pagemap.h +1 -0

+	struct dev_pagemap *pagemap;

 };

 struct drm_pagemap_devmem;

include/drm/drm_pagemap.h +0 -0

@@ -202,6 +206,37 @@ struct drm_pagemap_devmem_ops {
  			   unsigned long npages);
  };

include/drm/drm_pagemap.h +2 -0

+struct drm_pagemap *drm_pagemap_create(struct device *dev,
+				       struct dev_pagemap

*pagemap,

include/drm/drm_pagemap.h +1 -0

+				       const struct

drm_pagemap_ops *ops);

include/drm/drm_pagemap.h +28 -0

+
+#if IS_ENABLED(CONFIG_DRM_GPUSVM)
+
+void drm_pagemap_put(struct drm_pagemap *dpagemap);
+
+#else
+
+static inline void drm_pagemap_put(struct drm_pagemap *dpagemap)
+{
+}
+
+#endif /* IS_ENABLED(CONFIG_DRM_GPUSVM) */
+
+/**
+ * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap
+ * @dpagemap: Pointer to the struct drm_pagemap.
+ *
+ * Return: Pointer to the struct drm_pagemap.
+ */
+static inline struct drm_pagemap *
+drm_pagemap_get(struct drm_pagemap *dpagemap)
+{
+	if (likely(dpagemap))
+		kref_get(&dpagemap->ref);
+
+	return dpagemap;
+}
+

 /**
  * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation
  *

include/drm/drm_pagemap.h +0 -0

@@ -246,3 +281,4 @@ int drm_pagemap_populate_mm(struct drm_pagemap
 *dpagemap,
  			    unsigned long timeslice_ms);

 #endif
+

Signature · 1 line

2.51.0

To be able to keep track of drm_pagemap usage, add a refcounted backpointer to struct drm_pagemap_zdd. This will keep the drm_pagemap reference count from dropping to zero as long as there are drm_pagemap pages present in a CPU address space.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

drivers/gpu/drm/drm_pagemap.c +8 -2

index 4b8692f0b2a2..173b3ecb07d5 100644
@@ -62,6 +62,7 @@
  *
  * @refcount: Reference count for the zdd
  * @devmem_allocation: device memory allocation
+ * @dpagemap: Refcounted pointer to the underlying struct drm_pagemap.
  * @device_private_page_owner: Device private pages owner
  *
  * This structure serves as a generic wrapper installed in

@@ -74,11 +75,13 @@
 struct drm_pagemap_zdd {
 	struct kref refcount;
 	struct drm_pagemap_devmem *devmem_allocation;
+	struct drm_pagemap *dpagemap;
 	void *device_private_page_owner;
 };
 
 /**
  * drm_pagemap_zdd_alloc() - Allocate a zdd structure.
+ * @dpagemap: Pointer to the underlying struct drm_pagemap.
  * @device_private_page_owner: Device private pages owner
  *
  * This function allocates and initializes a new zdd structure. It sets up the

@@ -87,7 +90,7 @@ struct drm_pagemap_zdd {
  * Return: Pointer to the allocated zdd on success, ERR_PTR() on failure.
  */
 static struct drm_pagemap_zdd *
-drm_pagemap_zdd_alloc(void *device_private_page_owner)
+drm_pagemap_zdd_alloc(struct drm_pagemap *dpagemap, void *device_private_page_owner)
 {
 	struct drm_pagemap_zdd *zdd;
 

@@ -98,6 +101,7 @@ drm_pagemap_zdd_alloc(void *device_private_page_owner)
 	kref_init(&zdd->refcount);
 	zdd->devmem_allocation = NULL;
 	zdd->device_private_page_owner = device_private_page_owner;
+	zdd->dpagemap = drm_pagemap_get(dpagemap);
 
 	return zdd;
 }

@@ -127,6 +131,7 @@ static void drm_pagemap_zdd_destroy(struct kref *ref)
 	struct drm_pagemap_zdd *zdd =
 		container_of(ref, struct drm_pagemap_zdd, refcount);
 	struct drm_pagemap_devmem *devmem = zdd->devmem_allocation;
+	struct drm_pagemap *dpagemap = zdd->dpagemap;
 
 	if (devmem) {
 		complete_all(&devmem->detached);

@@ -134,6 +139,7 @@ static void drm_pagemap_zdd_destroy(struct kref *ref)
 			devmem->ops->devmem_release(devmem);
 	}
 	kfree(zdd);
+	drm_pagemap_put(dpagemap);
 }
 
 /**

@@ -366,7 +372,7 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 	pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages);
 	pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages;
 
-	zdd = drm_pagemap_zdd_alloc(pgmap_owner);
+	zdd = drm_pagemap_zdd_alloc(devmem_allocation->dpagemap, pgmap_owner);
 	if (!zdd) {
 		err = -ENOMEM;
 		goto err_free;

Signature · 1 line

2.51.0

Even if the drm_pagemap provider has released its reference on the drm_pagemap, references may be held by still active pages. Ensure that we hold a reference on the provider drm device and modules for as long as we might need to use the drm_pagemap ops.

Note that in theory, the drm_gpusvm_helper module may be unloaded as soon as the final module_put() of the provider driver module is executed, so we need to add a module_exit() function that waits for the work item executing the module_put() has completed.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c | 101 ++++++++++++++++++++++++++++++++--
 drivers/gpu/drm/xe/xe_svm.c   |  15 ++++-
 include/drm/drm_pagemap.h     |  10 +++-
 3 files changed, 117 insertions(+), 9 deletions(-)

drivers/gpu/drm/drm_pagemap.c +97 -4

index 173b3ecb07d5..fb18a80d6a1c 100644
@@ -8,6 +8,7 @@
 #include <linux/pagemap.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_pagemap.h>
+#include <drm/drm_print.h>
 
 /**
  * DOC: Overview

@@ -544,16 +545,92 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
 	return -ENOMEM;
 }
 
+static void drm_pagemap_dev_unhold_work(struct work_struct *work);
+static LLIST_HEAD(drm_pagemap_unhold_list);
+static DECLARE_WORK(drm_pagemap_work, drm_pagemap_dev_unhold_work);
+
+/**
+ * struct drm_pagemap_dev_hold - Struct to aid in drm_device release.
+ * @link: Link into drm_pagemap_unhold_list for deferred reference releases.
+ * @drm: drm device to put.
+ *
+ * When a struct drm_pagemap is released, we also need to release the
+ * reference it holds on the drm device. However, typically that needs
+ * to be done separately from a system-wide workqueue.
+ * Each time a struct drm_pagemap is initialized
+ * (or re-initialized if cached) therefore allocate a separate
+ * drm_pagemap_dev_hold item, from which we put the drm device and
+ * associated module.
+ */
+struct drm_pagemap_dev_hold {
+	struct llist_node link;
+	struct drm_device *drm;
+};
+
 static void drm_pagemap_release(struct kref *ref)
 {
 	struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref);
-
+	struct drm_pagemap_dev_hold *dev_hold = dpagemap->dev_hold;
+
+	/*
+	 * We know the pagemap provider is alive at this point, since
+	 * the struct drm_pagemap_dev_hold holds a reference to the
+	 * pagemap provider drm_device and its module.
+	 */
+	dpagemap->dev_hold = NULL;
 	kfree(dpagemap);
+	llist_add(&dev_hold->link, &drm_pagemap_unhold_list);
+	schedule_work(&drm_pagemap_work);
+	/*
+	 * Here, either the provider device is still alive, since if called from
+	 * page_free(), the caller is holding a reference on the dev_pagemap,
+	 * or if called from drm_pagemap_put(), the direct caller is still alive.
+	 * This ensures we can't race with THIS module unload.
+	 */
+}
+
+static void drm_pagemap_dev_unhold_work(struct work_struct *work)
+{
+	struct llist_node *node = llist_del_all(&drm_pagemap_unhold_list);
+	struct drm_pagemap_dev_hold *dev_hold, *next;
+
+	/*
+	 * Deferred release of drm_pagemap provider device and module.
+	 * THIS module is kept alive during the release by the
+	 * flush_work() in the drm_pagemap_exit() function.
+	 */
+	llist_for_each_entry_safe(dev_hold, next, node, link) {
+		struct drm_device *drm = dev_hold->drm;
+		struct module *module = drm->driver->fops->owner;
+
+		drm_dbg(drm, "Releasing reference on provider device and module.\n");
+		drm_dev_put(drm);
+		module_put(module);
+		kfree(dev_hold);
+	}
+}
+
+static struct drm_pagemap_dev_hold *
+drm_pagemap_dev_hold(struct drm_pagemap *dpagemap)
+{
+	struct drm_pagemap_dev_hold *dev_hold;
+	struct drm_device *drm = dpagemap->drm;
+
+	dev_hold = kzalloc(sizeof(*dev_hold), GFP_KERNEL);
+	if (!dev_hold)
+		return ERR_PTR(-ENOMEM);
+
+	init_llist_node(&dev_hold->link);
+	dev_hold->drm = drm;
+	(void)try_module_get(drm->driver->fops->owner);
+	drm_dev_get(drm);
+
+	return dev_hold;
 }
 
 /**
  * drm_pagemap_create() - Create a struct drm_pagemap.
- * @dev: Pointer to a struct device providing the device-private memory.
+ * @drm: Pointer to a struct drm_device providing the device-private memory.
  * @pagemap: Pointer to a pre-setup struct dev_pagemap providing the struct pages.
  * @ops: Pointer to the struct drm_pagemap_ops.
  *

@@ -563,20 +640,28 @@ static void drm_pagemap_release(struct kref *ref)
  * Error pointer on error.
  */
 struct drm_pagemap *
-drm_pagemap_create(struct device *dev,
+drm_pagemap_create(struct drm_device *drm,
 		   struct dev_pagemap *pagemap,
 		   const struct drm_pagemap_ops *ops)
 {
 	struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap), GFP_KERNEL);
+	struct drm_pagemap_dev_hold *dev_hold;
 
 	if (!dpagemap)
 		return ERR_PTR(-ENOMEM);
 
 	kref_init(&dpagemap->ref);
-	dpagemap->dev = dev;
+	dpagemap->drm = drm;
 	dpagemap->ops = ops;
 	dpagemap->pagemap = pagemap;
 
+	dev_hold = drm_pagemap_dev_hold(dpagemap);
+	if (IS_ERR(dev_hold)) {
+		kfree(dpagemap);
+		return ERR_CAST(dev_hold);
+	}
+	dpagemap->dev_hold = dev_hold;
+
 	return dpagemap;
 }
 EXPORT_SYMBOL(drm_pagemap_create);

@@ -937,3 +1022,11 @@ int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 	return err;
 }
 EXPORT_SYMBOL(drm_pagemap_populate_mm);
+
+static void drm_pagemap_exit(void)
+{
+	flush_work(&drm_pagemap_work);
+	if (WARN_ON(!llist_empty(&drm_pagemap_unhold_list)))
+		disable_work_sync(&drm_pagemap_work);
+}
+module_exit(drm_pagemap_exit);

drivers/gpu/drm/xe/xe_svm.c +10 -2

index 6d2c6c144315..f6ee22da2e95 100644
@@ -1437,7 +1437,7 @@ xe_drm_pagemap_device_map(struct drm_pagemap *dpagemap,
 			  unsigned int order,
 			  enum dma_data_direction dir)
 {
-	struct device *pgmap_dev = dpagemap->dev;
+	struct device *pgmap_dev = dpagemap->drm->dev;
 	enum drm_interconnect_protocol prot;
 	dma_addr_t addr;
 

@@ -1457,6 +1457,14 @@ static const struct drm_pagemap_ops xe_drm_pagemap_ops = {
 	.populate_mm = xe_drm_pagemap_populate_mm,
 };
 
+static void xe_devm_release(void *data)
+{
+	struct xe_vram_region *vr = data;
+
+	drm_pagemap_put(vr->dpagemap);
+	vr->dpagemap = NULL;
+}
+
 /**
  * xe_devm_add: Remap and provide memmap backing for device memory
  * @tile: tile that the memory region belongs to

@@ -1482,7 +1490,7 @@ int xe_devm_add(struct xe_tile *tile, struct xe_vram_region *vr)
 		return ret;
 	}
 
-	vr->dpagemap = drm_pagemap_create(dev, &vr->pagemap,
+	vr->dpagemap = drm_pagemap_create(&xe->drm, &vr->pagemap,
 					  &xe_drm_pagemap_ops);
 	if (IS_ERR(vr->dpagemap)) {
 		drm_err(&xe->drm, "Failed to create drm_pagemap tile %d memory: %pe\n",

drivers/gpu/drm/xe/xe_svm.c +0 -0

 	vr->pagemap.type = MEMORY_DEVICE_PRIVATE;
 	vr->pagemap.range.start = res->start;

include/drm/drm_pagemap.h +7 -3

index 2c7de928865b..5cfe54331ba7 100644
@@ -9,6 +9,7 @@
 #define NR_PAGES(order) (1U << (order))
 
 struct drm_pagemap;
+struct drm_pagemap_dev_hold;
 struct drm_pagemap_zdd;
 struct device;
 

@@ -130,14 +131,17 @@ struct drm_pagemap_ops {
  * used for device p2p handshaking.
  * @ops: The struct drm_pagemap_ops.
  * @ref: Reference count.
- * @dev: The struct drevice owning the device-private memory.
+ * @drm: The struct drm device owning the device-private memory.
  * @pagemap: Pointer to the underlying dev_pagemap.
+ * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for
+ * device referencing.
  */
 struct drm_pagemap {
 	const struct drm_pagemap_ops *ops;
 	struct kref ref;
-	struct device *dev;
+	struct drm_device *drm;
 	struct dev_pagemap *pagemap;
+	struct drm_pagemap_dev_hold *dev_hold;
 };
 
 struct drm_pagemap_devmem;

@@ -206,7 +210,7 @@ struct drm_pagemap_devmem_ops {
 			   unsigned long npages);
 };
 
-struct drm_pagemap *drm_pagemap_create(struct device *dev,
+struct drm_pagemap *drm_pagemap_create(struct drm_device *drm,
 				       struct dev_pagemap *pagemap,
 				       const struct drm_pagemap_ops *ops);
 

Signature · 1 line

2.51.0

Quoted 5 lines · Thomas Hellström

Even if the drm_pagemap provider has released its reference on the drm_pagemap, references may be held by still active pages. Ensure that we hold a reference on the provider drm device and modules for as long as we might need to use the drm_pagemap ops.

Just to make sure I’m understanding this correctly — this is intended to
guard against the devm action [1] running while a device is still holding references to another device’s pages, right?

[1]
https://elixir.bootlin.com/linux/v6.17.5/source/kernel/resource.c#L1993

Quoted 230 lines

Note that in theory, the drm_gpusvm_helper module may be unloaded as soon as the final module_put() of the provider driver module is executed, so we need to add a module_exit() function that waits for the work item executing the module_put() has completed.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c | 101
++++++++++++++++++++++++++++++++--
 drivers/gpu/drm/xe/xe_svm.c   |  15 ++++-
 include/drm/drm_pagemap.h     |  10 +++-

 3 files changed, 117 insertions(+), 9 deletions(-)

drivers/gpu/drm/drm_pagemap.c +1 -0

index 173b3ecb07d5..fb18a80d6a1c 100644
@@ -8,6 +8,7 @@
  #include <linux/pagemap.h>
  #include <drm/drm_drv.h>
  #include <drm/drm_pagemap.h>
+#include <drm/drm_print.h>

 /**
  * DOC: Overview

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -544,16 +545,92 @@ static int
 drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
  	return -ENOMEM;
  }

drivers/gpu/drm/drm_pagemap.c +3 -0

+static void drm_pagemap_dev_unhold_work(struct work_struct *work);
+static LLIST_HEAD(drm_pagemap_unhold_list);
+static DECLARE_WORK(drm_pagemap_work,

drm_pagemap_dev_unhold_work);

drivers/gpu/drm/drm_pagemap.c +3 -0

+
+/**
+ * struct drm_pagemap_dev_hold - Struct to aid in drm_device

release.
+ * @link: Link into drm_pagemap_unhold_list for deferred reference releases.
+ * @drm: drm device to put.
+ *
+ * When a struct drm_pagemap is released, we also need to release the
+ * reference it holds on the drm device. However, typically that needs
+ * to be done separately from a system-wide workqueue.
+ * Each time a struct drm_pagemap is initialized
+ * (or re-initialized if cached) therefore allocate a separate
+ * drm_pagemap_dev_hold item, from which we put the drm device and
+ * associated module.
+ */

drivers/gpu/drm/drm_pagemap.c +5 -0

+struct drm_pagemap_dev_hold {
+	struct llist_node link;
+	struct drm_device *drm;
+};
+

 static void drm_pagemap_release(struct kref *ref) {
  struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref);

drivers/gpu/drm/drm_pagemap.c +1 -1

-
+	struct drm_pagemap_dev_hold *dev_hold = dpagemap-

Quoted 1 line

dev_hold;

drivers/gpu/drm/drm_pagemap.c +3 -0

+
+	/*
+	 * We know the pagemap provider is alive at this point,

since

drivers/gpu/drm/drm_pagemap.c +1 -0

+	 * the struct drm_pagemap_dev_hold holds a reference to

the

drivers/gpu/drm/drm_pagemap.c +3 -0

+	 * pagemap provider drm_device and its module.
+	 */
+	dpagemap->dev_hold = NULL;

  kfree(dpagemap);

drivers/gpu/drm/drm_pagemap.c +4 -0

+	llist_add(&dev_hold->link, &drm_pagemap_unhold_list);
+	schedule_work(&drm_pagemap_work);
+	/*
+	 * Here, either the provider device is still alive, since

if called from

drivers/gpu/drm/drm_pagemap.c +1 -0

+	 * page_free(), the caller is holding a reference on the

dev_pagemap,

drivers/gpu/drm/drm_pagemap.c +1 -0

+	 * or if called from drm_pagemap_put(), the direct caller

is still alive.

drivers/gpu/drm/drm_pagemap.c +7 -0

+	 * This ensures we can't race with THIS module unload.
+	 */
+}
+
+static void drm_pagemap_dev_unhold_work(struct work_struct *work)
+{
+	struct llist_node *node =

llist_del_all(&drm_pagemap_unhold_list);

drivers/gpu/drm/drm_pagemap.c +4 -0

+	struct drm_pagemap_dev_hold *dev_hold, *next;
+
+	/*
+	 * Deferred release of drm_pagemap provider device and

module.

drivers/gpu/drm/drm_pagemap.c +8 -0

+	 * THIS module is kept alive during the release by the
+	 * flush_work() in the drm_pagemap_exit() function.
+	 */
+	llist_for_each_entry_safe(dev_hold, next, node, link) {
+		struct drm_device *drm = dev_hold->drm;
+		struct module *module = drm->driver->fops->owner;
+
+		drm_dbg(drm, "Releasing reference on provider

device and module.\n");

drivers/gpu/drm/drm_pagemap.c +22 -0

+		drm_dev_put(drm);
+		module_put(module);
+		kfree(dev_hold);
+	}
+}
+
+static struct drm_pagemap_dev_hold *
+drm_pagemap_dev_hold(struct drm_pagemap *dpagemap)
+{
+	struct drm_pagemap_dev_hold *dev_hold;
+	struct drm_device *drm = dpagemap->drm;
+
+	dev_hold = kzalloc(sizeof(*dev_hold), GFP_KERNEL);
+	if (!dev_hold)
+		return ERR_PTR(-ENOMEM);
+
+	init_llist_node(&dev_hold->link);
+	dev_hold->drm = drm;
+	(void)try_module_get(drm->driver->fops->owner);
+	drm_dev_get(drm);
+
+	return dev_hold;

 }

 /**
  * drm_pagemap_create() - Create a struct drm_pagemap.
- * @dev: Pointer to a struct device providing the device-private memory.
+ * @drm: Pointer to a struct drm_device providing the device- private memory.
  * @pagemap: Pointer to a pre-setup struct dev_pagemap providing the struct pages.
  * @ops: Pointer to the struct drm_pagemap_ops.
  *

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -563,20 +640,28 @@ static void drm_pagemap_release(struct kref
 *ref)
   * Error pointer on error.
   */

 struct drm_pagemap *

drivers/gpu/drm/drm_pagemap.c +1 -1

-drm_pagemap_create(struct device *dev,
+drm_pagemap_create(struct drm_device *drm,

     struct dev_pagemap *pagemap,
     const struct drm_pagemap_ops *ops)
 {
  struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap), GFP_KERNEL);

drivers/gpu/drm/drm_pagemap.c +1 -0

+	struct drm_pagemap_dev_hold *dev_hold;

  if (!dpagemap)
  return ERR_PTR(-ENOMEM);

  kref_init(&dpagemap->ref);

drivers/gpu/drm/drm_pagemap.c +1 -1

-	dpagemap->dev = dev;
+	dpagemap->drm = drm;

  dpagemap->ops = ops;
  dpagemap->pagemap = pagemap;

drivers/gpu/drm/drm_pagemap.c +7 -0

+	dev_hold = drm_pagemap_dev_hold(dpagemap);
+	if (IS_ERR(dev_hold)) {
+		kfree(dpagemap);
+		return ERR_CAST(dev_hold);
+	}
+	dpagemap->dev_hold = dev_hold;
+

  return dpagemap;
 }
 EXPORT_SYMBOL(drm_pagemap_create);

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -937,3 +1022,11 @@ int drm_pagemap_populate_mm(struct
 drm_pagemap *dpagemap,
  	return err;
  }

 EXPORT_SYMBOL(drm_pagemap_populate_mm);

drivers/gpu/drm/drm_pagemap.c +8 -0

+
+static void drm_pagemap_exit(void)
+{
+	flush_work(&drm_pagemap_work);
+	if (WARN_ON(!llist_empty(&drm_pagemap_unhold_list)))
+		disable_work_sync(&drm_pagemap_work);
+}
+module_exit(drm_pagemap_exit);

drivers/gpu/drm/xe/xe_svm.c +1 -1

index 6d2c6c144315..f6ee22da2e95 100644
@@ -1437,7 +1437,7 @@ xe_drm_pagemap_device_map(struct drm_pagemap
 *dpagemap,
  			  unsigned int order,
  			  enum dma_data_direction dir)
  {
-	struct device *pgmap_dev = dpagemap->dev;
+	struct device *pgmap_dev = dpagemap->drm->dev;
  	enum drm_interconnect_protocol prot;
  	dma_addr_t addr;

drivers/gpu/drm/xe/xe_svm.c +0 -0

@@ -1457,6 +1457,14 @@ static const struct drm_pagemap_ops
 xe_drm_pagemap_ops = {
  	.populate_mm = xe_drm_pagemap_populate_mm,
  };

drivers/gpu/drm/xe/xe_svm.c +8 -0

+static void xe_devm_release(void *data)
+{
+	struct xe_vram_region *vr = data;
+
+	drm_pagemap_put(vr->dpagemap);
+	vr->dpagemap = NULL;
+}
+

 /**
  * xe_devm_add: Remap and provide memmap backing for device memory
  * @tile: tile that the memory region belongs to

drivers/gpu/drm/xe/xe_svm.c +0 -0

@@ -1482,7 +1490,7 @@ int xe_devm_add(struct xe_tile *tile, struct
 xe_vram_region *vr)
  		return ret;
  	}

drivers/gpu/drm/xe/xe_svm.c +1 -1

-	vr->dpagemap = drm_pagemap_create(dev, &vr->pagemap,
+	vr->dpagemap = drm_pagemap_create(&xe->drm, &vr->pagemap,

    &xe_drm_pagemap_ops); if (IS_ERR(vr->dpagemap)) {
  drm_err(&xe->drm, "Failed to create drm_pagemap tile %d memory: %pe\n",

drivers/gpu/drm/xe/xe_svm.c int xe_devm_add(struct xe_tile *tile, struct · line 1504

Quoted 7 lines

drivers/gpu/drm/xe/xe_svm.c +3 -0

@@ -1490,6 +1498,9 @@ int xe_devm_add(struct xe_tile *tile, struct
 xe_vram_region *vr)
  		ret = PTR_ERR(vr->dpagemap);
  		goto out_no_dpagemap;
  	}
+	ret = devm_add_action_or_reset(dev, xe_devm_release, vr);
+	if (ret)
+		goto out_no_dpagemap;

I mentioned this in first patch that this was missing, maybe move this
part to the first patch even though this will get removed a bit later.

Matt

Quoted 49 lines

  vr->pagemap.type = MEMORY_DEVICE_PRIVATE;
  vr->pagemap.range.start = res->start;

include/drm/drm_pagemap.h +0 -0

index 2c7de928865b..5cfe54331ba7 100644
@@ -9,6 +9,7 @@
  #define NR_PAGES(order) (1U << (order))

 struct drm_pagemap;
+struct drm_pagemap_dev_hold;
 struct drm_pagemap_zdd;
 struct device;

include/drm/drm_pagemap.h +5 -2

@@ -130,14 +131,17 @@ struct drm_pagemap_ops {
   * used for device p2p handshaking.
   * @ops: The struct drm_pagemap_ops.
   * @ref: Reference count.
- * @dev: The struct drevice owning the device-private memory.
+ * @drm: The struct drm device owning the device-private memory.
   * @pagemap: Pointer to the underlying dev_pagemap.
+ * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for
+ * device referencing.
   */
  struct drm_pagemap {
  	const struct drm_pagemap_ops *ops;
  	struct kref ref;
-	struct device *dev;
+	struct drm_device *drm;
  	struct dev_pagemap *pagemap;
+	struct drm_pagemap_dev_hold *dev_hold;
  };

 struct drm_pagemap_devmem;

include/drm/drm_pagemap.h +0 -0

@@ -206,7 +210,7 @@ struct drm_pagemap_devmem_ops {
  			   unsigned long npages);
  };

include/drm/drm_pagemap.h +1 -1

-struct drm_pagemap *drm_pagemap_create(struct device *dev,
+struct drm_pagemap *drm_pagemap_create(struct drm_device *drm,

         struct dev_pagemap *pagemap,
         const struct drm_pagemap_ops *ops);

Signature · 1 line

2.51.0

Quoted 7 lines · Matthew Brost

Quoted 5 lines · Thomas Hellström

Even if the drm_pagemap provider has released its reference on the drm_pagemap, references may be held by still active pages. Ensure that we hold a reference on the provider drm device and modules for as long as we might need to use the drm_pagemap ops.

Quoted 7 lines

Just to make sure I’m understanding this correctly — this is intended to
guard against the devm action [1] running while a device is still holding references to another device’s pages, right?

[1]
https://elixir.bootlin.com/linux/v6.17.5/source/kernel/resource.c#L1993

Actually removing the dev_pagemap and its region is allowed while another device holds a reference on the *drm_pagemap*. For example if you have two devices. Device 0 executes from the memory of device 1. Suddenly you feel like offlining / unbinding device 1. When you execute unbind, the driver evicts all SVM bos and thereby frees all device- private pages. But device 0 still has a reference to the drm_pagemap, even if it's unusable: Any VRAM migration trying to use the drm_pagemap will error with -ENODEV, so depending on how the driver handles that, it will continue executing out of another memory region. At this point it would've been possible without this code to rmmod the drm_pagemap provider device module, and its drm device would've been freed without this code, and when drm_pagemap_put() eventually is called, things go boom. So the commit message is a bit misleading.

In the case where we only have pages left, the last page should be freed from the device remove callback where bos are evicted. At that point, the provider drm device is still alive as the devm callbacks haven't executed yet. Also a rmmod wold typically cause the devm callbacks to execute so that should also be safe without this patch. At least if the page freeing doesn't trigger any async callbacks that aren't waited on before removal.

So yeah, I need to update the commit message a bit. We should also craft an IGT that unbinds device 1 while device 0 is executing out of its memory and verify that execution completes with correct results anyway.

/Thomas

Quoted 294 lines

Quoted 230 lines

Note that in theory, the drm_gpusvm_helper module may be unloaded as soon as the final module_put() of the provider driver module is executed, so we need to add a module_exit() function that waits for the work item executing the module_put() has completed.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/drm_pagemap.c | 101
++++++++++++++++++++++++++++++++--
 drivers/gpu/drm/xe/xe_svm.c   |  15 ++++-
 include/drm/drm_pagemap.h     |  10 +++-

 3 files changed, 117 insertions(+), 9 deletions(-)

drivers/gpu/drm/drm_pagemap.c +1 -0

index 173b3ecb07d5..fb18a80d6a1c 100644
@@ -8,6 +8,7 @@
  #include <linux/pagemap.h>
  #include <drm/drm_drv.h>
  #include <drm/drm_pagemap.h>
+#include <drm/drm_print.h>

 /**
  * DOC: Overview

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -544,16 +545,92 @@ static int
 drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
  	return -ENOMEM;
  }

drivers/gpu/drm/drm_pagemap.c +3 -0

+static void drm_pagemap_dev_unhold_work(struct work_struct *work);
+static LLIST_HEAD(drm_pagemap_unhold_list);
+static DECLARE_WORK(drm_pagemap_work,

drm_pagemap_dev_unhold_work);

drivers/gpu/drm/drm_pagemap.c +3 -0

+
+/**
+ * struct drm_pagemap_dev_hold - Struct to aid in drm_device

release.
+ * @link: Link into drm_pagemap_unhold_list for deferred reference releases.
+ * @drm: drm device to put.
+ *
+ * When a struct drm_pagemap is released, we also need to release the
+ * reference it holds on the drm device. However, typically that needs
+ * to be done separately from a system-wide workqueue.
+ * Each time a struct drm_pagemap is initialized
+ * (or re-initialized if cached) therefore allocate a separate
+ * drm_pagemap_dev_hold item, from which we put the drm device and
+ * associated module.
+ */

drivers/gpu/drm/drm_pagemap.c +5 -0

+struct drm_pagemap_dev_hold {
+	struct llist_node link;
+	struct drm_device *drm;
+};
+

 static void drm_pagemap_release(struct kref *ref) {
  struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref);

drivers/gpu/drm/drm_pagemap.c +1 -1

-
+	struct drm_pagemap_dev_hold *dev_hold = dpagemap-

Quoted 1 line

dev_hold;

drivers/gpu/drm/drm_pagemap.c +3 -0

+
+	/*
+	 * We know the pagemap provider is alive at this point,

since

drivers/gpu/drm/drm_pagemap.c +1 -0

+	 * the struct drm_pagemap_dev_hold holds a reference to

the

drivers/gpu/drm/drm_pagemap.c +3 -0

+	 * pagemap provider drm_device and its module.
+	 */
+	dpagemap->dev_hold = NULL;

  kfree(dpagemap);

drivers/gpu/drm/drm_pagemap.c +4 -0

+	llist_add(&dev_hold->link, &drm_pagemap_unhold_list);
+	schedule_work(&drm_pagemap_work);
+	/*
+	 * Here, either the provider device is still alive, since

if called from

drivers/gpu/drm/drm_pagemap.c +1 -0

+	 * page_free(), the caller is holding a reference on the

dev_pagemap,

drivers/gpu/drm/drm_pagemap.c +1 -0

+	 * or if called from drm_pagemap_put(), the direct caller

is still alive.

drivers/gpu/drm/drm_pagemap.c +7 -0

+	 * This ensures we can't race with THIS module unload.
+	 */
+}
+
+static void drm_pagemap_dev_unhold_work(struct work_struct *work)
+{
+	struct llist_node *node =

llist_del_all(&drm_pagemap_unhold_list);

drivers/gpu/drm/drm_pagemap.c +4 -0

+	struct drm_pagemap_dev_hold *dev_hold, *next;
+
+	/*
+	 * Deferred release of drm_pagemap provider device and

module.

drivers/gpu/drm/drm_pagemap.c +8 -0

+	 * THIS module is kept alive during the release by the
+	 * flush_work() in the drm_pagemap_exit() function.
+	 */
+	llist_for_each_entry_safe(dev_hold, next, node, link) {
+		struct drm_device *drm = dev_hold->drm;
+		struct module *module = drm->driver->fops->owner;
+
+		drm_dbg(drm, "Releasing reference on provider

device and module.\n");

drivers/gpu/drm/drm_pagemap.c +22 -0

+		drm_dev_put(drm);
+		module_put(module);
+		kfree(dev_hold);
+	}
+}
+
+static struct drm_pagemap_dev_hold *
+drm_pagemap_dev_hold(struct drm_pagemap *dpagemap)
+{
+	struct drm_pagemap_dev_hold *dev_hold;
+	struct drm_device *drm = dpagemap->drm;
+
+	dev_hold = kzalloc(sizeof(*dev_hold), GFP_KERNEL);
+	if (!dev_hold)
+		return ERR_PTR(-ENOMEM);
+
+	init_llist_node(&dev_hold->link);
+	dev_hold->drm = drm;
+	(void)try_module_get(drm->driver->fops->owner);
+	drm_dev_get(drm);
+
+	return dev_hold;

 }

 /**
  * drm_pagemap_create() - Create a struct drm_pagemap.
- * @dev: Pointer to a struct device providing the device-private memory.
+ * @drm: Pointer to a struct drm_device providing the device- private memory.
  * @pagemap: Pointer to a pre-setup struct dev_pagemap providing the struct pages.
  * @ops: Pointer to the struct drm_pagemap_ops.
  *

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -563,20 +640,28 @@ static void drm_pagemap_release(struct kref
 *ref)
   * Error pointer on error.
   */

 struct drm_pagemap *

drivers/gpu/drm/drm_pagemap.c +1 -1

-drm_pagemap_create(struct device *dev,
+drm_pagemap_create(struct drm_device *drm,

     struct dev_pagemap *pagemap,
     const struct drm_pagemap_ops *ops)
 {
  struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap), GFP_KERNEL);

drivers/gpu/drm/drm_pagemap.c +1 -0

+	struct drm_pagemap_dev_hold *dev_hold;

  if (!dpagemap)
  return ERR_PTR(-ENOMEM);

  kref_init(&dpagemap->ref);

drivers/gpu/drm/drm_pagemap.c +1 -1

-	dpagemap->dev = dev;
+	dpagemap->drm = drm;

  dpagemap->ops = ops;
  dpagemap->pagemap = pagemap;

drivers/gpu/drm/drm_pagemap.c +7 -0

+	dev_hold = drm_pagemap_dev_hold(dpagemap);
+	if (IS_ERR(dev_hold)) {
+		kfree(dpagemap);
+		return ERR_CAST(dev_hold);
+	}
+	dpagemap->dev_hold = dev_hold;
+

  return dpagemap;
 }
 EXPORT_SYMBOL(drm_pagemap_create);

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -937,3 +1022,11 @@ int drm_pagemap_populate_mm(struct
 drm_pagemap *dpagemap,
  	return err;
  }

 EXPORT_SYMBOL(drm_pagemap_populate_mm);

drivers/gpu/drm/drm_pagemap.c +8 -0

+
+static void drm_pagemap_exit(void)
+{
+	flush_work(&drm_pagemap_work);
+	if (WARN_ON(!llist_empty(&drm_pagemap_unhold_list)))
+		disable_work_sync(&drm_pagemap_work);
+}
+module_exit(drm_pagemap_exit);

drivers/gpu/drm/xe/xe_svm.c +1 -1

index 6d2c6c144315..f6ee22da2e95 100644
@@ -1437,7 +1437,7 @@ xe_drm_pagemap_device_map(struct drm_pagemap
 *dpagemap,
  			  unsigned int order,
  			  enum dma_data_direction dir)
  {
-	struct device *pgmap_dev = dpagemap->dev;
+	struct device *pgmap_dev = dpagemap->drm->dev;
  	enum drm_interconnect_protocol prot;
  	dma_addr_t addr;

drivers/gpu/drm/xe/xe_svm.c +0 -0

@@ -1457,6 +1457,14 @@ static const struct drm_pagemap_ops
 xe_drm_pagemap_ops = {
  	.populate_mm = xe_drm_pagemap_populate_mm,
  };

drivers/gpu/drm/xe/xe_svm.c +8 -0

+static void xe_devm_release(void *data)
+{
+	struct xe_vram_region *vr = data;
+
+	drm_pagemap_put(vr->dpagemap);
+	vr->dpagemap = NULL;
+}
+

 /**
  * xe_devm_add: Remap and provide memmap backing for device memory
  * @tile: tile that the memory region belongs to

drivers/gpu/drm/xe/xe_svm.c +0 -0

@@ -1482,7 +1490,7 @@ int xe_devm_add(struct xe_tile *tile, struct
 xe_vram_region *vr)
  		return ret;
  	}

drivers/gpu/drm/xe/xe_svm.c +1 -1

-	vr->dpagemap = drm_pagemap_create(dev, &vr->pagemap,
+	vr->dpagemap = drm_pagemap_create(&xe->drm, &vr->pagemap,

    &xe_drm_pagemap_ops); if (IS_ERR(vr->dpagemap)) {
  drm_err(&xe->drm, "Failed to create drm_pagemap tile %d memory: %pe\n",

drivers/gpu/drm/xe/xe_svm.c int xe_devm_add(struct xe_tile *tile, struct · line 1504

Quoted 7 lines

drivers/gpu/drm/xe/xe_svm.c +3 -0

@@ -1490,6 +1498,9 @@ int xe_devm_add(struct xe_tile *tile, struct
 xe_vram_region *vr)
  		ret = PTR_ERR(vr->dpagemap);
  		goto out_no_dpagemap;
  	}
+	ret = devm_add_action_or_reset(dev, xe_devm_release, vr);
+	if (ret)
+		goto out_no_dpagemap;

I mentioned this in first patch that this was missing, maybe move this
part to the first patch even though this will get removed a bit later.

Matt

Quoted 49 lines

  vr->pagemap.type = MEMORY_DEVICE_PRIVATE;
  vr->pagemap.range.start = res->start;

include/drm/drm_pagemap.h +0 -0

index 2c7de928865b..5cfe54331ba7 100644
@@ -9,6 +9,7 @@
  #define NR_PAGES(order) (1U << (order))

 struct drm_pagemap;
+struct drm_pagemap_dev_hold;
 struct drm_pagemap_zdd;
 struct device;

include/drm/drm_pagemap.h +5 -2

@@ -130,14 +131,17 @@ struct drm_pagemap_ops {
   * used for device p2p handshaking.
   * @ops: The struct drm_pagemap_ops.
   * @ref: Reference count.
- * @dev: The struct drevice owning the device-private memory.
+ * @drm: The struct drm device owning the device-private memory.
   * @pagemap: Pointer to the underlying dev_pagemap.
+ * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for
+ * device referencing.
   */
  struct drm_pagemap {
  	const struct drm_pagemap_ops *ops;
  	struct kref ref;
-	struct device *dev;
+	struct drm_device *drm;
  	struct dev_pagemap *pagemap;
+	struct drm_pagemap_dev_hold *dev_hold;
  };

 struct drm_pagemap_devmem;

include/drm/drm_pagemap.h +0 -0

@@ -206,7 +210,7 @@ struct drm_pagemap_devmem_ops {
  			   unsigned long npages);
  };

include/drm/drm_pagemap.h +1 -1

-struct drm_pagemap *drm_pagemap_create(struct device *dev,
+struct drm_pagemap *drm_pagemap_create(struct drm_device *drm,

         struct dev_pagemap *pagemap,
         const struct drm_pagemap_ops *ops);

Signature · 1 line

2.51.0

Pagemaps are costly to set up and tear down, and they consume a lot of system memory for the struct pages. Ideally they should be created only when needed.

Add a caching mechanism to allow doing just that: Create the drm_pagemaps when needed for migration. Keep them around to avoid destruction and re-creation latencies and destroy inactive/unused drm_pagemaps on memory pressure using a shrinker.

Only add the helper functions. They will be hooked up to the xe driver in the upcoming patch.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/Makefile           |   3 +-
 drivers/gpu/drm/drm_pagemap.c      |  79 +++++-
 drivers/gpu/drm/drm_pagemap_util.c | 426 +++++++++++++++++++++++++++++
 include/drm/drm_pagemap.h          |  53 +++-
 include/drm/drm_pagemap_util.h     |  25 ++
 5 files changed, 569 insertions(+), 17 deletions(-)
 create mode 100644 drivers/gpu/drm/drm_pagemap_util.c
 create mode 100644 include/drm/drm_pagemap_util.h

drivers/gpu/drm/Makefile +2 -1

index c2672f369aed..cdca68fd9f23 100644
@@ -107,7 +107,8 @@ obj-$(CONFIG_DRM_GPUVM) += drm_gpuvm.o
 
 drm_gpusvm_helper-y := \
 	drm_gpusvm.o\
-	drm_pagemap.o
+	drm_pagemap.o\
+	drm_pagemap_util.o
 obj-$(CONFIG_DRM_GPUSVM) += drm_gpusvm_helper.o
 
 obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o

drivers/gpu/drm/drm_pagemap.c +66 -13

index fb18a80d6a1c..5ca5b2b53bc1 100644
@@ -8,6 +8,7 @@
 #include <linux/pagemap.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_pagemap.h>
+#include <drm/drm_pagemap_util.h>
 #include <drm/drm_print.h>
 
 /**

@@ -578,7 +579,7 @@ static void drm_pagemap_release(struct kref *ref)
 	 * pagemap provider drm_device and its module.
 	 */
 	dpagemap->dev_hold = NULL;
-	kfree(dpagemap);
+	drm_pagemap_shrinker_add(dpagemap);
 	llist_add(&dev_hold->link, &drm_pagemap_unhold_list);
 	schedule_work(&drm_pagemap_work);
 	/*

@@ -628,6 +629,58 @@ drm_pagemap_dev_hold(struct drm_pagemap *dpagemap)
 	return dev_hold;
 }
 
+/**
+ * drm_pagemap_reinit() - Reinitialize a drm_pagemap
+ * @dpagemap: The drm_pagemap to reinitialize
+ *
+ * Reinitialize a drm_pagemap, for which drm_pagemap_release
+ * has already been called. This interface is intended for the
+ * situation where the driver caches a destroyed drm_pagemap.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int drm_pagemap_reinit(struct drm_pagemap *dpagemap)
+{
+	dpagemap->dev_hold = drm_pagemap_dev_hold(dpagemap);
+	if (IS_ERR(dpagemap->dev_hold))
+		return PTR_ERR(dpagemap->dev_hold);
+
+	kref_init(&dpagemap->ref);
+	return 0;
+}
+EXPORT_SYMBOL(drm_pagemap_reinit);
+
+/**
+ * drm_pagemap_init() - Initialize a pre-allocated drm_pagemap
+ * @dpagemap: The drm_pagemap to initialize.
+ * @pagemap: The associated dev_pagemap providing the device
+ * private pages.
+ * @drm: The drm device. The drm_pagemap holds a reference on the
+ * drm_device and the module owning the drm_device until
+ * drm_pagemap_release(). This facilitates drm_pagemap exporting.
+ * @ops: The drm_pagemap ops.
+ *
+ * Initialize and take an initial reference on a drm_pagemap.
+ * After successful return, use drm_pagemap_put() to destroy.
+ *
+ ** Return: 0 on success, negative error code on error.
+ */
+int drm_pagemap_init(struct drm_pagemap *dpagemap,
+		     struct dev_pagemap *pagemap,
+		     struct drm_device *drm,
+		     const struct drm_pagemap_ops *ops)
+{
+	kref_init(&dpagemap->ref);
+	dpagemap->ops = ops;
+	dpagemap->pagemap = pagemap;
+	dpagemap->drm = drm;
+	dpagemap->cache = NULL;
+	INIT_LIST_HEAD(&dpagemap->shrink_link);
+
+	return drm_pagemap_reinit(dpagemap);
+}
+EXPORT_SYMBOL(drm_pagemap_init);
+
 /**
  * drm_pagemap_create() - Create a struct drm_pagemap.
  * @drm: Pointer to a struct drm_device providing the device-private memory.

@@ -645,22 +698,14 @@ drm_pagemap_create(struct drm_device *drm,
 		   const struct drm_pagemap_ops *ops)
 {
 	struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap), GFP_KERNEL);
-	struct drm_pagemap_dev_hold *dev_hold;
+	int err;
 
 	if (!dpagemap)
 		return ERR_PTR(-ENOMEM);
 
-	kref_init(&dpagemap->ref);
-	dpagemap->drm = drm;
-	dpagemap->ops = ops;
-	dpagemap->pagemap = pagemap;
-
-	dev_hold = drm_pagemap_dev_hold(dpagemap);
-	if (IS_ERR(dev_hold)) {
-		kfree(dpagemap);
-		return ERR_CAST(dev_hold);
-	}
-	dpagemap->dev_hold = dev_hold;
+	err = drm_pagemap_init(dpagemap, pagemap, drm, ops);
+	if (err)
+		return ERR_PTR(err);
 
 	return dpagemap;
 }

@@ -1023,6 +1068,14 @@ int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 }
 EXPORT_SYMBOL(drm_pagemap_populate_mm);
 
+void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim)
+{
+	if (dpagemap->ops->destroy)
+		dpagemap->ops->destroy(dpagemap, is_atomic_or_reclaim);
+	else
+		kfree(dpagemap);
+}
+
 static void drm_pagemap_exit(void)
 {
 	flush_work(&drm_pagemap_work);

drivers/gpu/drm/drm_pagemap_util.c new file +285 -0

new file mode 100644
index 000000000000..e1a1d6bf25f4
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#include <drm/drm_drv.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_pagemap.h>
+#include <drm/drm_pagemap_util.h>
+#include <drm/drm_print.h>
+
+/**
+ * struct drm_pagemap_cache - Lookup structure for pagemaps
+ *
+ * Structure to keep track of active (refcount > 1) and inactive
+ * (refcount == 0) pagemaps. Inactive pagemaps can be made active
+ * again by waiting for the @queued completion (indicating that the
+ * pagemap has been put on the @shrinker's list of shrinkable
+ * pagemaps, and then successfully removing it from @shrinker's
+ * list. The latter may fail if the shrinker is already in the
+ * process of freeing the pagemap. A struct drm_pagemap_cache can
+ * hold a single struct drm_pagemap.
+ */
+struct drm_pagemap_cache {
+	/** @lookup_mutex: Mutex making the lookup process atomic */
+	struct mutex lookup_mutex;
+	/** @lock: Lock protecting the @dpagemap pointer */
+	spinlock_t lock;
+	/** @shrinker: Pointer to the shrinker used for this cache. Immutable. */
+	struct drm_pagemap_shrinker *shrinker;
+	/** @dpagemap: Non-refcounted pointer to the drm_pagemap */
+	struct drm_pagemap *dpagemap;
+	/**
+	 * @queued: Signals when an inactive drm_pagemap has been put on
+	 * @shrinker's list.
+	 */
+	struct completion queued;
+};
+
+/**
+ * struct drm_pagemap_shrinker - Shrinker to remove unused pagemaps
+ */
+struct drm_pagemap_shrinker {
+	/** @drm: Pointer to the drm device. */
+	struct drm_device *drm;
+	/** @lock: Spinlock to protect the @dpagemaps list. */
+	spinlock_t lock;
+	/** @dpagemaps: List of unused dpagemaps. */
+	struct list_head dpagemaps;
+	/** @num_dpagemaps: Number of unused dpagemaps in @dpagemaps. */
+	atomic_t num_dpagemaps;
+	/** @shrink: Pointer to the struct shrinker. */
+	struct shrinker *shrink;
+};
+
+static bool drm_pagemap_shrinker_cancel(struct drm_pagemap *dpagemap);
+
+static void drm_pagemap_cache_fini(void *arg)
+{
+	struct drm_pagemap_cache *cache = arg;
+	struct drm_pagemap *dpagemap;
+
+	drm_dbg(cache->shrinker->drm, "Destroying dpagemap cache.\n");
+	spin_lock(&cache->lock);
+	dpagemap = cache->dpagemap;
+	if (!dpagemap) {
+		spin_unlock(&cache->lock);
+		goto out;
+	}
+
+	if (drm_pagemap_shrinker_cancel(dpagemap)) {
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+		drm_pagemap_destroy(dpagemap, false);
+	}
+
+out:
+	mutex_destroy(&cache->lookup_mutex);
+	kfree(cache);
+}
+
+/**
+ * drm_pagemap_cache_create_devm() - Create a drm_pagemap_cache
+ * @shrinker: Pointer to a struct drm_pagemap_shrinker.
+ *
+ * Create a device-managed drm_pagemap cache. The cache is automatically
+ * destroyed on struct device removal, at which point any *inactive*
+ * drm_pagemap's are destroyed.
+ *
+ * Return: Pointer to a struct drm_pagemap_cache on success. Error pointer
+ * on failure.
+ */
+struct drm_pagemap_cache *drm_pagemap_cache_create_devm(struct drm_pagemap_shrinker *shrinker)
+{
+	struct drm_pagemap_cache *cache = kzalloc(sizeof(*cache), GFP_KERNEL);
+	int err;
+
+	if (!cache)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&cache->lookup_mutex);
+	spin_lock_init(&cache->lock);
+	cache->shrinker = shrinker;
+	init_completion(&cache->queued);
+	err = devm_add_action_or_reset(shrinker->drm->dev, drm_pagemap_cache_fini, cache);
+	if (err)
+		return ERR_PTR(err);
+
+	return cache;
+}
+EXPORT_SYMBOL(drm_pagemap_cache_create_devm);
+
+/**
+ * DOC: Cache lookup
+ *
+ * Cache lookup should be done under a locked mutex, so that a
+ * failed drm_pagemap_get_from_cache() and a following
+ * drm_pagemap_cache_setpagemap() are carried out as an atomic
+ * operation WRT other lookups. Otherwise, racing lookups may
+ * unnecessarily concurrently create pagemaps to fulfill a
+ * failed lookup. The API provides two functions to perform this lock,
+ * drm_pagemap_lock_lookup() and drm_pagemap_unlock_lookup() and they
+ * should be used in the following way:
+ *
+ * .. code-block:: c
+ *
+ *		drm_pagemap_lock_lookup(cache);
+ *		dpagemap = drm_pagemap_get_from_cache(cache);
+ *		if (dpagemap)
+ *			goto out_unlock;
+ *
+ *		dpagemap = driver_create_new_dpagemap();
+ *		if (!IS_ERR(dpagemap))
+ *			drm_pagemap_cache_set_pagemap(cache, dpagemap);
+ *
+ *     out_unlock:
+ *		drm_pagemap_unlock_lookup(cache);
+ */
+
+/**
+ * drm_pagemap_cache_lock_lookup() Lock a drm_pagemap_cache for lookup
+ * @cache: The drm_pagemap_cache to lock.
+ *
+ * Return: %-EINTR if interrupted while blocking. %0 otherwise.
+ */
+int drm_pagemap_cache_lock_lookup(struct drm_pagemap_cache *cache)
+{
+	return mutex_lock_interruptible(&cache->lookup_mutex);
+}
+EXPORT_SYMBOL(drm_pagemap_cache_lock_lookup);
+
+/**
+ * drm_pagemap_cache_unlock_lookup() Unlock a drm_pagemap_cache after lookup
+ * @cache: The drm_pagemap_cache to unlock.
+ */
+void drm_pagemap_cache_unlock_lookup(struct drm_pagemap_cache *cache)
+{
+	mutex_unlock(&cache->lookup_mutex);
+}
+EXPORT_SYMBOL(drm_pagemap_cache_unlock_lookup);
+
+/**
+ * drm_pagemap_get_from_cache() -  Lookup of drm_pagemaps.
+ * @cache: The cache used for lookup.
+ *
+ * If an active pagemap is present in the cache, it is immediately returned.
+ * If an inactive pagemap is present, it's removed from the shrinker list and
+ * an attempt is made to make it active.
+ * If no pagemap present or the attempt to make it active failed, %NULL is returned
+ * to indicate to the caller to create a new drm_pagemap and insert it into
+ * the cache.
+ *
+ * Return: A reference-counted pointer to a drm_pagemap if successful. An error
+ * pointer if an error occurred, or %NULL if no drm_pagemap was found and
+ * the caller should insert a new one.
+ */
+struct drm_pagemap *drm_pagemap_get_from_cache(struct drm_pagemap_cache *cache)
+{
+	struct drm_pagemap *dpagemap;
+	int err;
+
+	lockdep_assert_held(&cache->lookup_mutex);
+retry:
+	spin_lock(&cache->lock);
+	dpagemap = cache->dpagemap;
+	if (drm_pagemap_get_unless_zero(dpagemap)) {
+		spin_unlock(&cache->lock);
+		return dpagemap;
+	}
+
+	if (!dpagemap) {
+		spin_unlock(&cache->lock);
+		return NULL;
+	}
+
+	if (!try_wait_for_completion(&cache->queued)) {
+		spin_unlock(&cache->lock);
+		err = wait_for_completion_interruptible(&cache->queued);
+		if (err)
+			return ERR_PTR(err);
+		goto retry;
+	}
+
+	if (drm_pagemap_shrinker_cancel(dpagemap)) {
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+		err = drm_pagemap_reinit(dpagemap);
+		if (err) {
+			drm_pagemap_destroy(dpagemap, false);
+			return ERR_PTR(err);
+		}
+		drm_pagemap_cache_set_pagemap(cache, dpagemap);
+	} else {
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+		dpagemap = NULL;
+	}
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_get_from_cache);
+
+/**
+ * drm_pagemap_cache_set_pagemap() - Assign a drm_pagemap to a drm_pagemap_cache
+ * @cache: The cache to assign the drm_pagemap to.
+ * @dpagemap: The drm_pagemap to assign.
+ *
+ * The function must be called to populate a drm_pagemap_cache only
+ * after a call to drm_pagemap_get_from_cache() returns NULL.
+ */
+void drm_pagemap_cache_set_pagemap(struct drm_pagemap_cache *cache, struct drm_pagemap *dpagemap)
+{
+	struct drm_device *drm = dpagemap->drm;
+
+	lockdep_assert_held(&cache->lookup_mutex);
+	spin_lock(&cache->lock);
+	dpagemap->cache = cache;
+	swap(cache->dpagemap, dpagemap);
+	reinit_completion(&cache->queued);
+	spin_unlock(&cache->lock);
+	drm_WARN_ON(drm, !!dpagemap);
+}
+EXPORT_SYMBOL(drm_pagemap_cache_set_pagemap);
+
+/**
+ * drm_pagemap_get_from_cache_if_active() - Quick lookup of active drm_pagemaps
+ * @cache: The cache to lookup from.
+ *
+ * Function that should be used to lookup a drm_pagemap that is already active.
+ * (refcount > 0).
+ *
+ * Return: A pointer to the cache's drm_pagemap if it's active; %NULL otherwise.
+ */
+struct drm_pagemap *drm_pagemap_get_from_cache_if_active(struct drm_pagemap_cache *cache)
+{
+	struct drm_pagemap *dpagemap;
+
+	spin_lock(&cache->lock);
+	dpagemap = drm_pagemap_get_unless_zero(cache->dpagemap);
+	spin_unlock(&cache->lock);
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_get_from_cache_if_active);
+
+static bool drm_pagemap_shrinker_cancel(struct drm_pagemap *dpagemap)
+{
+	struct drm_pagemap_cache *cache = dpagemap->cache;
+	struct drm_pagemap_shrinker *shrinker = cache->shrinker;
+
+	spin_lock(&shrinker->lock);
+	if (list_empty(&dpagemap->shrink_link)) {
+		spin_unlock(&shrinker->lock);
+		return false;
+	}
+
+	list_del_init(&dpagemap->shrink_link);
+	atomic_dec(&shrinker->num_dpagemaps);
+	spin_unlock(&shrinker->lock);
+	return true;
+}
+
+/**
+ * drm_pagemap_shrinker_add() - Add a drm_pagemap to the shrinker list or destroy
+ * @dpagemap: The drm_pagemap.

drivers/gpu/drm/drm_pagemap_util.c new file +131 -0

+{
+	struct drm_pagemap_cache *cache;
+	struct drm_pagemap_shrinker *shrinker;
+	int idx;
+
+	/*
+	 * The pagemap cache and shrinker are disabled at
+	 * pci device remove time. After that, dpagemaps
+	 * are freed directly.
+	 */
+	if (!drm_dev_enter(dpagemap->drm, &idx))
+		goto out_no_cache;
+
+	cache = dpagemap->cache;
+	if (!cache) {
+		drm_dev_exit(idx);
+		goto out_no_cache;
+	}
+
+	shrinker = cache->shrinker;
+	spin_lock(&shrinker->lock);
+	list_add_tail(&dpagemap->shrink_link, &shrinker->dpagemaps);
+	atomic_inc(&shrinker->num_dpagemaps);
+	spin_unlock(&shrinker->lock);
+	complete_all(&cache->queued);
+	drm_dev_exit(idx);
+	return;
+
+out_no_cache:
+	drm_pagemap_destroy(dpagemap, true);
+}
+
+static unsigned long
+drm_pagemap_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct drm_pagemap_shrinker *shrinker = shrink->private_data;
+	unsigned long count = atomic_read(&shrinker->num_dpagemaps);
+
+	return count ? : SHRINK_EMPTY;
+}
+
+static unsigned long
+drm_pagemap_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+	struct drm_pagemap_shrinker *shrinker = shrink->private_data;
+	struct drm_pagemap *dpagemap;
+	struct drm_pagemap_cache *cache;
+	unsigned long nr_freed = 0;
+
+	sc->nr_scanned = 0;
+	spin_lock(&shrinker->lock);
+	do {
+		dpagemap = list_first_entry_or_null(&shrinker->dpagemaps, typeof(*dpagemap),
+						    shrink_link);
+		if (!dpagemap)
+			break;
+
+		atomic_dec(&shrinker->num_dpagemaps);
+		list_del_init(&dpagemap->shrink_link);
+		spin_unlock(&shrinker->lock);
+
+		sc->nr_scanned++;
+		nr_freed++;
+
+		cache = dpagemap->cache;
+		spin_lock(&cache->lock);
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+
+		drm_dbg(dpagemap->drm, "Shrinking dpagemap %p.\n", dpagemap);
+		drm_pagemap_destroy(dpagemap, true);
+		spin_lock(&shrinker->lock);
+	} while (sc->nr_scanned < sc->nr_to_scan);
+	spin_unlock(&shrinker->lock);
+
+	return sc->nr_scanned ? nr_freed : SHRINK_STOP;
+}
+
+static void drm_pagemap_shrinker_fini(void *arg)
+{
+	struct drm_pagemap_shrinker *shrinker = arg;
+
+	drm_dbg(shrinker->drm, "Destroying dpagemap shrinker.\n");
+	drm_WARN_ON(shrinker->drm, !!atomic_read(&shrinker->num_dpagemaps));
+	shrinker_free(shrinker->shrink);
+	kfree(shrinker);
+}
+
+/**
+ * drm_pagemap_shrinker_create_devm() - Create and register a pagemap shrinker
+ * @drm: The drm device
+ *
+ * Create and register a pagemap shrinker that shrinks unused pagemaps
+ * and thereby reduces memory footprint.
+ * The shrinker is drm_device managed and unregisters itself when
+ * the drm device is removed.
+ *
+ * Return: %0 on success, negative error code on failure.
+ */
+struct drm_pagemap_shrinker *drm_pagemap_shrinker_create_devm(struct drm_device *drm)
+{
+	struct drm_pagemap_shrinker *shrinker;
+	struct shrinker *shrink;
+	int err;
+
+	shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL);
+	if (!shrinker)
+		return ERR_PTR(-ENOMEM);
+
+	shrink = shrinker_alloc(0, "drm-drm_pagemap:%s", drm->unique);
+	if (!shrink) {
+		kfree(shrinker);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	spin_lock_init(&shrinker->lock);
+	INIT_LIST_HEAD(&shrinker->dpagemaps);
+	shrinker->drm = drm;
+	shrinker->shrink = shrink;
+	shrink->count_objects = drm_pagemap_shrinker_count;
+	shrink->scan_objects = drm_pagemap_shrinker_scan;
+	shrink->private_data = shrinker;
+	shrinker_register(shrink);
+
+	err = devm_add_action_or_reset(drm->dev, drm_pagemap_shrinker_fini, shrinker);
+	if (err)
+		return ERR_PTR(err);
+
+	return shrinker;
+}
+EXPORT_SYMBOL(drm_pagemap_shrinker_create_devm);

include/drm/drm_pagemap.h +50 -3

index 5cfe54331ba7..4b9af5e785c6 100644
@@ -9,6 +9,7 @@
 #define NR_PAGES(order) (1U << (order))
 
 struct drm_pagemap;
+struct drm_pagemap_cache;
 struct drm_pagemap_dev_hold;
 struct drm_pagemap_zdd;
 struct device;

@@ -124,6 +125,25 @@ struct drm_pagemap_ops {
 			   unsigned long start, unsigned long end,
 			   struct mm_struct *mm,
 			   unsigned long timeslice_ms);
+	/**
+	 * @destroy: Destroy the drm_pagemap and associated resources.
+	 * @dpagemap: The drm_pagemap to destroy.
+	 * @is_atomic_or_reclaim: The function may be called from
+	 * atomic- or reclaim context.
+	 *
+	 * The implementation should take care not to attempt to
+	 * destroy resources that may already have been destroyed
+	 * using devm_ callbacks, since this function may be called
+	 * after the underlying struct device has been unbound.
+	 * If the implementation defers the execution to a work item
+	 * to avoid locking issues, then it must make sure the work
+	 * items are flushed before module exit. If the destroy call
+	 * happens after the provider's pci_remove() callback has
+	 * been executed, a module reference and drm device reference is
+	 * held across the destroy callback.
+	 */
+	void (*destroy)(struct drm_pagemap *dpagemap,
+			bool is_atomic_or_reclaim);
 };
 
 /**

@@ -135,6 +155,10 @@ struct drm_pagemap_ops {
  * @pagemap: Pointer to the underlying dev_pagemap.
  * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for
  * device referencing.
+ * @cache: Back-pointer to the &struct drm_pagemap_cache used for this
+ * &struct drm_pagemap. May be NULL if no cache is used.
+ * @shrink_link: Link into the shrinker's list of drm_pagemaps. Only
+ * used if also using a pagemap cache.
  */
 struct drm_pagemap {
 	const struct drm_pagemap_ops *ops;

@@ -142,6 +166,8 @@ struct drm_pagemap {
 	struct drm_device *drm;
 	struct dev_pagemap *pagemap;
 	struct drm_pagemap_dev_hold *dev_hold;
+	struct drm_pagemap_cache *cache;
+	struct list_head shrink_link;
 };
 
 struct drm_pagemap_devmem;

@@ -210,6 +236,11 @@ struct drm_pagemap_devmem_ops {
 			   unsigned long npages);
 };
 
+int drm_pagemap_init(struct drm_pagemap *dpagemap,
+		     struct dev_pagemap *pagemap,
+		     struct drm_device *drm,
+		     const struct drm_pagemap_ops *ops);
+
 struct drm_pagemap *drm_pagemap_create(struct drm_device *drm,
 				       struct dev_pagemap *pagemap,
 				       const struct drm_pagemap_ops *ops);

@@ -228,9 +259,9 @@ static inline void drm_pagemap_put(struct drm_pagemap *dpagemap)
 
 /**
  * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap
- * @dpagemap: Pointer to the struct drm_pagemap.
+ * @dpagemap: Pointer to the struct drm_pagemap, or NULL.
  *
- * Return: Pointer to the struct drm_pagemap.
+ * Return: Pointer to the struct drm_pagemap, or NULL.
  */
 static inline struct drm_pagemap *
 drm_pagemap_get(struct drm_pagemap *dpagemap)

@@ -241,6 +272,20 @@ drm_pagemap_get(struct drm_pagemap *dpagemap)
 	return dpagemap;
 }
 
+/**
+ * drm_pagemap_get_unless_zero() - Obtain a reference on a struct drm_pagemap
+ * unless the current reference count is zero.
+ * @dpagemap: Pointer to the drm_pagemap or NULL.
+ *
+ * Return: A pointer to @dpagemap if the reference count was successfully
+ * incremented. NULL if @dpagemap was NULL, or its refcount was 0.
+ */
+static inline struct drm_pagemap * __must_check
+drm_pagemap_get_unless_zero(struct drm_pagemap *dpagemap)
+{
+	return (dpagemap && kref_get_unless_zero(&dpagemap->ref)) ? dpagemap : NULL;
+}
+
 /**
  * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation
  *

@@ -284,5 +329,7 @@ int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 			    struct mm_struct *mm,
 			    unsigned long timeslice_ms);
 
-#endif
+void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim);
 
+int drm_pagemap_reinit(struct drm_pagemap *dpagemap);
+#endif

include/drm/drm_pagemap_util.h new file +25 -0

new file mode 100644
index 000000000000..292244d429ee
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef _DRM_PAGEMAP_UTIL_H_
+#define _DRM_PAGEMAP_UTIL_H_
+
+struct drm_device;
+struct drm_pagemap;
+struct drm_pagemap_cache;
+struct drm_pagemap_shrinker;
+
+void drm_pagemap_shrinker_add(struct drm_pagemap *dpagemap);
+
+int drm_pagemap_cache_lock_lookup(struct drm_pagemap_cache *cache);
+
+void drm_pagemap_cache_unlock_lookup(struct drm_pagemap_cache *cache);
+
+struct drm_pagemap_shrinker *drm_pagemap_shrinker_create_devm(struct drm_device *drm);
+
+struct drm_pagemap_cache *drm_pagemap_cache_create_devm(struct drm_pagemap_shrinker *shrinker);
+
+struct drm_pagemap *drm_pagemap_get_from_cache(struct drm_pagemap_cache *cache);
+
+void drm_pagemap_cache_set_pagemap(struct drm_pagemap_cache *cache, struct drm_pagemap *dpagemap);
+
+struct drm_pagemap *drm_pagemap_get_from_cache_if_active(struct drm_pagemap_cache *cache);
+#endif

Signature · 1 line

2.51.0

Quoted 509 lines · Thomas Hellström

Pagemaps are costly to set up and tear down, and they consume a lot of system memory for the struct pages. Ideally they should be created only when needed.

Add a caching mechanism to allow doing just that: Create the drm_pagemaps
when needed for migration. Keep them around to avoid destruction and
re-creation latencies and destroy inactive/unused drm_pagemaps on memory
pressure using a shrinker.

Only add the helper functions. They will be hooked up to the xe driver
in the upcoming patch.

Signed-off-by Thomas Hellström <[email protected]>

---
 drivers/gpu/drm/Makefile           |   3 +-
 drivers/gpu/drm/drm_pagemap.c      |  79 +++++-
 drivers/gpu/drm/drm_pagemap_util.c | 426
+++++++++++++++++++++++++++++
 include/drm/drm_pagemap.h          |  53 +++-
 include/drm/drm_pagemap_util.h     |  25 ++

 5 files changed, 569 insertions(+), 17 deletions(-)

 create mode 100644 drivers/gpu/drm/drm_pagemap_util.c create mode 100644 include/drm/drm_pagemap_util.h

drivers/gpu/drm/Makefile +0 -0

index c2672f369aed..cdca68fd9f23 100644
@@ -107,7 +107,8 @@ obj-$(CONFIG_DRM_GPUVM) += drm_gpuvm.o

 drm_gpusvm_helper-y := \
  drm_gpusvm.o\

drivers/gpu/drm/Makefile +2 -1

-	drm_pagemap.o
+	drm_pagemap.o\
+	drm_pagemap_util.o

 obj-$(CONFIG_DRM_GPUSVM) += drm_gpusvm_helper.o

 obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o

drivers/gpu/drm/drm_pagemap.c +1 -0

index fb18a80d6a1c..5ca5b2b53bc1 100644
@@ -8,6 +8,7 @@
  #include <linux/pagemap.h>
  #include <drm/drm_drv.h>
  #include <drm/drm_pagemap.h>
+#include <drm/drm_pagemap_util.h>
  #include <drm/drm_print.h>

 /**

drivers/gpu/drm/drm_pagemap.c +1 -1

@@ -578,7 +579,7 @@ static void drm_pagemap_release(struct kref
 *ref)
  	 * pagemap provider drm_device and its module.
  	 */
  	dpagemap->dev_hold = NULL;
-	kfree(dpagemap);
+	drm_pagemap_shrinker_add(dpagemap);
  	llist_add(&dev_hold->link, &drm_pagemap_unhold_list);
  	schedule_work(&drm_pagemap_work);

  /*

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -628,6 +629,58 @@ drm_pagemap_dev_hold(struct drm_pagemap
 *dpagemap)
  	return dev_hold;
  }

drivers/gpu/drm/drm_pagemap.c +52 -0

+/**
+ * drm_pagemap_reinit() - Reinitialize a drm_pagemap
+ * @dpagemap: The drm_pagemap to reinitialize
+ *
+ * Reinitialize a drm_pagemap, for which drm_pagemap_release
+ * has already been called. This interface is intended for the
+ * situation where the driver caches a destroyed drm_pagemap.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+int drm_pagemap_reinit(struct drm_pagemap *dpagemap)
+{
+	dpagemap->dev_hold = drm_pagemap_dev_hold(dpagemap);
+	if (IS_ERR(dpagemap->dev_hold))
+		return PTR_ERR(dpagemap->dev_hold);
+
+	kref_init(&dpagemap->ref);
+	return 0;
+}
+EXPORT_SYMBOL(drm_pagemap_reinit);
+
+/**
+ * drm_pagemap_init() - Initialize a pre-allocated drm_pagemap
+ * @dpagemap: The drm_pagemap to initialize.
+ * @pagemap: The associated dev_pagemap providing the device
+ * private pages.
+ * @drm: The drm device. The drm_pagemap holds a reference on the
+ * drm_device and the module owning the drm_device until
+ * drm_pagemap_release(). This facilitates drm_pagemap exporting.
+ * @ops: The drm_pagemap ops.
+ *
+ * Initialize and take an initial reference on a drm_pagemap.
+ * After successful return, use drm_pagemap_put() to destroy.
+ *
+ ** Return: 0 on success, negative error code on error.
+ */
+int drm_pagemap_init(struct drm_pagemap *dpagemap,
+		     struct dev_pagemap *pagemap,
+		     struct drm_device *drm,
+		     const struct drm_pagemap_ops *ops)
+{
+	kref_init(&dpagemap->ref);
+	dpagemap->ops = ops;
+	dpagemap->pagemap = pagemap;
+	dpagemap->drm = drm;
+	dpagemap->cache = NULL;
+	INIT_LIST_HEAD(&dpagemap->shrink_link);
+
+	return drm_pagemap_reinit(dpagemap);
+}
+EXPORT_SYMBOL(drm_pagemap_init);
+

 /**
  * drm_pagemap_create() - Create a struct drm_pagemap.
  * @drm: Pointer to a struct drm_device providing the device- private memory.

drivers/gpu/drm/drm_pagemap.c +1 -1

@@ -645,22 +698,14 @@ drm_pagemap_create(struct drm_device *drm,
  		   const struct drm_pagemap_ops *ops)
  {
  	struct drm_pagemap *dpagemap = kzalloc(sizeof(*dpagemap),
 GFP_KERNEL);
-	struct drm_pagemap_dev_hold *dev_hold;
+	int err;

  if (!dpagemap)
  return ERR_PTR(-ENOMEM);

drivers/gpu/drm/drm_pagemap.c +3 -11

-	kref_init(&dpagemap->ref);
-	dpagemap->drm = drm;
-	dpagemap->ops = ops;
-	dpagemap->pagemap = pagemap;
-
-	dev_hold = drm_pagemap_dev_hold(dpagemap);
-	if (IS_ERR(dev_hold)) {
-		kfree(dpagemap);
-		return ERR_CAST(dev_hold);
-	}
-	dpagemap->dev_hold = dev_hold;
+	err = drm_pagemap_init(dpagemap, pagemap, drm, ops);
+	if (err)
+		return ERR_PTR(err);

  return dpagemap;
 }

drivers/gpu/drm/drm_pagemap.c +0 -0

@@ -1023,6 +1068,14 @@ int drm_pagemap_populate_mm(struct
 drm_pagemap *dpagemap,
  }
  EXPORT_SYMBOL(drm_pagemap_populate_mm);

+void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim)

drivers/gpu/drm/drm_pagemap.c +3 -0

+{
+	if (dpagemap->ops->destroy)
+		dpagemap->ops->destroy(dpagemap,

is_atomic_or_reclaim);

drivers/gpu/drm/drm_pagemap.c +4 -0

+	else
+		kfree(dpagemap);
+}
+

 static void drm_pagemap_exit(void)
 {
  flush_work(&drm_pagemap_work);

drivers/gpu/drm/drm_pagemap_util.c new file +17 -0

new file mode 100644
index 000000000000..e1a1d6bf25f4
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright © 2025 Intel Corporation
+ */
+
+#include <drm/drm_drv.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_pagemap.h>
+#include <drm/drm_pagemap_util.h>
+#include <drm/drm_print.h>
+
+/**
+ * struct drm_pagemap_cache - Lookup structure for pagemaps
+ *
+ * Structure to keep track of active (refcount > 1) and inactive
+ * (refcount == 0) pagemaps. Inactive pagemaps can be made active
+ * again by waiting for the @queued completion (indicating that

the
+ * pagemap has been put on the @shrinker's list of shrinkable
+ * pagemaps, and then successfully removing it from @shrinker's
+ * list. The latter may fail if the shrinker is already in the
+ * process of freeing the pagemap. A struct drm_pagemap_cache can
+ * hold a single struct drm_pagemap.
+ */

drivers/gpu/drm/drm_pagemap_util.c new file +2 -0

+struct drm_pagemap_cache {
+	/** @lookup_mutex: Mutex making the lookup process atomic

*/

drivers/gpu/drm/drm_pagemap_util.c new file +4 -0

+	struct mutex lookup_mutex;
+	/** @lock: Lock protecting the @dpagemap pointer */
+	spinlock_t lock;
+	/** @shrinker: Pointer to the shrinker used for this

cache. Immutable. */

drivers/gpu/drm/drm_pagemap_util.c new file +2 -0

+	struct drm_pagemap_shrinker *shrinker;
+	/** @dpagemap: Non-refcounted pointer to the drm_pagemap

*/

drivers/gpu/drm/drm_pagemap_util.c new file +3 -0

+	struct drm_pagemap *dpagemap;
+	/**
+	 * @queued: Signals when an inactive drm_pagemap has been

put on

drivers/gpu/drm/drm_pagemap_util.c new file +7 -0

+	 * @shrinker's list.
+	 */
+	struct completion queued;
+};
+
+/**
+ * struct drm_pagemap_shrinker - Shrinker to remove unused

pagemaps
+ */

drivers/gpu/drm/drm_pagemap_util.c new file +8 -0

+struct drm_pagemap_shrinker {
+	/** @drm: Pointer to the drm device. */
+	struct drm_device *drm;
+	/** @lock: Spinlock to protect the @dpagemaps list. */
+	spinlock_t lock;
+	/** @dpagemaps: List of unused dpagemaps. */
+	struct list_head dpagemaps;
+	/** @num_dpagemaps: Number of unused dpagemaps in

@dpagemaps. */

drivers/gpu/drm/drm_pagemap_util.c new file +6 -0

+	atomic_t num_dpagemaps;
+	/** @shrink: Pointer to the struct shrinker. */
+	struct shrinker *shrink;
+};
+
+static bool drm_pagemap_shrinker_cancel(struct drm_pagemap

*dpagemap);

drivers/gpu/drm/drm_pagemap_util.c new file +7 -0

+
+static void drm_pagemap_cache_fini(void *arg)
+{
+	struct drm_pagemap_cache *cache = arg;
+	struct drm_pagemap *dpagemap;
+
+	drm_dbg(cache->shrinker->drm, "Destroying dpagemap

cache.\n");

drivers/gpu/drm/drm_pagemap_util.c new file +23 -0

+	spin_lock(&cache->lock);
+	dpagemap = cache->dpagemap;
+	if (!dpagemap) {
+		spin_unlock(&cache->lock);
+		goto out;
+	}
+
+	if (drm_pagemap_shrinker_cancel(dpagemap)) {
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+		drm_pagemap_destroy(dpagemap, false);
+	}
+
+out:
+	mutex_destroy(&cache->lookup_mutex);
+	kfree(cache);
+}
+
+/**
+ * drm_pagemap_cache_create_devm() - Create a drm_pagemap_cache
+ * @shrinker: Pointer to a struct drm_pagemap_shrinker.
+ *
+ * Create a device-managed drm_pagemap cache. The cache is

automatically
+ * destroyed on struct device removal, at which point any *inactive*
+ * drm_pagemap's are destroyed.
+ *
+ * Return: Pointer to a struct drm_pagemap_cache on success. Error pointer
+ * on failure.
+ */
+struct drm_pagemap_cache *drm_pagemap_cache_create_devm(struct drm_pagemap_shrinker *shrinker)

drivers/gpu/drm/drm_pagemap_util.c new file +2 -0

+{
+	struct drm_pagemap_cache *cache = kzalloc(sizeof(*cache),

GFP_KERNEL);

drivers/gpu/drm/drm_pagemap_util.c new file +10 -0

+	int err;
+
+	if (!cache)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&cache->lookup_mutex);
+	spin_lock_init(&cache->lock);
+	cache->shrinker = shrinker;
+	init_completion(&cache->queued);
+	err = devm_add_action_or_reset(shrinker->drm->dev,

drm_pagemap_cache_fini, cache);

drivers/gpu/drm/drm_pagemap_util.c new file +16 -0

+	if (err)
+		return ERR_PTR(err);
+
+	return cache;
+}
+EXPORT_SYMBOL(drm_pagemap_cache_create_devm);
+
+/**
+ * DOC: Cache lookup
+ *
+ * Cache lookup should be done under a locked mutex, so that a
+ * failed drm_pagemap_get_from_cache() and a following
+ * drm_pagemap_cache_setpagemap() are carried out as an atomic
+ * operation WRT other lookups. Otherwise, racing lookups may
+ * unnecessarily concurrently create pagemaps to fulfill a
+ * failed lookup. The API provides two functions to perform this

lock,
+ * drm_pagemap_lock_lookup() and drm_pagemap_unlock_lookup() and they
+ * should be used in the following way:
+ *
+ * .. code-block:: c
+ *
+ * drm_pagemap_lock_lookup(cache);
+ * dpagemap = drm_pagemap_get_from_cache(cache);
+ * if (dpagemap)
+ * goto out_unlock;
+ *
+ * dpagemap = driver_create_new_dpagemap();
+ * if (!IS_ERR(dpagemap))
+ * drm_pagemap_cache_set_pagemap(cache,
dpagemap);
+ *
+ *     out_unlock:
+ * drm_pagemap_unlock_lookup(cache);
+ */

drivers/gpu/drm/drm_pagemap_util.c new file +3 -0

+
+/**
+ * drm_pagemap_cache_lock_lookup() Lock a drm_pagemap_cache for

lookup
+ * @cache: The drm_pagemap_cache to lock.
+ *
+ * Return: %-EINTR if interrupted while blocking. %0 otherwise.
+ */

drivers/gpu/drm/drm_pagemap_util.c new file +8 -0

+int drm_pagemap_cache_lock_lookup(struct drm_pagemap_cache *cache)
+{
+	return mutex_lock_interruptible(&cache->lookup_mutex);
+}
+EXPORT_SYMBOL(drm_pagemap_cache_lock_lookup);
+
+/**
+ * drm_pagemap_cache_unlock_lookup() Unlock a drm_pagemap_cache

after lookup
+ * @cache: The drm_pagemap_cache to unlock.
+ */
+void drm_pagemap_cache_unlock_lookup(struct drm_pagemap_cache *cache)

drivers/gpu/drm/drm_pagemap_util.c new file +10 -0

+{
+	mutex_unlock(&cache->lookup_mutex);
+}
+EXPORT_SYMBOL(drm_pagemap_cache_unlock_lookup);
+
+/**
+ * drm_pagemap_get_from_cache() -  Lookup of drm_pagemaps.
+ * @cache: The cache used for lookup.
+ *
+ * If an active pagemap is present in the cache, it is immediately

returned.
+ * If an inactive pagemap is present, it's removed from the shrinker list and
+ * an attempt is made to make it active.
+ * If no pagemap present or the attempt to make it active failed, %NULL is returned
+ * to indicate to the caller to create a new drm_pagemap and insert it into
+ * the cache.
+ *
+ * Return: A reference-counted pointer to a drm_pagemap if successful. An error
+ * pointer if an error occurred, or %NULL if no drm_pagemap was found and
+ * the caller should insert a new one.
+ */
+struct drm_pagemap *drm_pagemap_get_from_cache(struct drm_pagemap_cache *cache)

drivers/gpu/drm/drm_pagemap_util.c new file +21 -0

+{
+	struct drm_pagemap *dpagemap;
+	int err;
+
+	lockdep_assert_held(&cache->lookup_mutex);
+retry:
+	spin_lock(&cache->lock);
+	dpagemap = cache->dpagemap;
+	if (drm_pagemap_get_unless_zero(dpagemap)) {
+		spin_unlock(&cache->lock);
+		return dpagemap;
+	}
+
+	if (!dpagemap) {
+		spin_unlock(&cache->lock);
+		return NULL;
+	}
+
+	if (!try_wait_for_completion(&cache->queued)) {
+		spin_unlock(&cache->lock);
+		err = wait_for_completion_interruptible(&cache-

Quoted 1 line

queued);

drivers/gpu/drm/drm_pagemap_util.c new file +26 -0

+		if (err)
+			return ERR_PTR(err);
+		goto retry;
+	}
+
+	if (drm_pagemap_shrinker_cancel(dpagemap)) {
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+		err = drm_pagemap_reinit(dpagemap);
+		if (err) {
+			drm_pagemap_destroy(dpagemap, false);
+			return ERR_PTR(err);
+		}
+		drm_pagemap_cache_set_pagemap(cache, dpagemap);
+	} else {
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+		dpagemap = NULL;
+	}
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_get_from_cache);
+
+/**
+ * drm_pagemap_cache_set_pagemap() - Assign a drm_pagemap to a

drm_pagemap_cache
+ * @cache: The cache to assign the drm_pagemap to.
+ * @dpagemap: The drm_pagemap to assign.
+ *
+ * The function must be called to populate a drm_pagemap_cache only
+ * after a call to drm_pagemap_get_from_cache() returns NULL.
+ */
+void drm_pagemap_cache_set_pagemap(struct drm_pagemap_cache *cache, struct drm_pagemap *dpagemap)

drivers/gpu/drm/drm_pagemap_util.c new file +15 -0

+{
+	struct drm_device *drm = dpagemap->drm;
+
+	lockdep_assert_held(&cache->lookup_mutex);
+	spin_lock(&cache->lock);
+	dpagemap->cache = cache;
+	swap(cache->dpagemap, dpagemap);
+	reinit_completion(&cache->queued);
+	spin_unlock(&cache->lock);
+	drm_WARN_ON(drm, !!dpagemap);
+}
+EXPORT_SYMBOL(drm_pagemap_cache_set_pagemap);
+
+/**
+ * drm_pagemap_get_from_cache_if_active() - Quick lookup of active

drm_pagemaps
+ * @cache: The cache to lookup from.
+ *
+ * Function that should be used to lookup a drm_pagemap that is already active.
+ * (refcount > 0).
+ *
+ * Return: A pointer to the cache's drm_pagemap if it's active; %NULL otherwise.
+ */
+struct drm_pagemap *drm_pagemap_get_from_cache_if_active(struct drm_pagemap_cache *cache)

drivers/gpu/drm/drm_pagemap_util.c new file +12 -0

+{
+	struct drm_pagemap *dpagemap;
+
+	spin_lock(&cache->lock);
+	dpagemap = drm_pagemap_get_unless_zero(cache->dpagemap);
+	spin_unlock(&cache->lock);
+
+	return dpagemap;
+}
+EXPORT_SYMBOL(drm_pagemap_get_from_cache_if_active);
+
+static bool drm_pagemap_shrinker_cancel(struct drm_pagemap

*dpagemap)

drivers/gpu/drm/drm_pagemap_util.c new file +18 -0

+{
+	struct drm_pagemap_cache *cache = dpagemap->cache;
+	struct drm_pagemap_shrinker *shrinker = cache->shrinker;
+
+	spin_lock(&shrinker->lock);
+	if (list_empty(&dpagemap->shrink_link)) {
+		spin_unlock(&shrinker->lock);
+		return false;
+	}
+
+	list_del_init(&dpagemap->shrink_link);
+	atomic_dec(&shrinker->num_dpagemaps);
+	spin_unlock(&shrinker->lock);
+	return true;
+}
+
+/**
+ * drm_pagemap_shrinker_add() - Add a drm_pagemap to the shrinker

list or destroy
+ * @dpagemap: The drm_pagemap.
+ *
+ * If @dpagemap is associated with a &struct drm_pagemap_cache AND the
+ * struct device backing the drm device is still alive, add @dpagemap to
+ * the &struct drm_pagemap_shrinker list of shrinkable drm_pagemaps.

Quoted 8 lines

+ *
+ * Otherwise destroy the pagemap directly using drm_pagemap_destroy().
+ *
+ * This is an internal function which is not intended to be exposed to drivers.
+ */
+void drm_pagemap_shrinker_add(struct drm_pagemap *dpagemap)

Not a full review - slowly wrapping my head around the first 6 patches
but one quick question.

This is called from drm_pagemap_put. How do we know what type of context
we're in? It seems like this could be called from either process context
or atomic context (e.g., via drm_pagemap_zdd_destroy through drm_pagemap_page_free). This code doesn’t appear to work in atomic contexts—if I recall correctly, drm_dev_enter can’t be called from atomic context. Also, we're missing irqsave on the spinlock.

We had a worker for ZDD destroy at one point—should we revive that? If
we did, I think we could safely enforce a rule that drm_pagemap operations must only be called from process context.

Matt

Quoted 316 lines

drivers/gpu/drm/drm_pagemap_util.c new file +22 -0

+{
+	struct drm_pagemap_cache *cache;
+	struct drm_pagemap_shrinker *shrinker;
+	int idx;
+
+	/*
+	 * The pagemap cache and shrinker are disabled at
+	 * pci device remove time. After that, dpagemaps
+	 * are freed directly.
+	 */
+	if (!drm_dev_enter(dpagemap->drm, &idx))
+		goto out_no_cache;
+
+	cache = dpagemap->cache;
+	if (!cache) {
+		drm_dev_exit(idx);
+		goto out_no_cache;
+	}
+
+	shrinker = cache->shrinker;
+	spin_lock(&shrinker->lock);
+	list_add_tail(&dpagemap->shrink_link, &shrinker-

Quoted 1 line

dpagemaps);

drivers/gpu/drm/drm_pagemap_util.c new file +12 -0

+	atomic_inc(&shrinker->num_dpagemaps);
+	spin_unlock(&shrinker->lock);
+	complete_all(&cache->queued);
+	drm_dev_exit(idx);
+	return;
+
+out_no_cache:
+	drm_pagemap_destroy(dpagemap, true);
+}
+
+static unsigned long
+drm_pagemap_shrinker_count(struct shrinker *shrink, struct

shrink_control *sc)

drivers/gpu/drm/drm_pagemap_util.c new file +2 -0

+{
+	struct drm_pagemap_shrinker *shrinker = shrink-

Quoted 1 line

private_data;

drivers/gpu/drm/drm_pagemap_util.c new file +1 -0

+	unsigned long count = atomic_read(&shrinker-

Quoted 1 line

num_dpagemaps);

drivers/gpu/drm/drm_pagemap_util.c new file +6 -0

+
+	return count ? : SHRINK_EMPTY;
+}
+
+static unsigned long
+drm_pagemap_shrinker_scan(struct shrinker *shrink, struct

shrink_control *sc)

drivers/gpu/drm/drm_pagemap_util.c new file +2 -0

+{
+	struct drm_pagemap_shrinker *shrinker = shrink-

Quoted 1 line

private_data;

drivers/gpu/drm/drm_pagemap_util.c new file +8 -0

+	struct drm_pagemap *dpagemap;
+	struct drm_pagemap_cache *cache;
+	unsigned long nr_freed = 0;
+
+	sc->nr_scanned = 0;
+	spin_lock(&shrinker->lock);
+	do {
+		dpagemap = list_first_entry_or_null(&shrinker-

Quoted 1 line

dpagemaps, typeof(*dpagemap),

drivers/gpu/drm/drm_pagemap_util.c new file +17 -0

+						    shrink_link);
+		if (!dpagemap)
+			break;
+
+		atomic_dec(&shrinker->num_dpagemaps);
+		list_del_init(&dpagemap->shrink_link);
+		spin_unlock(&shrinker->lock);
+
+		sc->nr_scanned++;
+		nr_freed++;
+
+		cache = dpagemap->cache;
+		spin_lock(&cache->lock);
+		cache->dpagemap = NULL;
+		spin_unlock(&cache->lock);
+
+		drm_dbg(dpagemap->drm, "Shrinking dpagemap %p.\n",

dpagemap);

drivers/gpu/drm/drm_pagemap_util.c new file +14 -0

+		drm_pagemap_destroy(dpagemap, true);
+		spin_lock(&shrinker->lock);
+	} while (sc->nr_scanned < sc->nr_to_scan);
+	spin_unlock(&shrinker->lock);
+
+	return sc->nr_scanned ? nr_freed : SHRINK_STOP;
+}
+
+static void drm_pagemap_shrinker_fini(void *arg)
+{
+	struct drm_pagemap_shrinker *shrinker = arg;
+
+	drm_dbg(shrinker->drm, "Destroying dpagemap shrinker.\n");
+	drm_WARN_ON(shrinker->drm, !!atomic_read(&shrinker-

Quoted 1 line

num_dpagemaps));

drivers/gpu/drm/drm_pagemap_util.c new file +6 -0

+	shrinker_free(shrinker->shrink);
+	kfree(shrinker);
+}
+
+/**
+ * drm_pagemap_shrinker_create_devm() - Create and register a

pagemap shrinker
+ * @drm: The drm device
+ *
+ * Create and register a pagemap shrinker that shrinks unused pagemaps
+ * and thereby reduces memory footprint.
+ * The shrinker is drm_device managed and unregisters itself when
+ * the drm device is removed.
+ *
+ * Return: %0 on success, negative error code on failure.
+ */
+struct drm_pagemap_shrinker *drm_pagemap_shrinker_create_devm(struct drm_device *drm)

drivers/gpu/drm/drm_pagemap_util.c new file +10 -0

+{
+	struct drm_pagemap_shrinker *shrinker;
+	struct shrinker *shrink;
+	int err;
+
+	shrinker = kzalloc(sizeof(*shrinker), GFP_KERNEL);
+	if (!shrinker)
+		return ERR_PTR(-ENOMEM);
+
+	shrink = shrinker_alloc(0, "drm-drm_pagemap:%s", drm-

Quoted 1 line

unique);

drivers/gpu/drm/drm_pagemap_util.c new file +15 -0

+	if (!shrink) {
+		kfree(shrinker);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	spin_lock_init(&shrinker->lock);
+	INIT_LIST_HEAD(&shrinker->dpagemaps);
+	shrinker->drm = drm;
+	shrinker->shrink = shrink;
+	shrink->count_objects = drm_pagemap_shrinker_count;
+	shrink->scan_objects = drm_pagemap_shrinker_scan;
+	shrink->private_data = shrinker;
+	shrinker_register(shrink);
+
+	err = devm_add_action_or_reset(drm->dev,

drm_pagemap_shrinker_fini, shrinker);

drivers/gpu/drm/drm_pagemap_util.c new file +6 -0

+	if (err)
+		return ERR_PTR(err);
+
+	return shrinker;
+}
+EXPORT_SYMBOL(drm_pagemap_shrinker_create_devm);

include/drm/drm_pagemap.h +0 -0

index 5cfe54331ba7..4b9af5e785c6 100644
@@ -9,6 +9,7 @@
  #define NR_PAGES(order) (1U << (order))

 struct drm_pagemap;
+struct drm_pagemap_cache;
 struct drm_pagemap_dev_hold;
 struct drm_pagemap_zdd;
 struct device;

include/drm/drm_pagemap.h +2 -0

@@ -124,6 +125,25 @@ struct drm_pagemap_ops {
  			   unsigned long start, unsigned long end,
  			   struct mm_struct *mm,
  			   unsigned long timeslice_ms);
+	/**
+	 * @destroy: Destroy the drm_pagemap and associated

resources.

include/drm/drm_pagemap.h +7 -0

+	 * @dpagemap: The drm_pagemap to destroy.
+	 * @is_atomic_or_reclaim: The function may be called from
+	 * atomic- or reclaim context.
+	 *
+	 * The implementation should take care not to attempt to
+	 * destroy resources that may already have been destroyed
+	 * using devm_ callbacks, since this function may be

called

include/drm/drm_pagemap.h +2 -0

+	 * after the underlying struct device has been unbound.
+	 * If the implementation defers the execution to a work

item

include/drm/drm_pagemap.h +1 -0

+	 * to avoid locking issues, then it must make sure the

work

include/drm/drm_pagemap.h +1 -0

+	 * items are flushed before module exit. If the destroy

call

include/drm/drm_pagemap.h +2 -0

+	 * happens after the provider's pci_remove() callback has
+	 * been executed, a module reference and drm device

reference is

include/drm/drm_pagemap.h +4 -0

+	 * held across the destroy callback.
+	 */
+	void (*destroy)(struct drm_pagemap *dpagemap,
+			bool is_atomic_or_reclaim);

 };

 /**

include/drm/drm_pagemap.h +1 -0

@@ -135,6 +155,10 @@ struct drm_pagemap_ops {
   * @pagemap: Pointer to the underlying dev_pagemap.
   * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for
   * device referencing.
+ * @cache: Back-pointer to the &struct drm_pagemap_cache used for

this
+ * &struct drm_pagemap. May be NULL if no cache is used.
+ * @shrink_link: Link into the shrinker's list of drm_pagemaps. Only
+ * used if also using a pagemap cache.
  */
 struct drm_pagemap {
  const struct drm_pagemap_ops *ops;

include/drm/drm_pagemap.h +2 -0

@@ -142,6 +166,8 @@ struct drm_pagemap {
  	struct drm_device *drm;
  	struct dev_pagemap *pagemap;
  	struct drm_pagemap_dev_hold *dev_hold;
+	struct drm_pagemap_cache *cache;
+	struct list_head shrink_link;
  };

 struct drm_pagemap_devmem;

include/drm/drm_pagemap.h +0 -0

@@ -210,6 +236,11 @@ struct drm_pagemap_devmem_ops {
  			   unsigned long npages);
  };

include/drm/drm_pagemap.h +5 -0

+int drm_pagemap_init(struct drm_pagemap *dpagemap,
+		     struct dev_pagemap *pagemap,
+		     struct drm_device *drm,
+		     const struct drm_pagemap_ops *ops);
+

 struct drm_pagemap *drm_pagemap_create(struct drm_device *drm, struct dev_pagemap *pagemap,
         const struct drm_pagemap_ops *ops);

include/drm/drm_pagemap.h +0 -0

@@ -228,9 +259,9 @@ static inline void drm_pagemap_put(struct
 drm_pagemap *dpagemap)

 /**
  * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap
- * @dpagemap: Pointer to the struct drm_pagemap.
+ * @dpagemap: Pointer to the struct drm_pagemap, or NULL. *
- * Return: Pointer to the struct drm_pagemap.
+ * Return: Pointer to the struct drm_pagemap, or NULL. */
 static inline struct drm_pagemap * drm_pagemap_get(struct drm_pagemap *dpagemap)

include/drm/drm_pagemap.h +0 -0

@@ -241,6 +272,20 @@ drm_pagemap_get(struct drm_pagemap *dpagemap)
  	return dpagemap;
  }

+/**
+ * drm_pagemap_get_unless_zero() - Obtain a reference on a struct drm_pagemap
+ * unless the current reference count is zero.
+ * @dpagemap: Pointer to the drm_pagemap or NULL.
+ *
+ * Return: A pointer to @dpagemap if the reference count was successfully
+ * incremented. NULL if @dpagemap was NULL, or its refcount was 0.
+ */

include/drm/drm_pagemap.h +4 -0

+static inline struct drm_pagemap * __must_check
+drm_pagemap_get_unless_zero(struct drm_pagemap *dpagemap)
+{
+	return (dpagemap && kref_get_unless_zero(&dpagemap->ref))

? dpagemap : NULL;

include/drm/drm_pagemap.h +2 -0

+}
+

 /**
  * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation
  *

include/drm/drm_pagemap.h +0 -0

@@ -284,5 +329,7 @@ int drm_pagemap_populate_mm(struct drm_pagemap
 *dpagemap,
  			    struct mm_struct *mm,
  			    unsigned long timeslice_ms);

include/drm/drm_pagemap.h +1 -1

-#endif
+void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool

is_atomic_or_reclaim);

include/drm/drm_pagemap.h +2 -0

+int drm_pagemap_reinit(struct drm_pagemap *dpagemap);
+#endif

include/drm/drm_pagemap_util.h new file +12 -0

new file mode 100644
index 000000000000..292244d429ee
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef _DRM_PAGEMAP_UTIL_H_
+#define _DRM_PAGEMAP_UTIL_H_
+
+struct drm_device;
+struct drm_pagemap;
+struct drm_pagemap_cache;
+struct drm_pagemap_shrinker;
+
+void drm_pagemap_shrinker_add(struct drm_pagemap *dpagemap);
+
+int drm_pagemap_cache_lock_lookup(struct drm_pagemap_cache

*cache);

include/drm/drm_pagemap_util.h new file +2 -0

+
+void drm_pagemap_cache_unlock_lookup(struct drm_pagemap_cache

*cache);

include/drm/drm_pagemap_util.h new file +2 -0

+
+struct drm_pagemap_shrinker

*drm_pagemap_shrinker_create_devm(struct drm_device *drm);

include/drm/drm_pagemap_util.h new file +2 -0

+
+struct drm_pagemap_cache *drm_pagemap_cache_create_devm(struct

drm_pagemap_shrinker *shrinker);

include/drm/drm_pagemap_util.h new file +2 -0

+
+struct drm_pagemap *drm_pagemap_get_from_cache(struct

drm_pagemap_cache *cache);

include/drm/drm_pagemap_util.h new file +2 -0

+
+void drm_pagemap_cache_set_pagemap(struct drm_pagemap_cache

*cache, struct drm_pagemap *dpagemap);

include/drm/drm_pagemap_util.h new file +2 -0

+
+struct drm_pagemap *drm_pagemap_get_from_cache_if_active(struct

drm_pagemap_cache *cache);
+#endif

Signature · 1 line

2.51.0