drm/v3d: Add support for write-back cacheable BOs

index fe3436d1235d..4183def83cdd 100644
@@ -905,22 +905,26 @@ unsigned long drm_prime_get_contiguous_size(struct sg_table *sgt)
 EXPORT_SYMBOL(drm_prime_get_contiguous_size);
 
 /**
- * drm_gem_prime_export - helper library implementation of the export callback
+ * drm_gem_prime_export_with_ops - export a GEM object with custom dma_buf_ops
  * @obj: GEM object to export
+ * @dma_buf_ops: &dma_buf_ops of the exported DMA-BUF
  * @flags: flags like DRM_CLOEXEC and DRM_RDWR
  *
- * This is the implementation of the &drm_gem_object_funcs.export functions for GEM drivers
- * using the PRIME helpers. It is used as the default in
- * drm_gem_prime_handle_to_fd().
+ * Same as drm_gem_prime_export(), but with driver-provided &dma_buf_ops.
+ * Drivers using this can implement &drm_driver.gem_prime_is_exporter for
+ * the PRIME import helpers to recognize their own buffers.
+ *
+ * Return: the exported DMA-BUF on success, or an ERR_PTR() on failure.
  */
-struct dma_buf *drm_gem_prime_export(struct drm_gem_object *obj,
-				     int flags)
+struct dma_buf *drm_gem_prime_export_with_ops(struct drm_gem_object *obj,
+					      const struct dma_buf_ops *dma_buf_ops,
+					      int flags)
 {
 	struct drm_device *dev = obj->dev;
 	struct dma_buf_export_info exp_info = {
 		.exp_name = KBUILD_MODNAME, /* white lie for debug */
 		.owner = dev->driver->fops->owner,
-		.ops = &drm_gem_prime_dmabuf_ops,
+		.ops = dma_buf_ops,
 		.size = obj->size,
 		.flags = flags,
 		.priv = obj,

@@ -929,24 +933,47 @@ struct dma_buf *drm_gem_prime_export(struct drm_gem_object *obj,
 
 	return drm_gem_dmabuf_export(dev, &exp_info);
 }
+EXPORT_SYMBOL(drm_gem_prime_export_with_ops);
+
+/**
+ * drm_gem_prime_export - helper library implementation of the export callback
+ * @obj: GEM object to export
+ * @flags: flags like DRM_CLOEXEC and DRM_RDWR
+ *
+ * This is the implementation of the &drm_gem_object_funcs.export function for
+ * GEM drivers using the PRIME helpers. It is used as the default in
+ * drm_gem_prime_handle_to_fd().
+ *
+ * Return: the exported DMA-BUF on success, or an ERR_PTR() on failure.
+ */
+struct dma_buf *drm_gem_prime_export(struct drm_gem_object *obj,
+				     int flags)
+{
+	return drm_gem_prime_export_with_ops(obj, &drm_gem_prime_dmabuf_ops, flags);
+}
 EXPORT_SYMBOL(drm_gem_prime_export);
 
 
 /**
- * drm_gem_is_prime_exported_dma_buf -
- * checks if the DMA-BUF was exported from a GEM object belonging to @dev.
+ * drm_gem_is_prime_exported_dma_buf - check if a DMA-BUF was exported by @dev
  * @dev: drm_device to check against
  * @dma_buf: dma-buf object to import
  *
+ * DMA-BUFs exported with custom &dma_buf_ops are recognized through
+ * &drm_driver.gem_prime_is_exporter.
+ *
  * Return: true if the DMA-BUF was exported from a GEM object belonging
  * to @dev, false otherwise.
  */
-
 bool drm_gem_is_prime_exported_dma_buf(struct drm_device *dev,
 				       struct dma_buf *dma_buf)
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 
+	if (dev->driver->gem_prime_is_exporter &&
+	    dev->driver->gem_prime_is_exporter(dev, dma_buf))
+		return true;
+
 	return (dma_buf->ops == &drm_gem_prime_dmabuf_ops) && (obj->dev == dev);
 }
 EXPORT_SYMBOL(drm_gem_is_prime_exported_dma_buf);

index b23830494ed4..d613740a06d4 100644
@@ -310,6 +310,17 @@ struct drm_driver {
 	 */
 	struct drm_gem_object * (*gem_prime_import)(struct drm_device *dev,
 				struct dma_buf *dma_buf);
+	/**
+	 * @gem_prime_is_exporter:
+	 *
+	 * Returns true if @dma_buf was exported from a GEM object of @dev.
+	 * Drivers exporting with a custom &dma_buf_ops can implement this
+	 * hook to let the PRIME import helpers recognize their own buffers.
+	 *
+	 * This callback is optional.
+	 */
+	bool (*gem_prime_is_exporter)(struct drm_device *dev,
+				      struct dma_buf *dma_buf);
 	/**
 	 * @gem_prime_import_sg_table:
 	 *

index f50f862f0d8b..9b704e3ca919 100644
@@ -53,6 +53,7 @@ struct device;
 
 struct dma_buf_export_info;
 struct dma_buf;
+struct dma_buf_ops;
 struct dma_buf_attachment;
 struct iosys_map;
 

@@ -96,6 +97,9 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev,
 				       struct page **pages, unsigned int nr_pages);
 struct dma_buf *drm_gem_prime_export(struct drm_gem_object *obj,
 				     int flags);
+struct dma_buf *drm_gem_prime_export_with_ops(struct drm_gem_object *obj,
+					      const struct dma_buf_ops *dma_buf_ops,
+					      int flags);
 
 unsigned long drm_prime_get_contiguous_size(struct sg_table *sgt);
 

index d654a22a30de..13c4a68fdde0 100644
@@ -993,6 +993,121 @@ struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev,
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_no_map);
 
+/**
+ * drm_gem_shmem_prime_map_dma_buf - Default map_dma_buf() for exported buffers
+ * @attach: attachment
+ * @dir: type of access done by this attachment
+ *
+ * Default implementation for dma_buf_ops::map_dma_buf(). This is just a wrapper
+ * around drm_gem_map_dma_buf() that lets us set the dma_buf_attachment::priv
+ * to the sgt so that drm_gem_shmem_prime_{begin,end}_cpu_access() can sync
+ * around CPU accesses.
+ *
+ * Return: the mapped SG table on success, or an ERR_PTR() on failure.
+ */
+struct sg_table *
+drm_gem_shmem_prime_map_dma_buf(struct dma_buf_attachment *attach,
+				enum dma_data_direction dir)
+{
+	struct sg_table *sgt = drm_gem_map_dma_buf(attach, dir);
+
+	if (!IS_ERR(sgt))
+		attach->priv = sgt;
+
+	return sgt;
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_map_dma_buf);
+
+/**
+ * drm_gem_shmem_prime_unmap_dma_buf - Default unmap_dma_buf() for exported buffers
+ * @attach: attachment
+ * @sgt: SG table to unmap
+ * @dir: type of access done by this attachment
+ *
+ * Default implementation for dma_buf_ops::unmap_dma_buf(). This is just a wrapper
+ * around drm_gem_unmap_dma_buf() that lets us reset the dma_buf_attachment::priv
+ * field so that drm_gem_shmem_prime_{begin,end}_cpu_access() don't consider it
+ * as a mapped attachment to sync against.
+ */
+void drm_gem_shmem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
+				       struct sg_table *sgt,
+				       enum dma_data_direction dir)
+{
+	attach->priv = NULL;
+	drm_gem_unmap_dma_buf(attach, sgt, dir);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_unmap_dma_buf);
+
+/**
+ * drm_gem_shmem_prime_begin_cpu_access - Default begin_cpu_access() for exported buffers
+ * @dma_buf: The exported DMA buffer this acts on
+ * @dir: direction of the access
+ *
+ * Default implementation for dma_buf_ops::begin_cpu_access(). This only takes care of
+ * cache maintenance.
+ */
+int drm_gem_shmem_prime_begin_cpu_access(struct dma_buf *dma_buf,
+					 enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct dma_buf_attachment *attach;
+
+	dma_resv_lock(obj->resv, NULL);
+	if (shmem->sgt)
+		dma_sync_sgtable_for_cpu(dev->dev, shmem->sgt, dir);
+
+	if (shmem->vaddr)
+		invalidate_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+
+	list_for_each_entry(attach, &dma_buf->attachments, node) {
+		struct sg_table *sgt = attach->priv;
+
+		if (sgt)
+			dma_sync_sgtable_for_cpu(attach->dev, sgt, dir);
+	}
+	dma_resv_unlock(obj->resv);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_begin_cpu_access);
+
+/**
+ * drm_gem_shmem_prime_end_cpu_access - Default end_cpu_access() for exported buffers
+ * @dma_buf: The exported DMA buffer this acts on
+ * @dir: direction of the access
+ *
+ * Default implementation for dma_buf_ops::end_cpu_access(). This only takes care of
+ * cache maintenance.
+ */
+int drm_gem_shmem_prime_end_cpu_access(struct dma_buf *dma_buf,
+				       enum dma_data_direction dir)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct drm_device *dev = obj->dev;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct dma_buf_attachment *attach;
+
+	dma_resv_lock(obj->resv, NULL);
+	list_for_each_entry(attach, &dma_buf->attachments, node) {
+		struct sg_table *sgt = attach->priv;
+
+		if (sgt)
+			dma_sync_sgtable_for_device(attach->dev, sgt, dir);
+	}
+
+	if (shmem->vaddr)
+		flush_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+
+	if (shmem->sgt)
+		dma_sync_sgtable_for_device(dev->dev, shmem->sgt, dir);
+
+	dma_resv_unlock(obj->resv);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_end_cpu_access);
+
 /*
  * Kunit helpers
  */

index bcbb92f3fe9b..358cd247f803 100644
@@ -293,6 +293,16 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev,
 			      struct drm_mode_create_dumb *args);
 struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev,
 							 struct dma_buf *buf);
+struct sg_table *
+drm_gem_shmem_prime_map_dma_buf(struct dma_buf_attachment *attach,
+				enum dma_data_direction dir);
+void drm_gem_shmem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
+				       struct sg_table *sgt,
+				       enum dma_data_direction dir);
+int drm_gem_shmem_prime_begin_cpu_access(struct dma_buf *dma_buf,
+					 enum dma_data_direction dir);
+int drm_gem_shmem_prime_end_cpu_access(struct dma_buf *dma_buf,
+				       enum dma_data_direction dir);
 
 /**
  * DRM_GEM_SHMEM_DRIVER_OPS - Default shmem GEM operations

index 13c4a68fdde0..9bf727fc1c19 100644
@@ -790,6 +790,99 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct
 }
 EXPORT_SYMBOL_GPL(drm_gem_shmem_mmap);
 
+/**
+ * drm_gem_shmem_sync - Sync CPU-mapped data
+ * @shmem: shmem GEM object
+ * @offset: Offset into the GEM object
+ * @size: Size of the area to sync
+ * @type: Type of synchronization
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_shmem_sync(struct drm_gem_shmem_object *shmem, size_t offset,
+		       size_t size, enum drm_gem_shmem_sync_type type)
+{
+	const struct drm_device *dev = shmem->base.dev;
+	struct sg_table *sgt;
+	struct scatterlist *sgl;
+	unsigned int count;
+
+	/* Make sure the range is in bounds. */
+	if (offset + size < offset || offset + size > shmem->base.size)
+		return -EINVAL;
+
+	/* Disallow CPU-cache maintenance on imported buffers. */
+	if (drm_gem_is_imported(&shmem->base))
+		return -EINVAL;
+
+	switch (type) {
+	case DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH:
+	case DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE:
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	/* Don't bother if it's WC-mapped */
+	if (shmem->map_wc)
+		return 0;
+
+	/* Nothing to do if the size is zero. */
+	if (size == 0)
+		return 0;
+
+	sgt = drm_gem_shmem_get_pages_sgt(shmem);
+	if (IS_ERR(sgt))
+		return PTR_ERR(sgt);
+
+	for_each_sgtable_dma_sg(sgt, sgl, count) {
+		if (size == 0)
+			break;
+
+		dma_addr_t paddr = sg_dma_address(sgl);
+		size_t len = sg_dma_len(sgl);
+
+		if (len <= offset) {
+			offset -= len;
+			continue;
+		}
+
+		paddr += offset;
+		len -= offset;
+		len = min_t(size_t, len, size);
+		size -= len;
+		offset = 0;
+
+		/* It's unclear whether dma_sync_xxx() is the right API to do CPU
+		 * cache maintenance given an IOMMU can register their own
+		 * implementation doing more than just CPU cache flushes/invalidation,
+		 * and what we really care about here is CPU caches only, but that's
+		 * the best we have that is both arch-agnostic and does at least the
+		 * CPU cache maintenance on a <page,offset,size> tuple.
+		 *
+		 * Also, I wish we could do a single
+		 *
+		 *	dma_sync_single_for_device(BIDIR)
+		 *
+		 * and get a flush+invalidate, but that's not how it's implemented
+		 * in practice (at least on arm64), so we have to make it
+		 *
+		 *	dma_sync_single_for_device(TO_DEVICE)
+		 *	dma_sync_single_for_cpu(FROM_DEVICE)
+		 *
+		 * for the flush+invalidate case.
+		 */
+		dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
+		if (type == DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
+			dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_sync);
+
 /**
  * drm_gem_shmem_print_info() - Print &drm_gem_shmem_object info for debugfs
  * @shmem: shmem GEM object

index 358cd247f803..3c6ed3a5f0ac 100644
@@ -122,6 +122,23 @@ void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem,
 				 struct iosys_map *map);
 int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct *vma);
 
+/**
+ * enum drm_gem_shmem_sync_type - Type of synchronization
+ */
+enum drm_gem_shmem_sync_type {
+	/** @DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH: Flush CPU caches */
+	DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH = 0,
+
+	/**
+	 * @DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE: Flush and
+	 * invalidate CPU caches
+	 */
+	DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE,
+};
+
+int drm_gem_shmem_sync(struct drm_gem_shmem_object *shmem, size_t offset,
+		       size_t size, enum drm_gem_shmem_sync_type type);
+
 int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem);
 void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem);
 

index 7c348a0bf1ee..d26f8ca52906 100644
@@ -824,7 +824,7 @@ static const struct drm_driver panfrost_drm_driver = {
 	.minor			= 7,
 
 	.gem_create_object	= panfrost_gem_create_object,
-	.gem_prime_import	= panfrost_gem_prime_import,
+	.gem_prime_is_exporter	= panfrost_gem_prime_is_exporter,
 	.gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = panfrost_device_debugfs_init,

index 518f24549b9d..207110f33700 100644
@@ -380,35 +380,15 @@ static const struct dma_buf_ops panfrost_dma_buf_ops = {
 static struct dma_buf *
 panfrost_gem_prime_export(struct drm_gem_object *obj, int flags)
 {
-	struct drm_device *dev = obj->dev;
-	struct dma_buf_export_info exp_info = {
-		.exp_name = KBUILD_MODNAME,
-		.owner = THIS_MODULE,
-		.ops = &panfrost_dma_buf_ops,
-		.size = obj->size,
-		.flags = flags,
-		.priv = obj,
-		.resv = obj->resv,
-	};
-
-	return drm_gem_dmabuf_export(dev, &exp_info);
+	return drm_gem_prime_export_with_ops(obj, &panfrost_dma_buf_ops, flags);
 }
 
-struct drm_gem_object *
-panfrost_gem_prime_import(struct drm_device *dev,
-			  struct dma_buf *dma_buf)
+bool panfrost_gem_prime_is_exporter(struct drm_device *dev,
+				    struct dma_buf *dma_buf)
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 
-	if (dma_buf->ops == &panfrost_dma_buf_ops && obj->dev == dev) {
-		/* Importing dmabuf exported from our own gem increases
-		 * refcount on gem itself instead of f_count of dmabuf.
-		 */
-		drm_gem_object_get(obj);
-		return obj;
-	}
-
-	return drm_gem_prime_import(dev, dma_buf);
+	return dma_buf->ops == &panfrost_dma_buf_ops && obj->dev == dev;
 }
 
 static const struct drm_gem_object_funcs panfrost_gem_funcs = {

index 3008afcf4bff..9d30c2b768d7 100644
@@ -138,9 +138,8 @@ struct drm_gem_object *
 panfrost_gem_prime_import_sg_table(struct drm_device *dev,
 				   struct dma_buf_attachment *attach,
 				   struct sg_table *sgt);
-struct drm_gem_object *
-panfrost_gem_prime_import(struct drm_device *dev,
-			  struct dma_buf *dma_buf);
+bool panfrost_gem_prime_is_exporter(struct drm_device *dev,
+				    struct dma_buf *dma_buf);
 
 struct panfrost_gem_object *
 panfrost_gem_create(struct drm_device *dev, size_t size, u32 flags);

index 207110f33700..5f099c478edd 100644
@@ -289,92 +289,17 @@ static size_t panfrost_gem_rss(struct drm_gem_object *obj)
 	return 0;
 }
 
-static struct sg_table *
-panfrost_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
-			       enum dma_data_direction dir)
-{
-	struct sg_table *sgt = drm_gem_map_dma_buf(attach, dir);
-
-	if (!IS_ERR(sgt))
-		attach->priv = sgt;
-
-	return sgt;
-}
-
-static void
-panfrost_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
-				 struct sg_table *sgt,
-				 enum dma_data_direction dir)
-{
-	attach->priv = NULL;
-	drm_gem_unmap_dma_buf(attach, sgt, dir);
-}
-
-static int
-panfrost_gem_prime_begin_cpu_access(struct dma_buf *dma_buf,
-				    enum dma_data_direction dir)
-{
-	struct drm_gem_object *obj = dma_buf->priv;
-	struct drm_device *dev = obj->dev;
-	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
-	struct dma_buf_attachment *attach;
-
-	dma_resv_lock(obj->resv, NULL);
-	if (shmem->sgt)
-		dma_sync_sgtable_for_cpu(dev->dev, shmem->sgt, dir);
-
-	if (shmem->vaddr)
-		invalidate_kernel_vmap_range(shmem->vaddr, shmem->base.size);
-
-	list_for_each_entry(attach, &dma_buf->attachments, node) {
-		struct sg_table *sgt = attach->priv;
-
-		if (sgt)
-			dma_sync_sgtable_for_cpu(attach->dev, sgt, dir);
-	}
-	dma_resv_unlock(obj->resv);
-
-	return 0;
-}
-
-static int
-panfrost_gem_prime_end_cpu_access(struct dma_buf *dma_buf,
-				  enum dma_data_direction dir)
-{
-	struct drm_gem_object *obj = dma_buf->priv;
-	struct drm_device *dev = obj->dev;
-	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
-	struct dma_buf_attachment *attach;
-
-	dma_resv_lock(obj->resv, NULL);
-	list_for_each_entry(attach, &dma_buf->attachments, node) {
-		struct sg_table *sgt = attach->priv;
-
-		if (sgt)
-			dma_sync_sgtable_for_device(attach->dev, sgt, dir);
-	}
-
-	if (shmem->vaddr)
-		flush_kernel_vmap_range(shmem->vaddr, shmem->base.size);
-
-	if (shmem->sgt)
-		dma_sync_sgtable_for_device(dev->dev, shmem->sgt, dir);
-
-	dma_resv_unlock(obj->resv);
-	return 0;
-}
-
 static const struct dma_buf_ops panfrost_dma_buf_ops = {
 	.attach = drm_gem_map_attach,
 	.detach = drm_gem_map_detach,
-	.map_dma_buf = panfrost_gem_prime_map_dma_buf,
-	.unmap_dma_buf = panfrost_gem_prime_unmap_dma_buf,
+	.map_dma_buf = drm_gem_shmem_prime_map_dma_buf,
+	.unmap_dma_buf = drm_gem_shmem_prime_unmap_dma_buf,
 	.release = drm_gem_dmabuf_release,
 	.mmap = drm_gem_dmabuf_mmap,
 	.vmap = drm_gem_dmabuf_vmap,
 	.vunmap = drm_gem_dmabuf_vunmap,
-	.begin_cpu_access = panfrost_gem_prime_begin_cpu_access,
-	.end_cpu_access = panfrost_gem_prime_end_cpu_access,
+	.begin_cpu_access = drm_gem_shmem_prime_begin_cpu_access,
+	.end_cpu_access = drm_gem_shmem_prime_end_cpu_access,
 };
 
 static struct dma_buf *

index 5f099c478edd..80a4a1771ce7 100644
@@ -460,84 +460,20 @@ int
 panfrost_gem_sync(struct drm_gem_object *obj, u32 type, u32 offset, u32 size)
 {
 	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
-	struct drm_gem_shmem_object *shmem = &bo->base;
-	const struct drm_device *dev = shmem->base.dev;
-	struct sg_table *sgt;
-	struct scatterlist *sgl;
-	unsigned int count;
-
-	/* Make sure the range is in bounds. */
-	if (offset + size < offset || offset + size > shmem->base.size)
-		return -EINVAL;
-
-	/* Disallow CPU-cache maintenance on imported buffers. */
-	if (drm_gem_is_imported(&shmem->base))
-		return -EINVAL;
+	enum drm_gem_shmem_sync_type shmem_type;
 
 	switch (type) {
 	case PANFROST_BO_SYNC_CPU_CACHE_FLUSH:
+		shmem_type = DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH;
+		break;
 	case PANFROST_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE:
+		shmem_type = DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE;
 		break;
-
 	default:
 		return -EINVAL;
 	}
 
-	/* Don't bother if it's WC-mapped */
-	if (shmem->map_wc)
-		return 0;
-
-	/* Nothing to do if the size is zero. */
-	if (size == 0)
-		return 0;
-
-	sgt = drm_gem_shmem_get_pages_sgt(shmem);
-	if (IS_ERR(sgt))
-		return PTR_ERR(sgt);
-
-	for_each_sgtable_dma_sg(sgt, sgl, count) {
-		if (size == 0)
-			break;
-
-		dma_addr_t paddr = sg_dma_address(sgl);
-		size_t len = sg_dma_len(sgl);
-
-		if (len <= offset) {
-			offset -= len;
-			continue;
-		}
-
-		paddr += offset;
-		len -= offset;
-		len = min_t(size_t, len, size);
-		size -= len;
-		offset = 0;
-
-		/* It's unclear whether dma_sync_xxx() is the right API to do CPU
-		 * cache maintenance given an IOMMU can register their own
-		 * implementation doing more than just CPU cache flushes/invalidation,
-		 * and what we really care about here is CPU caches only, but that's
-		 * the best we have that is both arch-agnostic and does at least the
-		 * CPU cache maintenance on a <page,offset,size> tuple.
-		 *
-		 * Also, I wish we could do a single
-		 *
-		 *      dma_sync_single_for_device(BIDIR)
-		 *
-		 * and get a flush+invalidate, but that's not how it's implemented
-		 * in practice (at least on arm64), so we have to make it
-		 *
-		 *      dma_sync_single_for_device(TO_DEVICE)
-		 *      dma_sync_single_for_cpu(FROM_DEVICE)
-		 *
-		 * for the flush+invalidate case.
-		 */
-		dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
-		if (type == PANFROST_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
-			dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
-	}
-
-	return 0;
+	return drm_gem_shmem_sync(&bo->base, offset, size, shmem_type);
 }
 
 void

index dc8d0191aef4..37048030afcb 100644
@@ -1801,7 +1801,7 @@ static const struct drm_driver panthor_drm_driver = {
 	.minor = 9,
 
 	.gem_prime_import_sg_table = panthor_gem_prime_import_sg_table,
-	.gem_prime_import = panthor_gem_prime_import,
+	.gem_prime_is_exporter = panthor_gem_prime_is_exporter,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = panthor_debugfs_init,
 #endif

index 22b57ab6f929..2b7e4d5f5f5c 100644
@@ -562,39 +562,19 @@ static const struct dma_buf_ops panthor_dma_buf_ops = {
 static struct dma_buf *
 panthor_gem_prime_export(struct drm_gem_object *obj, int flags)
 {
-	struct drm_device *dev = obj->dev;
-	struct dma_buf_export_info exp_info = {
-		.exp_name = KBUILD_MODNAME,
-		.owner = THIS_MODULE,
-		.ops = &panthor_dma_buf_ops,
-		.size = obj->size,
-		.flags = flags,
-		.priv = obj,
-		.resv = obj->resv,
-	};
-
 	/* We can't export GEMs that have an exclusive VM. */
 	if (to_panthor_bo(obj)->exclusive_vm_root_gem)
 		return ERR_PTR(-EINVAL);
 
-	return drm_gem_dmabuf_export(dev, &exp_info);
+	return drm_gem_prime_export_with_ops(obj, &panthor_dma_buf_ops, flags);
 }
 
-struct drm_gem_object *
-panthor_gem_prime_import(struct drm_device *dev,
-			 struct dma_buf *dma_buf)
+bool panthor_gem_prime_is_exporter(struct drm_device *dev,
+				   struct dma_buf *dma_buf)
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 
-	if (dma_buf->ops == &panthor_dma_buf_ops && obj->dev == dev) {
-		/* Importing dmabuf exported from our own gem increases
-		 * refcount on gem itself instead of f_count of dmabuf.
-		 */
-		drm_gem_object_get(obj);
-		return obj;
-	}
-
-	return drm_gem_prime_import(dev, dma_buf);
+	return dma_buf->ops == &panthor_dma_buf_ops && obj->dev == dev;
 }
 
 static void panthor_gem_print_info(struct drm_printer *p, unsigned int indent,

index 5ae37d0d3646..a68cd3c280ed 100644
@@ -274,9 +274,8 @@ void panthor_gem_kernel_bo_set_label(struct panthor_kernel_bo *bo, const char *l
 int panthor_gem_sync(struct drm_gem_object *obj,
 		     u32 type, u64 offset, u64 size);
 
-struct drm_gem_object *
-panthor_gem_prime_import(struct drm_device *dev,
-			 struct dma_buf *dma_buf);
+bool panthor_gem_prime_is_exporter(struct drm_device *dev,
+				   struct dma_buf *dma_buf);
 
 static inline u64
 panthor_kernel_bo_gpuva(struct panthor_kernel_bo *bo)

index 4031ebe075d8..2e4a1d1b6322 100644
@@ -407,7 +407,7 @@ const struct dma_buf_ops amdgpu_dmabuf_ops = {
  * @gobj: GEM BO
  * @flags: Flags such as DRM_CLOEXEC and DRM_RDWR.
  *
- * The main work is done by the &drm_gem_prime_export helper.
+ * The main work is done by drm_gem_prime_export_with_ops().
  *
  * Returns:
  * Shared DMA buffer representing the GEM BO from the given device.

@@ -416,7 +416,6 @@ struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj,
 					int flags)
 {
 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
-	struct dma_buf *buf;
 	struct ttm_operation_ctx ctx = {
 		.interruptible = true,
 		.no_wait_gpu = true,

@@ -434,11 +433,7 @@ struct dma_buf *amdgpu_gem_prime_export(struct drm_gem_object *gobj,
 	if (ret)
 		return ERR_PTR(ret);
 
-	buf = drm_gem_prime_export(gobj, flags);
-	if (!IS_ERR(buf))
-		buf->ops = &amdgpu_dmabuf_ops;
-
-	return buf;
+	return drm_gem_prime_export_with_ops(gobj, &amdgpu_dmabuf_ops, flags);
 }
 
 /**

index db95a2fac5d0..803f3bbb7144 100644
@@ -63,12 +63,38 @@ void v3d_free_object(struct drm_gem_object *obj)
 	drm_gem_shmem_free(&bo->base);
 }
 
+static const struct dma_buf_ops v3d_dma_buf_ops = {
+	.attach = drm_gem_map_attach,
+	.detach = drm_gem_map_detach,
+	.map_dma_buf = drm_gem_shmem_prime_map_dma_buf,
+	.unmap_dma_buf = drm_gem_shmem_prime_unmap_dma_buf,
+	.release = drm_gem_dmabuf_release,
+	.mmap = drm_gem_dmabuf_mmap,
+	.vmap = drm_gem_dmabuf_vmap,
+	.vunmap = drm_gem_dmabuf_vunmap,
+	.begin_cpu_access = drm_gem_shmem_prime_begin_cpu_access,
+	.end_cpu_access = drm_gem_shmem_prime_end_cpu_access,
+};
+
+static struct dma_buf *v3d_prime_export(struct drm_gem_object *obj, int flags)
+{
+	return drm_gem_prime_export_with_ops(obj, &v3d_dma_buf_ops, flags);
+}
+
+bool v3d_prime_is_exporter(struct drm_device *dev, struct dma_buf *dma_buf)
+{
+	struct drm_gem_object *obj = dma_buf->priv;
+
+	return dma_buf->ops == &v3d_dma_buf_ops && obj->dev == dev;
+}
+
 static const struct drm_gem_object_funcs v3d_gem_funcs = {
 	.free = v3d_free_object,
 	.print_info = drm_gem_shmem_object_print_info,
 	.pin = drm_gem_shmem_object_pin,
 	.unpin = drm_gem_shmem_object_unpin,
 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
+	.export = v3d_prime_export,
 	.vmap = drm_gem_shmem_object_vmap,
 	.vunmap = drm_gem_shmem_object_vunmap,
 	.mmap = drm_gem_shmem_object_mmap,

index b790be8a7fac..7d91c528c1e9 100644
@@ -283,6 +283,7 @@ static const struct drm_driver v3d_drm_driver = {
 
 	.gem_create_object = v3d_create_object,
 	.gem_prime_import_sg_table = v3d_prime_import_sg_table,
+	.gem_prime_is_exporter = v3d_prime_is_exporter,
 
 	.ioctls = v3d_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(v3d_drm_ioctls),

index 7c7bdf6e51a5..e0eed4cfef61 100644
@@ -606,6 +606,7 @@ int v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
 						 struct dma_buf_attachment *attach,
 						 struct sg_table *sgt);
+bool v3d_prime_is_exporter(struct drm_device *dev, struct dma_buf *dma_buf);
 
 /* v3d_debugfs.c */
 void v3d_debugfs_init(struct drm_minor *minor);

index 803f3bbb7144..0a5a056d4b26 100644
@@ -173,18 +173,19 @@ v3d_bo_create_finish(struct drm_gem_object *obj)
 	return 0;
 }
 
-struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
-			     size_t unaligned_size)
+struct v3d_bo *v3d_bo_create(struct drm_device *dev, size_t size, u32 flags)
 {
 	struct drm_gem_shmem_object *shmem_obj;
 	struct v3d_bo *bo;
 	int ret;
 
-	shmem_obj = drm_gem_shmem_create(dev, unaligned_size);
+	shmem_obj = drm_gem_shmem_create(dev, size);
 	if (IS_ERR(shmem_obj))
 		return ERR_CAST(shmem_obj);
+
 	bo = to_v3d_bo(&shmem_obj->base);
 	bo->vaddr = NULL;
+	bo->base.map_wc = !(flags & DRM_V3D_BO_WB_MMAP);
 
 	ret = v3d_bo_create_finish(&shmem_obj->base);
 	if (ret)

@@ -239,12 +240,12 @@ int v3d_create_bo_ioctl(struct drm_device *dev, void *data,
 	struct v3d_bo *bo = NULL;
 	int ret;
 
-	if (args->flags != 0) {
+	if (args->flags & ~DRM_V3D_BO_WB_MMAP) {
 		drm_dbg(dev, "unknown create_bo flags: %d\n", args->flags);
 		return -EINVAL;
 	}
 
-	bo = v3d_bo_create(dev, file_priv, PAGE_ALIGN(args->size));
+	bo = v3d_bo_create(dev, PAGE_ALIGN(args->size), args->flags);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 

index e0eed4cfef61..4f32fc3ab1de 100644
@@ -591,8 +591,7 @@ struct v3d_submit_ext {
 /* v3d_bo.c */
 struct drm_gem_object *v3d_create_object(struct drm_device *dev, size_t size);
 void v3d_free_object(struct drm_gem_object *gem_obj);
-struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
-			     size_t size);
+struct v3d_bo *v3d_bo_create(struct drm_device *dev, size_t size, u32 flags);
 void v3d_get_bo_vaddr(struct v3d_bo *bo);
 void v3d_put_bo_vaddr(struct v3d_bo *bo);
 int v3d_create_bo_ioctl(struct drm_device *dev, void *data,

index 963d711dc16b..19315acf567a 100644
@@ -43,7 +43,7 @@ v3d_overflow_mem_work(struct work_struct *work)
 	struct v3d_dev *v3d =
 		container_of(work, struct v3d_dev, overflow_mem_work);
 	struct drm_device *dev = &v3d->drm;
-	struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
+	struct v3d_bo *bo = v3d_bo_create(dev, SZ_256K, 0);
 	struct v3d_queue_state *queue = &v3d->queue[V3D_BIN];
 	struct v3d_bin_job *bin_job;
 	struct drm_gem_object *obj;

index 2750a6fbb6b8..4221c7544d08 100644
@@ -237,11 +237,16 @@ struct drm_v3d_wait_bo {
 	__u64 timeout_ns;
 };
 
+/*
+ * Map the BO CPU-cached instead of write-combined. Userspace is responsible
+ * for CPU cache maintenance.
+ */
+#define DRM_V3D_BO_WB_MMAP			  0x01
+
 /**
  * struct drm_v3d_create_bo - ioctl argument for creating V3D BOs.
  *
- * There are currently no values for the flags argument, but it may be
- * used in a future extension.
+ * @flags is a mask of DRM_V3D_BO_* flags.
  */
 struct drm_v3d_create_bo {
 	__u32 size;

index 0a5a056d4b26..78eb3a1cc794 100644
@@ -280,6 +280,70 @@ int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
+static int v3d_bo_sync(struct drm_gem_object *obj,
+		       const struct drm_v3d_bo_sync_op *op)
+{
+	struct v3d_bo *bo = to_v3d_bo(obj);
+	enum drm_gem_shmem_sync_type type;
+
+	switch (op->type) {
+	case DRM_V3D_BO_SYNC_CPU_CACHE_FLUSH:
+		type = DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH;
+		break;
+	case DRM_V3D_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE:
+		type = DRM_GEM_SHMEM_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return drm_gem_shmem_sync(&bo->base, op->offset, op->size, type);
+}
+
+int v3d_sync_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_v3d_sync_bo *args = data;
+	struct drm_v3d_bo_sync_op *ops;
+	struct drm_gem_object *obj;
+	int ret = 0;
+
+	if (args->pad)
+		return -EINVAL;
+
+	if (!args->op_count)
+		return 0;
+
+	/* Limits the number of sync operations requested by userspace. */
+	if (args->op_count > SZ_64K) {
+		drm_dbg(dev, "Too many sync ops: %u\n", args->op_count);
+		return -EINVAL;
+	}
+
+	ops = vmemdup_array_user(u64_to_user_ptr(args->ops), args->op_count,
+				 sizeof(*ops));
+	if (IS_ERR(ops))
+		return PTR_ERR(ops);
+
+	for (u32 i = 0; i < args->op_count; i++) {
+		obj = drm_gem_object_lookup(file_priv, ops[i].handle);
+		if (!obj) {
+			drm_dbg(dev, "Failed to look up GEM BO %d\n",
+				ops[i].handle);
+			ret = -ENOENT;
+			break;
+		}
+
+		ret = v3d_bo_sync(obj, &ops[i]);
+		drm_gem_object_put(obj);
+		if (ret)
+			break;
+	}
+
+	kvfree(ops);
+	return ret;
+}
+
 int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv)
 {

index 7d91c528c1e9..9e6e4f832eee 100644
@@ -267,6 +267,7 @@ static const struct drm_ioctl_desc v3d_drm_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(V3D_SUBMIT_CPU, v3d_submit_cpu_ioctl, DRM_RENDER_ALLOW | DRM_AUTH),
 	DRM_IOCTL_DEF_DRV(V3D_PERFMON_GET_COUNTER, v3d_perfmon_get_counter_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(V3D_PERFMON_SET_GLOBAL, v3d_perfmon_set_global_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(V3D_SYNC_BO, v3d_sync_bo_ioctl, DRM_RENDER_ALLOW),
 };
 
 static const struct drm_driver v3d_drm_driver = {

index 4f32fc3ab1de..e5fb3a7b8d96 100644
@@ -602,6 +602,8 @@ int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
 int v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 		      struct drm_file *file_priv);
+int v3d_sync_bo_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv);
 struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
 						 struct dma_buf_attachment *attach,
 						 struct sg_table *sgt);

66 lines not on this page · raw message

index 9e6e4f832eee..e613af3ac81f 100644
@@ -132,6 +132,9 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
 	case DRM_V3D_PARAM_SUPPORTS_TFU_CCOL:
 		args->value = (v3d->ver >= V3D_GEN_71);
 		return 0;
+	case DRM_V3D_PARAM_SUPPORTS_WB_MMAP:
+		args->value = 1;
+		return 0;
 	default:
 		drm_dbg(dev, "Unknown parameter %d\n", args->param);
 		return -EINVAL;

index c2f7f1409ae4..7035319766e8 100644
@@ -304,6 +304,7 @@ enum drm_v3d_param {
 	DRM_V3D_PARAM_GLOBAL_RESET_COUNTER,
 	DRM_V3D_PARAM_CONTEXT_RESET_COUNTER,
 	DRM_V3D_PARAM_SUPPORTS_TFU_CCOL,
+	DRM_V3D_PARAM_SUPPORTS_WB_MMAP,
 };
 
 struct drm_v3d_get_param {