Skip to contentSkip to conversation

Refactor VM bind code

PATCH Superseded Matthew Brost 3 participants 28 messages Feb 6, 2024 – Mar 28, 2024

Subsystems (2)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Matthew Brost

author maintainer
#0 lore raw

v3 cover Refactor VM bind code

Implement proper error handling for VM bind IOCTL by allowing failures of GPU memory allocation (either system or VRAM) to be propagated to user without corrupting VM state. Mainly implemented by converting VM bind IOCTL to 1 job per IOCTL rather than potential many jobs.

Broken into roughly 4 parts:
Part 1: Prep patches, patches 1-11
Part 2: 1 jobs per VM bind IOCTL and error handling, patches 12-17 Part 3: CPU binds, patches 18-21
Part 4: Error injection for testing, patch 22

For reviewing, let's focus on part 1 for now and see if patches from that part can start to get merged.

Tested with [1] and new error handling appears to be working. Also tested with existing tests at every patch in the series and should be working at every patch in the series.

Matt

[1] https://patchwork.freedesktop.org/series/129606/

Matthew Brost (22):
drm/xe: Lock all gpuva ops during VM bind IOCTL
drm/xe: Add ops_execute function which returns a fence drm/xe: Move migrate to prefetch to op_lock funtion drm/xe: Add struct xe_vma_ops abstraction
drm/xe: Update xe_vm_rebind to use dummy VMA operations drm/xe: Simplify VM bind IOCTL error handling and cleanup drm/xe: Update pagefaults to use dummy VMA operations drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue drm/xe: Add vm_bind_ioctl_ops_install_fences helper drm/xe: Move setting last fence to vm_bind_ioctl_ops_install_fences drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use
this
drm/xe: Add some members to xe_vma_ops
drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
drm/xe: Convert multiple bind ops into single job
drm/xe: Remove old functions defs in xe_pt.h
drm/xe: Update PT layer with better error handling drm/xe: Update VM trace events
drm/xe: Update clear / populate arguments
drm/xe: Add __xe_migrate_update_pgtables_cpu helper drm/xe: CPU binds for jobs
drm/xe: Don't use migrate exec queue for page fault binds drm/xe: Add VM bind IOCTL error injection

 drivers/gpu/drm/xe/xe_bo.c                  |    7 +-
 drivers/gpu/drm/xe/xe_bo.h                  |    4 +-
 drivers/gpu/drm/xe/xe_device.c              |   35 +
 drivers/gpu/drm/xe/xe_device.h              |    2 +
 drivers/gpu/drm/xe/xe_device_types.h        |   16 +
 drivers/gpu/drm/xe/xe_exec.c                |   25 +-
 drivers/gpu/drm/xe/xe_gt_pagefault.c        |   10 +-
 drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c |   60 +-
 drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h |    3 +
 drivers/gpu/drm/xe/xe_guc_submit.c          |   47 +-
 drivers/gpu/drm/xe/xe_migrate.c             |  387 ++----
 drivers/gpu/drm/xe/xe_migrate.h             |   46 +-
 drivers/gpu/drm/xe/xe_pt.c                  | 1223 ++++++++++++-------
 drivers/gpu/drm/xe/xe_pt.h                  |   15 +-
 drivers/gpu/drm/xe/xe_pt_types.h            |   53 +
 drivers/gpu/drm/xe/xe_sched_job.c           |   24 +-
 drivers/gpu/drm/xe/xe_sched_job_types.h     |   31 +-
 drivers/gpu/drm/xe/xe_trace.h               |   10 +-
 drivers/gpu/drm/xe/xe_vm.c                  |  980 +++++++--------
 drivers/gpu/drm/xe/xe_vm.h                  |    7 +
 drivers/gpu/drm/xe/xe_vm_types.h            |  198 +--
 21 files changed, 1773 insertions(+), 1410 deletions(-)
Signature · 1 line
2.34.1
author maintainer
#1 lore raw

PATCH v3 1/22 drm/xe: Lock all gpuva ops during VM bind IOCTL

Lock all gpuva ops and validate all BOs in a single step durin the VM bind IOCTL. This help with the transition to making all gpuva ops in a VM bind IOCTL a single atomic job.

---
 drivers/gpu/drm/xe/xe_vm.c | 142 ++++++++++++++++++++++++++-----------
 1 file changed, 101 insertions(+), 41 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +101 -41
index 9d2e8088d07e..3a7b82ca4b35 100644
@@ -411,19 +411,23 @@ int __xe_vm_userptr_needs_repin(struct xe_vm *vm)
 
 #define XE_VM_REBIND_RETRY_TIMEOUT_MS 1000
 
-static void xe_vm_kill(struct xe_vm *vm)
+static void xe_vm_kill(struct xe_vm *vm, bool unlocked)
 {
 	struct xe_exec_queue *q;
 
 	lockdep_assert_held(&vm->lock);
 
-	xe_vm_lock(vm, false);
+	if (unlocked)
+		xe_vm_lock(vm, false);
+
 	vm->flags |= XE_VM_FLAG_BANNED;
 	trace_xe_vm_kill(vm);
 
 	list_for_each_entry(q, &vm->preempt.exec_queues, compute.link)
 		q->ops->kill(q);
-	xe_vm_unlock(vm);
+
+	if (unlocked)
+		xe_vm_unlock(vm);
 
 	/* TODO: Inform user the VM is banned */
 }
@@ -619,7 +623,7 @@ static void preempt_rebind_work_func(struct work_struct *w)
 
 	if (err) {
 		drm_warn(&vm->xe->drm, "VM worker error: %d\n", err);
-		xe_vm_kill(vm);
+		xe_vm_kill(vm, true);
 	}
 	up_write(&vm->lock);
 
@@ -1773,17 +1777,9 @@ static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue
 		      u32 num_syncs, bool immediate, bool first_op,
 		      bool last_op)
 {
-	int err;
-
 	xe_vm_assert_held(vm);
 	xe_bo_assert_held(bo);
 
-	if (bo && immediate) {
-		err = xe_bo_validate(bo, vm, true);
-		if (err)
-			return err;
-	}
-
 	return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op,
 			    last_op);
 }
@@ -2414,17 +2410,12 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 	return 0;
 }
 
-static int op_execute(struct drm_exec *exec, struct xe_vm *vm,
-		      struct xe_vma *vma, struct xe_vma_op *op)
+static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
+		      struct xe_vma_op *op)
 {
 	int err;
 
 	lockdep_assert_held_write(&vm->lock);
-
-	err = xe_vm_prepare_vma(exec, vma, 1);
-	if (err)
-		return err;
-
 	xe_vm_assert_held(vm);
 	xe_bo_assert_held(xe_vma_bo(vma));
 
@@ -2505,19 +2496,10 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm,
 static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
 			       struct xe_vma_op *op)
 {
-	struct drm_exec exec;
 	int err;
 
 retry_userptr:
-	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
-	drm_exec_until_all_locked(&exec) {
-		err = op_execute(&exec, vm, vma, op);
-		drm_exec_retry_on_contention(&exec);
-		if (err)
-			break;
-	}
-	drm_exec_fini(&exec);
-
+	err = op_execute(vm, vma, op);
 	if (err == -EAGAIN) {
 		lockdep_assert_held_write(&vm->lock);
 
@@ -2682,29 +2664,107 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
 	}
 }
 
+static int vma_lock(struct drm_exec *exec, struct xe_vma *vma, bool validate)
+{
+	struct xe_bo *bo = xe_vma_bo(vma);
+	int err = 0;
+
+	if (bo) {
+		if (!bo->vm)
+			err = drm_exec_prepare_obj(exec, &bo->ttm.base, 1);
+		if (!err && validate)
+			err = xe_bo_validate(bo, xe_vma_vm(vma), true);
+	}
+
+	return err;
+}
+
+static int op_lock(struct drm_exec *exec, struct xe_vm *vm,
+		   struct xe_vma_op *op)
+{
+	int err = 0;
+
+	switch (op->base.op) {
+	case DRM_GPUVA_OP_MAP:
+		err = vma_lock(exec, op->map.vma,
+			       op->map.immediate || !xe_vm_in_fault_mode(vm));
+		break;
+	case DRM_GPUVA_OP_REMAP:
+		err = vma_lock(exec, gpuva_to_vma(op->base.remap.unmap->va),
+			       false);
+		if (!err && op->remap.prev)
+			err = vma_lock(exec, op->remap.prev, true);
+		if (!err && op->remap.next)
+			err = vma_lock(exec, op->remap.next, true);
+		break;
+	case DRM_GPUVA_OP_UNMAP:
+		err = vma_lock(exec, gpuva_to_vma(op->base.unmap.va), false);
+		break;
+	case DRM_GPUVA_OP_PREFETCH:
+		err = vma_lock(exec, gpuva_to_vma(op->base.prefetch.va), true);
+		break;
+	default:
+		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
+	}
+
+	return err;
+}
+
+static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
+				  struct xe_vm *vm,
+				  struct list_head *ops_list)
+{
+	struct xe_vma_op *op;
+	int err;
+
+	err = drm_exec_prepare_obj(exec, xe_vm_obj(vm), 1);
+	if (err)
+		return err;
+
+	list_for_each_entry(op, ops_list, link) {
+		err = op_lock(exec, vm, op);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 				     struct list_head *ops_list)
 {
+	struct drm_exec exec;
 	struct xe_vma_op *op, *next;
 	int err;
 
 	lockdep_assert_held_write(&vm->lock);
 
-	list_for_each_entry_safe(op, next, ops_list, link) {
-		err = xe_vma_op_execute(vm, op);
-		if (err) {
-			drm_warn(&vm->xe->drm, "VM op(%d) failed with %d",
-				 op->base.op, err);
-			/*
-			 * FIXME: Killing VM rather than proper error handling
-			 */
-			xe_vm_kill(vm);
-			return -ENOSPC;
+	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
+	drm_exec_until_all_locked(&exec) {
+		err = vm_bind_ioctl_ops_lock(&exec, vm, ops_list);
+		drm_exec_retry_on_contention(&exec);
+		if (err)
+			goto unlock;
+
+		list_for_each_entry_safe(op, next, ops_list, link) {
+			err = xe_vma_op_execute(vm, op);
+			if (err) {
+				drm_warn(&vm->xe->drm, "VM op(%d) failed with %d",
+					 op->base.op, err);
+				/*
+				 * FIXME: Killing VM rather than proper error handling
+				 */
+				xe_vm_kill(vm, false);
+				err = -ENOSPC;
+				goto unlock;
+			}
+			xe_vma_op_cleanup(vm, op);
 		}
-		xe_vma_op_cleanup(vm, op);
 	}
 
-	return 0;
+unlock:
+	drm_exec_fini(&exec);
+	return err;
 }
 
 #define SUPPORTED_FLAGS	\
Signature · 1 line
2.34.1
author maintainer
#2 lore raw

PATCH v3 2/22 drm/xe: Add ops_execute function which returns a fence

In reply to Matthew Brost

Add ops_execute function which returns a fence. This will be helpful to initiate all binds (VM bind IOCTL, rebinds in exec IOCTL, rebinds in preempt rebind worker, and rebinds in pagefaults) via a gpuva ops list. Returning a fence is needed in various paths.

---
 drivers/gpu/drm/xe/xe_vm.c | 212 ++++++++++++++++++++-----------------
 1 file changed, 112 insertions(+), 100 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +112 -100
index 3a7b82ca4b35..a33c1486ab64 100644
@@ -1738,21 +1738,22 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
 	return ERR_PTR(err);
 }
 
-static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
-			struct xe_exec_queue *q, struct xe_sync_entry *syncs,
-			u32 num_syncs, bool immediate, bool first_op,
-			bool last_op)
+static struct dma_fence *
+xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
+	   struct xe_bo *bo, struct xe_sync_entry *syncs, u32 num_syncs,
+	   bool immediate, bool first_op, bool last_op)
 {
 	struct dma_fence *fence;
 	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
 
 	xe_vm_assert_held(vm);
+	xe_bo_assert_held(bo);
 
 	if (immediate) {
 		fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op,
 				       last_op);
 		if (IS_ERR(fence))
-			return PTR_ERR(fence);
+			return fence;
 	} else {
 		int i;
 
@@ -1767,26 +1768,14 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma,
 
 	if (last_op)
 		xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
-	dma_fence_put(fence);
-
-	return 0;
-}
-
-static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
-		      struct xe_bo *bo, struct xe_sync_entry *syncs,
-		      u32 num_syncs, bool immediate, bool first_op,
-		      bool last_op)
-{
-	xe_vm_assert_held(vm);
-	xe_bo_assert_held(bo);
 
-	return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op,
-			    last_op);
+	return fence;
 }
 
-static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
-			struct xe_exec_queue *q, struct xe_sync_entry *syncs,
-			u32 num_syncs, bool first_op, bool last_op)
+static struct dma_fence *
+xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
+	     struct xe_exec_queue *q, struct xe_sync_entry *syncs,
+	     u32 num_syncs, bool first_op, bool last_op)
 {
 	struct dma_fence *fence;
 	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
@@ -1796,14 +1785,13 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
 
 	fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op);
 	if (IS_ERR(fence))
-		return PTR_ERR(fence);
+		return fence;
 
 	xe_vma_destroy(vma, fence);
 	if (last_op)
 		xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
-	dma_fence_put(fence);
 
-	return 0;
+	return fence;
 }
 
 #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \
@@ -1946,10 +1934,11 @@ static const u32 region_to_mem_type[] = {
 	XE_PL_VRAM1,
 };
 
-static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
-			  struct xe_exec_queue *q, u32 region,
-			  struct xe_sync_entry *syncs, u32 num_syncs,
-			  bool first_op, bool last_op)
+static struct dma_fence *
+xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
+	       struct xe_exec_queue *q, u32 region,
+	       struct xe_sync_entry *syncs, u32 num_syncs,
+	       bool first_op, bool last_op)
 {
 	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
 	int err;
@@ -1959,27 +1948,24 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
 	if (!xe_vma_has_no_bo(vma)) {
 		err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
 		if (err)
-			return err;
+			return ERR_PTR(err);
 	}
 
 	if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
 		return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
 				  true, first_op, last_op);
 	} else {
+		struct dma_fence *fence =
+			xe_exec_queue_last_fence_get(wait_exec_queue, vm);
 		int i;
 
 		/* Nothing to do, signal fences now */
 		if (last_op) {
-			for (i = 0; i < num_syncs; i++) {
-				struct dma_fence *fence =
-					xe_exec_queue_last_fence_get(wait_exec_queue, vm);
-
+			for (i = 0; i < num_syncs; i++)
 				xe_sync_entry_signal(&syncs[i], NULL, fence);
-				dma_fence_put(fence);
-			}
 		}
 
-		return 0;
+		return fence;
 	}
 }
 
@@ -2410,10 +2396,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 	return 0;
 }
 
-static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
-		      struct xe_vma_op *op)
+static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
+				    struct xe_vma_op *op)
 {
-	int err;
+	struct dma_fence *fence = NULL;
 
 	lockdep_assert_held_write(&vm->lock);
 	xe_vm_assert_held(vm);
@@ -2421,11 +2407,12 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
-		err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
-				 op->syncs, op->num_syncs,
-				 op->map.immediate || !xe_vm_in_fault_mode(vm),
-				 op->flags & XE_VMA_OP_FIRST,
-				 op->flags & XE_VMA_OP_LAST);
+		fence = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma),
+				   op->syncs, op->num_syncs,
+				   op->map.immediate ||
+				   !xe_vm_in_fault_mode(vm),
+				   op->flags & XE_VMA_OP_FIRST,
+				   op->flags & XE_VMA_OP_LAST);
 		break;
 	case DRM_GPUVA_OP_REMAP:
 	{
@@ -2435,37 +2422,39 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
 		if (!op->remap.unmap_done) {
 			if (prev || next)
 				vma->gpuva.flags |= XE_VMA_FIRST_REBIND;
-			err = xe_vm_unbind(vm, vma, op->q, op->syncs,
-					   op->num_syncs,
-					   op->flags & XE_VMA_OP_FIRST,
-					   op->flags & XE_VMA_OP_LAST &&
-					   !prev && !next);
-			if (err)
+			fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
+					     op->num_syncs,
+					     op->flags & XE_VMA_OP_FIRST,
+					     op->flags & XE_VMA_OP_LAST &&
+					     !prev && !next);
+			if (IS_ERR(fence))
 				break;
 			op->remap.unmap_done = true;
 		}
 
 		if (prev) {
 			op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND;
-			err = xe_vm_bind(vm, op->remap.prev, op->q,
-					 xe_vma_bo(op->remap.prev), op->syncs,
-					 op->num_syncs, true, false,
-					 op->flags & XE_VMA_OP_LAST && !next);
+			dma_fence_put(fence);
+			fence = xe_vm_bind(vm, op->remap.prev, op->q,
+					   xe_vma_bo(op->remap.prev), op->syncs,
+					   op->num_syncs, true, false,
+					   op->flags & XE_VMA_OP_LAST && !next);
 			op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND;
-			if (err)
+			if (IS_ERR(fence))
 				break;
 			op->remap.prev = NULL;
 		}
 
 		if (next) {
 			op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND;
-			err = xe_vm_bind(vm, op->remap.next, op->q,
-					 xe_vma_bo(op->remap.next),
-					 op->syncs, op->num_syncs,
-					 true, false,
-					 op->flags & XE_VMA_OP_LAST);
+			dma_fence_put(fence);
+			fence = xe_vm_bind(vm, op->remap.next, op->q,
+					   xe_vma_bo(op->remap.next),
+					   op->syncs, op->num_syncs,
+					   true, false,
+					   op->flags & XE_VMA_OP_LAST);
 			op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND;
-			if (err)
+			if (IS_ERR(fence))
 				break;
 			op->remap.next = NULL;
 		}
@@ -2473,34 +2462,36 @@ static int op_execute(struct xe_vm *vm, struct xe_vma *vma,
 		break;
 	}
 	case DRM_GPUVA_OP_UNMAP:
-		err = xe_vm_unbind(vm, vma, op->q, op->syncs,
-				   op->num_syncs, op->flags & XE_VMA_OP_FIRST,
-				   op->flags & XE_VMA_OP_LAST);
+		fence = xe_vm_unbind(vm, vma, op->q, op->syncs,
+				     op->num_syncs, op->flags & XE_VMA_OP_FIRST,
+				     op->flags & XE_VMA_OP_LAST);
 		break;
 	case DRM_GPUVA_OP_PREFETCH:
-		err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
-				     op->syncs, op->num_syncs,
-				     op->flags & XE_VMA_OP_FIRST,
-				     op->flags & XE_VMA_OP_LAST);
+		fence = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
+				       op->syncs, op->num_syncs,
+				       op->flags & XE_VMA_OP_FIRST,
+				       op->flags & XE_VMA_OP_LAST);
 		break;
 	default:
 		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
 	}
 
-	if (err)
+	if (IS_ERR(fence))
 		trace_xe_vma_fail(vma);
 
-	return err;
+	return fence;
 }
 
-static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
-			       struct xe_vma_op *op)
+static struct dma_fence *
+__xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
+		    struct xe_vma_op *op)
 {
+	struct dma_fence *fence;
 	int err;
 
 retry_userptr:
-	err = op_execute(vm, vma, op);
-	if (err == -EAGAIN) {
+	fence = op_execute(vm, vma, op);
+	if (IS_ERR(fence) && PTR_ERR(fence) == -EAGAIN) {
 		lockdep_assert_held_write(&vm->lock);
 
 		if (op->base.op == DRM_GPUVA_OP_REMAP) {
@@ -2517,22 +2508,24 @@ static int __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
 			if (!err)
 				goto retry_userptr;
 
+			fence = ERR_PTR(err);
 			trace_xe_vma_fail(vma);
 		}
 	}
 
-	return err;
+	return fence;
 }
 
-static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
+static struct dma_fence *
+xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
 {
-	int ret = 0;
+	struct dma_fence *fence = ERR_PTR(-ENOMEM);
 
 	lockdep_assert_held_write(&vm->lock);
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
-		ret = __xe_vma_op_execute(vm, op->map.vma, op);
+		fence = __xe_vma_op_execute(vm, op->map.vma, op);
 		break;
 	case DRM_GPUVA_OP_REMAP:
 	{
@@ -2545,23 +2538,23 @@ static int xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
 		else
 			vma = op->remap.next;
 
-		ret = __xe_vma_op_execute(vm, vma, op);
+		fence = __xe_vma_op_execute(vm, vma, op);
 		break;
 	}
 	case DRM_GPUVA_OP_UNMAP:
-		ret = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
-					  op);
+		fence = __xe_vma_op_execute(vm, gpuva_to_vma(op->base.unmap.va),
+					    op);
 		break;
 	case DRM_GPUVA_OP_PREFETCH:
-		ret = __xe_vma_op_execute(vm,
-					  gpuva_to_vma(op->base.prefetch.va),
-					  op);
+		fence = __xe_vma_op_execute(vm,
+					    gpuva_to_vma(op->base.prefetch.va),
+					    op);
 		break;
 	default:
 		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
 	}
 
-	return ret;
+	return fence;
 }
 
 static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
@@ -2730,11 +2723,35 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
 	return 0;
 }
 
+static struct dma_fence *ops_execute(struct xe_vm *vm,
+				     struct list_head *ops_list,
+				     bool cleanup)
+{
+	struct xe_vma_op *op, *next;
+	struct dma_fence *fence = NULL;
+
+	list_for_each_entry_safe(op, next, ops_list, link) {
+		if (!IS_ERR(fence)) {
+			dma_fence_put(fence);
+			fence = xe_vma_op_execute(vm, op);
+		}
+		if (IS_ERR(fence)) {
+			drm_warn(&vm->xe->drm, "VM op(%d) failed with %ld",
+				 op->base.op, PTR_ERR(fence));
+			fence = ERR_PTR(-ENOSPC);
+		}
+		if (cleanup)
+			xe_vma_op_cleanup(vm, op);
+	}
+
+	return fence;
+}
+
 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 				     struct list_head *ops_list)
 {
 	struct drm_exec exec;
-	struct xe_vma_op *op, *next;
+	struct dma_fence *fence;
 	int err;
 
 	lockdep_assert_held_write(&vm->lock);
@@ -2746,19 +2763,14 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 		if (err)
 			goto unlock;
 
-		list_for_each_entry_safe(op, next, ops_list, link) {
-			err = xe_vma_op_execute(vm, op);
-			if (err) {
-				drm_warn(&vm->xe->drm, "VM op(%d) failed with %d",
-					 op->base.op, err);
-				/*
-				 * FIXME: Killing VM rather than proper error handling
-				 */
-				xe_vm_kill(vm, false);
-				err = -ENOSPC;
-				goto unlock;
-			}
-			xe_vma_op_cleanup(vm, op);
+		fence = ops_execute(vm, ops_list, true);
+		if (IS_ERR(fence)) {
+			err = PTR_ERR(fence);
+			/* FIXME: Killing VM rather than proper error handling */
+			xe_vm_kill(vm, false);
+			goto unlock;
+		} else {
+			dma_fence_put(fence);
 		}
 	}
 
Signature · 1 line
2.34.1
author maintainer
#3 lore raw

PATCH v3 3/22 drm/xe: Move migrate to prefetch to op_lock funtion

In reply to Matthew Brost

Migrates need to be done under drm exec to make lockdep happy, move the migrate done for prefetches under the op_lock function.

---
 drivers/gpu/drm/xe/xe_vm.c | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +13 -15
index a33c1486ab64..0f552fd8d7e0 100644
@@ -1936,20 +1936,10 @@ static const u32 region_to_mem_type[] = {
 
 static struct dma_fence *
 xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
-	       struct xe_exec_queue *q, u32 region,
-	       struct xe_sync_entry *syncs, u32 num_syncs,
-	       bool first_op, bool last_op)
+	       struct xe_exec_queue *q, struct xe_sync_entry *syncs,
+	       u32 num_syncs, bool first_op, bool last_op)
 {
 	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
-	int err;
-
-	xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type));
-
-	if (!xe_vma_has_no_bo(vma)) {
-		err = xe_bo_migrate(xe_vma_bo(vma), region_to_mem_type[region]);
-		if (err)
-			return ERR_PTR(err);
-	}
 
 	if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) {
 		return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
@@ -2467,8 +2457,7 @@ static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
 				     op->flags & XE_VMA_OP_LAST);
 		break;
 	case DRM_GPUVA_OP_PREFETCH:
-		fence = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region,
-				       op->syncs, op->num_syncs,
+		fence = xe_vm_prefetch(vm, vma, op->q, op->syncs, op->num_syncs,
 				       op->flags & XE_VMA_OP_FIRST,
 				       op->flags & XE_VMA_OP_LAST);
 		break;
@@ -2694,8 +2683,17 @@ static int op_lock(struct drm_exec *exec, struct xe_vm *vm,
 		err = vma_lock(exec, gpuva_to_vma(op->base.unmap.va), false);
 		break;
 	case DRM_GPUVA_OP_PREFETCH:
-		err = vma_lock(exec, gpuva_to_vma(op->base.prefetch.va), true);
+	{
+		struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
+		u32 region = op->prefetch.region;
+
+		xe_assert(vm->xe, region <= ARRAY_SIZE(region_to_mem_type));
+
+		err = vma_lock(exec, vma, false);
+		if (!err && !xe_vma_has_no_bo(vma))
+			err = xe_bo_migrate(xe_vma_bo(vma), region);
 		break;
+	}
 	default:
 		drm_warn(&vm->xe->drm, "NOT POSSIBLE");
 	}
Signature · 1 line
2.34.1
author maintainer
#4 lore raw

PATCH v3 4/22 drm/xe: Add struct xe_vma_ops abstraction

In reply to Matthew Brost

Having a structure which encapsulates a list of VMA operations will help enable 1 job for the entire list.

---
 drivers/gpu/drm/xe/xe_vm.c       | 37 ++++++++++++++++++--------------
 drivers/gpu/drm/xe/xe_vm_types.h |  7 ++++++
 2 files changed, 28 insertions(+), 16 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +21 -16
index 0f552fd8d7e0..768fb223019b 100644
@@ -2244,7 +2244,7 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op)
 static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 				   struct drm_gpuva_ops *ops,
 				   struct xe_sync_entry *syncs, u32 num_syncs,
-				   struct list_head *ops_list, bool last)
+				   struct xe_vma_ops *vops, bool last)
 {
 	struct xe_vma_op *last_op = NULL;
 	struct drm_gpuva_op *__op;
@@ -2255,11 +2255,11 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 	drm_gpuva_for_each_op(__op, ops) {
 		struct xe_vma_op *op = gpuva_op_to_vma_op(__op);
 		struct xe_vma *vma;
-		bool first = list_empty(ops_list);
+		bool first = list_empty(&vops->list);
 		unsigned int flags = 0;
 
 		INIT_LIST_HEAD(&op->link);
-		list_add_tail(&op->link, ops_list);
+		list_add_tail(&op->link, &vops->list);
 
 		if (first) {
 			op->flags |= XE_VMA_OP_FIRST;
@@ -2371,7 +2371,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 	}
 
 	/* FIXME: Unhandled corner case */
-	XE_WARN_ON(!last_op && last && !list_empty(ops_list));
+	XE_WARN_ON(!last_op && last && !list_empty(&vops->list));
 
 	if (!last_op)
 		return 0;
@@ -2703,7 +2703,7 @@ static int op_lock(struct drm_exec *exec, struct xe_vm *vm,
 
 static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
 				  struct xe_vm *vm,
-				  struct list_head *ops_list)
+				  struct xe_vma_ops *vops)
 {
 	struct xe_vma_op *op;
 	int err;
@@ -2712,7 +2712,7 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
 	if (err)
 		return err;
 
-	list_for_each_entry(op, ops_list, link) {
+	list_for_each_entry(op, &vops->list, link) {
 		err = op_lock(exec, vm, op);
 		if (err)
 			return err;
@@ -2722,13 +2722,13 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
 }
 
 static struct dma_fence *ops_execute(struct xe_vm *vm,
-				     struct list_head *ops_list,
+				     struct xe_vma_ops *vops,
 				     bool cleanup)
 {
 	struct xe_vma_op *op, *next;
 	struct dma_fence *fence = NULL;
 
-	list_for_each_entry_safe(op, next, ops_list, link) {
+	list_for_each_entry_safe(op, next, &vops->list, link) {
 		if (!IS_ERR(fence)) {
 			dma_fence_put(fence);
 			fence = xe_vma_op_execute(vm, op);
@@ -2746,7 +2746,7 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
 }
 
 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
-				     struct list_head *ops_list)
+				     struct xe_vma_ops *vops)
 {
 	struct drm_exec exec;
 	struct dma_fence *fence;
@@ -2756,12 +2756,12 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 
 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0);
 	drm_exec_until_all_locked(&exec) {
-		err = vm_bind_ioctl_ops_lock(&exec, vm, ops_list);
+		err = vm_bind_ioctl_ops_lock(&exec, vm, vops);
 		drm_exec_retry_on_contention(&exec);
 		if (err)
 			goto unlock;
 
-		fence = ops_execute(vm, ops_list, true);
+		fence = ops_execute(vm, vops, true);
 		if (IS_ERR(fence)) {
 			err = PTR_ERR(fence);
 			/* FIXME: Killing VM rather than proper error handling */
@@ -2922,6 +2922,11 @@ static int vm_bind_ioctl_signal_fences(struct xe_vm *vm,
 	return err;
 }
 
+static void xe_vma_ops_init(struct xe_vma_ops *vops)
+{
+	INIT_LIST_HEAD(&vops->list);
+}
+
 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
 	struct xe_device *xe = to_xe_device(dev);
@@ -2935,7 +2940,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	u32 num_syncs, num_ufence = 0;
 	struct xe_sync_entry *syncs = NULL;
 	struct drm_xe_vm_bind_op *bind_ops;
-	LIST_HEAD(ops_list);
+	struct xe_vma_ops vops;
 	int err;
 	int i;
 
@@ -3084,6 +3089,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		goto free_syncs;
 	}
 
+	xe_vma_ops_init(&vops);
 	for (i = 0; i < args->num_binds; ++i) {
 		u64 range = bind_ops[i].range;
 		u64 addr = bind_ops[i].addr;
@@ -3103,14 +3109,13 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		}
 
 		err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs,
-					      &ops_list,
-					      i == args->num_binds - 1);
+					      &vops, i == args->num_binds - 1);
 		if (err)
 			goto unwind_ops;
 	}
 
 	/* Nothing to do */
-	if (list_empty(&ops_list)) {
+	if (list_empty(&vops.list)) {
 		err = -ENODATA;
 		goto unwind_ops;
 	}
@@ -3119,7 +3124,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	if (q)
 		xe_exec_queue_get(q);
 
-	err = vm_bind_ioctl_ops_execute(vm, &ops_list);
+	err = vm_bind_ioctl_ops_execute(vm, &vops);
 
 	up_write(&vm->lock);
 
drivers/gpu/drm/xe/xe_vm_types.h +7 -0
index 5ac9c5bebabc..0c70062f3065 100644
@@ -366,4 +366,11 @@ struct xe_vma_op {
 		struct xe_vma_op_prefetch prefetch;
 	};
 };
+
+/** struct xe_vma_ops - VMA operations */
+struct xe_vma_ops {
+	/** @list: list of VMA operations */
+	struct list_head list;
+};
+
 #endif
Signature · 1 line
2.34.1
author maintainer
#5 lore raw

PATCH v3 5/22 drm/xe: Update xe_vm_rebind to use dummy VMA operations

In reply to Matthew Brost

All bind interfaces are transitioning to use VMA ops, update xe_vm_rebind to use VMA ops.

---
 drivers/gpu/drm/xe/xe_vm.c       |  44 ++++---
 drivers/gpu/drm/xe/xe_vm_types.h | 191 ++++++++++++++++---------------
 2 files changed, 130 insertions(+), 105 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +31 -13
index 768fb223019b..f59439500095 100644
@@ -753,10 +753,22 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
 		list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN;
 }
 
-static struct dma_fence *
-xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
-	       struct xe_sync_entry *syncs, u32 num_syncs,
-	       bool first_op, bool last_op);
+static void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
+{
+	vm->dummy_ops.op.base.op = DRM_GPUVA_OP_MAP;
+	vm->dummy_ops.op.base.map.va.addr = vma->gpuva.va.addr;
+	vm->dummy_ops.op.base.map.va.range = vma->gpuva.va.range;
+	vm->dummy_ops.op.base.map.gem.obj = vma->gpuva.gem.obj;
+	vm->dummy_ops.op.base.map.gem.offset = vma->gpuva.gem.offset;
+	vm->dummy_ops.op.map.vma = vma;
+	vm->dummy_ops.op.map.immediate = true;
+	vm->dummy_ops.op.map.read_only = xe_vma_read_only(vma);
+	vm->dummy_ops.op.map.is_null = xe_vma_is_null(vma);
+}
+
+static struct dma_fence *ops_execute(struct xe_vm *vm,
+				     struct xe_vma_ops *vops,
+				     bool cleanup);
 
 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 {
@@ -778,7 +790,9 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 			trace_xe_vma_rebind_worker(vma);
 		else
 			trace_xe_vma_rebind_exec(vma);
-		fence = xe_vm_bind_vma(vma, NULL, NULL, 0, false, false);
+
+		xe_vm_populate_dummy_rebind(vm, vma);
+		fence = ops_execute(vm, &vm->dummy_ops.vops, false);
 		if (IS_ERR(fence))
 			return fence;
 	}
@@ -1267,6 +1281,11 @@ static void xe_vm_free_scratch(struct xe_vm *vm)
 	}
 }
 
+static void xe_vma_ops_init(struct xe_vma_ops *vops)
+{
+	INIT_LIST_HEAD(&vops->list);
+}
+
 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 {
 	struct drm_gem_object *vm_resv_obj;
@@ -1287,6 +1306,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 
 	init_rwsem(&vm->lock);
 
+	xe_vma_ops_init(&vm->dummy_ops.vops);
+	INIT_LIST_HEAD(&vm->dummy_ops.op.link);
+	list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
+
 	INIT_LIST_HEAD(&vm->rebind_list);
 
 	INIT_LIST_HEAD(&vm->userptr.repin_list);
@@ -2391,7 +2414,7 @@ static struct dma_fence *op_execute(struct xe_vm *vm, struct xe_vma *vma,
 {
 	struct dma_fence *fence = NULL;
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 	xe_bo_assert_held(xe_vma_bo(vma));
 
@@ -2481,7 +2504,7 @@ __xe_vma_op_execute(struct xe_vm *vm, struct xe_vma *vma,
 retry_userptr:
 	fence = op_execute(vm, vma, op);
 	if (IS_ERR(fence) && PTR_ERR(fence) == -EAGAIN) {
-		lockdep_assert_held_write(&vm->lock);
+		lockdep_assert_held(&vm->lock);
 
 		if (op->base.op == DRM_GPUVA_OP_REMAP) {
 			if (!op->remap.unmap_done)
@@ -2510,7 +2533,7 @@ xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
 {
 	struct dma_fence *fence = ERR_PTR(-ENOMEM);
 
-	lockdep_assert_held_write(&vm->lock);
+	lockdep_assert_held(&vm->lock);
 
 	switch (op->base.op) {
 	case DRM_GPUVA_OP_MAP:
@@ -2922,11 +2945,6 @@ static int vm_bind_ioctl_signal_fences(struct xe_vm *vm,
 	return err;
 }
 
-static void xe_vma_ops_init(struct xe_vma_ops *vops)
-{
-	INIT_LIST_HEAD(&vops->list);
-}
-
 int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
 	struct xe_device *xe = to_xe_device(dev);
drivers/gpu/drm/xe/xe_vm_types.h +99 -92

213 lines not on this page · raw message

Signature · 1 line
2.34.1
author maintainer
#6 lore raw

PATCH v3 6/22 drm/xe: Simplify VM bind IOCTL error handling and cleanup

In reply to Matthew Brost

Clean up everything in VM bind IOCTL in 1 path for both errors and non-errors. Also move VM bind IOCTL cleanup from ops (also used by non-IOCTL binds) to the VM bind IOCTL.

---
 drivers/gpu/drm/xe/xe_vm.c       | 60 +++++---------------------------
 drivers/gpu/drm/xe/xe_vm_types.h |  5 ---
 2 files changed, 9 insertions(+), 56 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +9 -51
index f59439500095..50c85da430c6 100644
@@ -767,8 +767,7 @@ static void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma)
 }
 
 static struct dma_fence *ops_execute(struct xe_vm *vm,
-				     struct xe_vma_ops *vops,
-				     bool cleanup);
+				     struct xe_vma_ops *vops);
 
 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 {
@@ -792,7 +791,7 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker)
 			trace_xe_vma_rebind_exec(vma);
 
 		xe_vm_populate_dummy_rebind(vm, vma);
-		fence = ops_execute(vm, &vm->dummy_ops.vops, false);
+		fence = ops_execute(vm, &vm->dummy_ops.vops);
 		if (IS_ERR(fence))
 			return fence;
 	}
@@ -2399,7 +2398,6 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q,
 	if (!last_op)
 		return 0;
 
-	last_op->ops = ops;
 	if (last) {
 		last_op->flags |= XE_VMA_OP_LAST;
 		last_op->num_syncs = num_syncs;
@@ -2569,25 +2567,6 @@ xe_vma_op_execute(struct xe_vm *vm, struct xe_vma_op *op)
 	return fence;
 }
 
-static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op)
-{
-	bool last = op->flags & XE_VMA_OP_LAST;
-
-	if (last) {
-		while (op->num_syncs--)
-			xe_sync_entry_cleanup(&op->syncs[op->num_syncs]);
-		kfree(op->syncs);
-		if (op->q)
-			xe_exec_queue_put(op->q);
-	}
-	if (!list_empty(&op->link))
-		list_del(&op->link);
-	if (op->ops)
-		drm_gpuva_ops_free(&vm->gpuvm, op->ops);
-	if (last)
-		xe_vm_put(vm);
-}
-
 static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op,
 			     bool post_commit, bool prev_post_commit,
 			     bool next_post_commit)
@@ -2664,8 +2643,6 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm,
 					 op->flags & XE_VMA_OP_PREV_COMMITTED,
 					 op->flags & XE_VMA_OP_NEXT_COMMITTED);
 		}
-
-		drm_gpuva_ops_free(&vm->gpuvm, __ops);
 	}
 }
 
@@ -2745,8 +2722,7 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
 }
 
 static struct dma_fence *ops_execute(struct xe_vm *vm,
-				     struct xe_vma_ops *vops,
-				     bool cleanup)
+				     struct xe_vma_ops *vops)
 {
 	struct xe_vma_op *op, *next;
 	struct dma_fence *fence = NULL;
@@ -2761,8 +2737,6 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
 				 op->base.op, PTR_ERR(fence));
 			fence = ERR_PTR(-ENOSPC);
 		}
-		if (cleanup)
-			xe_vma_op_cleanup(vm, op);
 	}
 
 	return fence;
@@ -2784,7 +2758,7 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 		if (err)
 			goto unlock;
 
-		fence = ops_execute(vm, vops, true);
+		fence = ops_execute(vm, vops);
 		if (IS_ERR(fence)) {
 			err = PTR_ERR(fence);
 			/* FIXME: Killing VM rather than proper error handling */
@@ -3138,30 +3112,14 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		goto unwind_ops;
 	}
 
-	xe_vm_get(vm);
-	if (q)
-		xe_exec_queue_get(q);
-
 	err = vm_bind_ioctl_ops_execute(vm, &vops);
 
-	up_write(&vm->lock);
-
-	if (q)
-		xe_exec_queue_put(q);
-	xe_vm_put(vm);
-
-	for (i = 0; bos && i < args->num_binds; ++i)
-		xe_bo_put(bos[i]);
-
-	kfree(bos);
-	kfree(ops);
-	if (args->num_binds > 1)
-		kfree(bind_ops);
-
-	return err;
-
 unwind_ops:
-	vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
+	if (err && err != -ENODATA)
+		vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);
+	for (i = args->num_binds - 1; i >= 0; --i)
+		if (ops[i])
+			drm_gpuva_ops_free(&vm->gpuvm, ops[i]);
 free_syncs:
 	if (err == -ENODATA)
 		err = vm_bind_ioctl_signal_fences(vm, q, syncs, num_syncs);
drivers/gpu/drm/xe/xe_vm_types.h +0 -5
index 56665f946949..5020e0571108 100644
@@ -171,11 +171,6 @@ enum xe_vma_op_flags {
 struct xe_vma_op {
 	/** @base: GPUVA base operation */
 	struct drm_gpuva_op base;
-	/**
-	 * @ops: GPUVA ops, when set call drm_gpuva_ops_free after this
-	 * operations is processed
-	 */
-	struct drm_gpuva_ops *ops;
 	/** @q: exec queue for this operation */
 	struct xe_exec_queue *q;
 	/**
Signature · 1 line
2.34.1
author maintainer
#7 lore raw

PATCH v3 7/22 drm/xe: Update pagefaults to use dummy VMA operations

In reply to Matthew Brost

All bind interfaces are transitioning to use VMA ops, update pagefaults to use VMA ops.

---
 drivers/gpu/drm/xe/xe_gt_pagefault.c |  7 ++--
 drivers/gpu/drm/xe/xe_vm.c           | 54 +++++++++++++++++++---------
 drivers/gpu/drm/xe/xe_vm.h           |  3 ++
 drivers/gpu/drm/xe/xe_vm_types.h     |  2 ++
 4 files changed, 47 insertions(+), 19 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +4 -3
index c26e4fcca01e..4fd81a553bd0 100644
@@ -19,7 +19,6 @@
 #include "xe_guc.h"
 #include "xe_guc_ct.h"
 #include "xe_migrate.h"
-#include "xe_pt.h"
 #include "xe_trace.h"
 #include "xe_vm.h"
 
@@ -207,8 +206,10 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
 
 	/* Bind VMA only to the GT that has faulted */
 	trace_xe_vma_pf_bind(vma);
-	fence = __xe_pt_bind_vma(tile, vma, xe_tile_migrate_engine(tile), NULL, 0,
-				 vma->tile_present & BIT(tile->id));
+	xe_vm_populate_dummy_rebind(vm, vma);
+	vm->dummy_ops.op.tile_mask = BIT(tile->id);
+	vm->dummy_ops.op.q = xe_tile_migrate_engine(tile);
+	fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
 	if (IS_ERR(fence)) {
 		ret = PTR_ERR(fence);
 		goto unlock_dma_resv;
drivers/gpu/drm/xe/xe_vm.c +38 -16

158 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +3 -0
index df4a82e960ff..1d9a3b13aecc 100644
@@ -262,6 +262,9 @@ static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)
  */
 #define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))
 
+void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma);
+struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
+
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
 #define vm_dbg drm_dbg
 #else
drivers/gpu/drm/xe/xe_vm_types.h +2 -0
index 5020e0571108..6655a0645a18 100644
@@ -184,6 +184,8 @@ struct xe_vma_op {
 	struct list_head link;
 	/** @flags: operation flags */
 	enum xe_vma_op_flags flags;
+	/** @tile_mask: Tile mask for operation */
+	u8 tile_mask;
 
 	union {
 		/** @map: VMA map operation specific data */
Signature · 1 line
2.34.1
author maintainer
#8 lore raw

PATCH v3 8/22 drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue

In reply to Matthew Brost

xe_engine is now xe_exec_queue, adjust this function's name to reflect.

---
 drivers/gpu/drm/xe/xe_gt_pagefault.c | 2 +-
 drivers/gpu/drm/xe/xe_migrate.c      | 8 ++++----
 drivers/gpu/drm/xe/xe_migrate.h      | 3 ++-
 3 files changed, 7 insertions(+), 6 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +1 -1
index 4fd81a553bd0..5a7bd63025b3 100644
@@ -208,7 +208,7 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
 	trace_xe_vma_pf_bind(vma);
 	xe_vm_populate_dummy_rebind(vm, vma);
 	vm->dummy_ops.op.tile_mask = BIT(tile->id);
-	vm->dummy_ops.op.q = xe_tile_migrate_engine(tile);
+	vm->dummy_ops.op.q = xe_tile_migrate_exec_queue(tile);
 	fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
 	if (IS_ERR(fence)) {
 		ret = PTR_ERR(fence);
drivers/gpu/drm/xe/xe_migrate.c +4 -4
index 3d2438dc86ee..7b1dde578176 100644
@@ -83,15 +83,15 @@ struct xe_migrate {
 #define MAX_PTE_PER_SDI 0x1FE
 
 /**
- * xe_tile_migrate_engine() - Get this tile's migrate engine.
+ * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue.
  * @tile: The tile.
  *
- * Returns the default migrate engine of this tile.
+ * Returns the default migrate exec queue of this tile.
  * TODO: Perhaps this function is slightly misplaced, and even unneeded?
  *
- * Return: The default migrate engine
+ * Return: The default migrate exec queue
  */
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile)
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)
 {
 	return tile->migrate->q;
 }
drivers/gpu/drm/xe/xe_migrate.h +2 -1
index 951f19318ea4..9935ce336bae 100644
@@ -106,5 +106,6 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
 
 void xe_migrate_wait(struct xe_migrate *m);
 
-struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile);
+struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
+
 #endif
Signature · 1 line
2.34.1
author maintainer
#9 lore raw

PATCH v3 9/22 drm/xe: Add vm_bind_ioctl_ops_install_fences helper

In reply to Matthew Brost

Simplify VM bind code by signaling out-fences / destroying VMAs in a single location. Will help with transition single job for many bind ops.

---
 drivers/gpu/drm/xe/xe_vm.c | 55 ++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 32 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +23 -32
index ebe822855233..383d05f87a79 100644
@@ -1625,7 +1625,7 @@ xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
 	struct dma_fence *fence = NULL;
 	struct dma_fence **fences = NULL;
 	struct dma_fence_array *cf = NULL;
-	int cur_fence = 0, i;
+	int cur_fence = 0;
 	int number_tiles = hweight8(vma->tile_present);
 	int err;
 	u8 id;
@@ -1673,10 +1673,6 @@ xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
 
 	fence = cf ? &cf->base : !fence ?
 		xe_exec_queue_last_fence_get(wait_exec_queue, vm) : fence;
-	if (last_op) {
-		for (i = 0; i < num_syncs; i++)
-			xe_sync_entry_signal(&syncs[i], NULL, fence);
-	}
 
 	return fence;
 
@@ -1700,7 +1696,7 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
 	struct dma_fence **fences = NULL;
 	struct dma_fence_array *cf = NULL;
 	struct xe_vm *vm = xe_vma_vm(vma);
-	int cur_fence = 0, i;
+	int cur_fence = 0;
 	int number_tiles = hweight8(vma->tile_mask);
 	int err;
 	u8 id;
@@ -1747,12 +1743,6 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q,
 		}
 	}
 
-	if (last_op) {
-		for (i = 0; i < num_syncs; i++)
-			xe_sync_entry_signal(&syncs[i], NULL,
-					     cf ? &cf->base : fence);
-	}
-
 	return cf ? &cf->base : fence;
 
 err_fences:
@@ -1782,15 +1772,8 @@ xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
 		if (IS_ERR(fence))
 			return fence;
 	} else {
-		int i;
-
 		xe_assert(vm->xe, xe_vm_in_fault_mode(vm));
-
 		fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
-		if (last_op) {
-			for (i = 0; i < num_syncs; i++)
-				xe_sync_entry_signal(&syncs[i], NULL, fence);
-		}
 	}
 
 	if (last_op)
@@ -1814,7 +1797,6 @@ xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
 	if (IS_ERR(fence))
 		return fence;
 
-	xe_vma_destroy(vma, fence);
 	if (last_op)
 		xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
 
@@ -1972,17 +1954,7 @@ xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma,
 		return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs,
 				  vma->tile_mask, true, first_op, last_op);
 	} else {
-		struct dma_fence *fence =
-			xe_exec_queue_last_fence_get(wait_exec_queue, vm);
-		int i;
-
-		/* Nothing to do, signal fences now */
-		if (last_op) {
-			for (i = 0; i < num_syncs; i++)
-				xe_sync_entry_signal(&syncs[i], NULL, fence);
-		}
-
-		return fence;
+		return xe_exec_queue_last_fence_get(wait_exec_queue, vm);
 	}
 }
 
@@ -2764,6 +2736,25 @@ struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops)
 	return fence;
 }
 
+static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
+					     struct xe_vma_ops *vops,
+					     struct dma_fence *fence)
+{
+	struct xe_vma_op *op;
+	int i;
+
+	list_for_each_entry(op, &vops->list, link) {
+		if (op->base.op == DRM_GPUVA_OP_UNMAP)
+			xe_vma_destroy(gpuva_to_vma(op->base.unmap.va), fence);
+		else if (op->base.op == DRM_GPUVA_OP_REMAP)
+			xe_vma_destroy(gpuva_to_vma(op->base.remap.unmap->va),
+				       fence);
+	}
+	for (i = 0; i < vops->num_syncs; i++)
+		xe_sync_entry_signal(vops->syncs + i, NULL, fence);
+	dma_fence_put(fence);
+}
+
 static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 				     struct xe_vma_ops *vops)
 {
@@ -2787,7 +2778,7 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 			xe_vm_kill(vm, false);
 			goto unlock;
 		} else {
-			dma_fence_put(fence);
+			vm_bind_ioctl_ops_install_fences(vm, vops, fence);
 		}
 	}
 
Signature · 1 line
2.34.1
author maintainer
#10 lore raw

PATCH v3 10/22 drm/xe: Move setting last fence to vm_bind_ioctl_ops_install_fences

In reply to Matthew Brost

This moves setting of the last fence to a single location.

---
 drivers/gpu/drm/xe/xe_vm.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +2 -7
index 383d05f87a79..72f6faf6f9ef 100644
@@ -1776,9 +1776,6 @@ xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q,
 		fence = xe_exec_queue_last_fence_get(wait_exec_queue, vm);
 	}
 
-	if (last_op)
-		xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
-
 	return fence;
 }
 
@@ -1788,7 +1785,6 @@ xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
 	     u32 num_syncs, bool first_op, bool last_op)
 {
 	struct dma_fence *fence;
-	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, q);
 
 	xe_vm_assert_held(vm);
 	xe_bo_assert_held(xe_vma_bo(vma));
@@ -1797,9 +1793,6 @@ xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma,
 	if (IS_ERR(fence))
 		return fence;
 
-	if (last_op)
-		xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
-
 	return fence;
 }
 
@@ -2740,6 +2733,7 @@ static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
 					     struct xe_vma_ops *vops,
 					     struct dma_fence *fence)
 {
+	struct xe_exec_queue *wait_exec_queue = to_wait_exec_queue(vm, vops->q);
 	struct xe_vma_op *op;
 	int i;
 
@@ -2752,6 +2746,7 @@ static void vm_bind_ioctl_ops_install_fences(struct xe_vm *vm,
 	}
 	for (i = 0; i < vops->num_syncs; i++)
 		xe_sync_entry_signal(vops->syncs + i, NULL, fence);
+	xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence);
 	dma_fence_put(fence);
 }
 
Signature · 1 line
2.34.1
author maintainer
#11 lore raw

PATCH v3 11/22 drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this

In reply to Matthew Brost

xe_gt_tlb_invalidation_range accepts a start and end address rather than a VMA. This will enable multiple VMAs to be invalidated in a single invalidation. Update the PT layer to use this new function.

---
 drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 60 +++++++++++++++------
 drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h |  3 ++
 drivers/gpu/drm/xe/xe_pt.c                  | 25 ++++++---
 3 files changed, 66 insertions(+), 22 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +45 -15
index e3a4131ebb58..3babc143abc3 100644
@@ -223,11 +223,15 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
 }
 
 /**
- * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
+ * xe_gt_tlb_invalidation_range - Issue a TLB invalidation on this GT for an
+ * address range
+ *
  * @gt: graphics tile
  * @fence: invalidation fence which will be signal on TLB invalidation
  * completion, can be NULL
- * @vma: VMA to invalidate
+ * @start: start address
+ * @end: end address
+ * @asid: address space id
  *
  * Issue a range based TLB invalidation if supported, if not fallback to a full
  * TLB invalidation. Completion of TLB is asynchronous and caller can either use
@@ -237,24 +241,22 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
  * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
  * negative error code on error.
  */
-int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
-			       struct xe_gt_tlb_invalidation_fence *fence,
-			       struct xe_vma *vma)
+int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
+				 struct xe_gt_tlb_invalidation_fence *fence,
+				 u64 start, u64 end, u32 asid)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 #define MAX_TLB_INVALIDATION_LEN	7
 	u32 action[MAX_TLB_INVALIDATION_LEN];
 	int len = 0;
 
-	xe_gt_assert(gt, vma);
-
 	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
 	action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
 	if (!xe->info.has_range_tlb_invalidation) {
 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
 	} else {
-		u64 start = xe_vma_start(vma);
-		u64 length = xe_vma_size(vma);
+		u64 orig_start = start;
+		u64 length = end - start;
 		u64 align, end;
 
 		if (length < SZ_4K)
@@ -267,12 +269,12 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
 		 * address mask covering the required range.
 		 */
 		align = roundup_pow_of_two(length);
-		start = ALIGN_DOWN(xe_vma_start(vma), align);
-		end = ALIGN(xe_vma_end(vma), align);
+		start = ALIGN_DOWN(start, align);
+		end = ALIGN(end, align);
 		length = align;
 		while (start + length < end) {
 			length <<= 1;
-			start = ALIGN_DOWN(xe_vma_start(vma), length);
+			start = ALIGN_DOWN(orig_start, length);
 		}
 
 		/*
@@ -281,16 +283,17 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
 		 */
 		if (length >= SZ_2M) {
 			length = max_t(u64, SZ_16M, length);
-			start = ALIGN_DOWN(xe_vma_start(vma), length);
+			start = ALIGN_DOWN(orig_start, length);
 		}
 
 		xe_gt_assert(gt, length >= SZ_4K);
 		xe_gt_assert(gt, is_power_of_2(length));
-		xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, ilog2(SZ_2M) + 1)));
+		xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
+						    ilog2(SZ_2M) + 1)));
 		xe_gt_assert(gt, IS_ALIGNED(start, length));
 
 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
-		action[len++] = xe_vma_vm(vma)->usm.asid;
+		action[len++] = asid;
 		action[len++] = lower_32_bits(start);
 		action[len++] = upper_32_bits(start);
 		action[len++] = ilog2(length) - ilog2(SZ_4K);
@@ -299,6 +302,33 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
 	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
 
 	return send_tlb_invalidation(&gt->uc.guc, fence, action, len);
+
+}
+
+/**
+ * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
+ * @gt: graphics tile
+ * @fence: invalidation fence which will be signal on TLB invalidation
+ * completion, can be NULL
+ * @vma: VMA to invalidate
+ *
+ * Issue a range based TLB invalidation if supported, if not fallback to a full
+ * TLB invalidation. Completion of TLB is asynchronous and caller can either use
+ * the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for
+ * completion.
+ *
+ * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
+ * negative error code on error.
+ */
+int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
+			       struct xe_gt_tlb_invalidation_fence *fence,
+			       struct xe_vma *vma)
+{
+	xe_gt_assert(gt, vma);
+
+	return xe_gt_tlb_invalidation_range(gt, fence, xe_vma_start(vma),
+					    xe_vma_end(vma),
+					    xe_vma_vm(vma)->usm.asid);
 }
 
 /**
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +3 -0
index b333c1709397..5bb09885aa0f 100644
@@ -20,6 +20,9 @@ int xe_gt_tlb_invalidation_guc(struct xe_gt *gt);
 int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
 			       struct xe_gt_tlb_invalidation_fence *fence,
 			       struct xe_vma *vma);
+int xe_gt_tlb_invalidation_range(struct xe_gt *gt,
+				 struct xe_gt_tlb_invalidation_fence *fence,
+				 u64 start, u64 end, u32 asid);
 int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno);
 int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
 
drivers/gpu/drm/xe/xe_pt.c +18 -7
index 3a99bf6e558f..ada5a65e62dc 100644
@@ -1058,10 +1058,12 @@ static const struct xe_migrate_pt_update_ops userptr_bind_ops = {
 struct invalidation_fence {
 	struct xe_gt_tlb_invalidation_fence base;
 	struct xe_gt *gt;
-	struct xe_vma *vma;
 	struct dma_fence *fence;
 	struct dma_fence_cb cb;
 	struct work_struct work;
+	u64 start;
+	u64 end;
+	u32 asid;
 };
 
 static const char *
@@ -1104,13 +1106,14 @@ static void invalidation_fence_work_func(struct work_struct *w)
 		container_of(w, struct invalidation_fence, work);
 
 	trace_xe_gt_tlb_invalidation_fence_work_func(&ifence->base);
-	xe_gt_tlb_invalidation_vma(ifence->gt, &ifence->base, ifence->vma);
+	xe_gt_tlb_invalidation_range(ifence->gt, &ifence->base, ifence->start,
+				     ifence->end, ifence->asid);
 }
 
 static int invalidation_fence_init(struct xe_gt *gt,
 				   struct invalidation_fence *ifence,
 				   struct dma_fence *fence,
-				   struct xe_vma *vma)
+				   u64 start, u64 end, u32 asid)
 {
 	int ret;
 
@@ -1128,7 +1131,9 @@ static int invalidation_fence_init(struct xe_gt *gt,
 	dma_fence_get(&ifence->base.base);	/* Ref for caller */
 	ifence->fence = fence;
 	ifence->gt = gt;
-	ifence->vma = vma;
+	ifence->start = start;
+	ifence->end = end;
+	ifence->asid = asid;
 
 	INIT_WORK(&ifence->work, invalidation_fence_work_func);
 	ret = dma_fence_add_callback(fence, &ifence->cb, invalidation_fence_cb);
@@ -1270,8 +1275,11 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue
 
 		/* TLB invalidation must be done before signaling rebind */
 		if (ifence) {
-			int err = invalidation_fence_init(tile->primary_gt, ifence, fence,
-							  vma);
+			int err = invalidation_fence_init(tile->primary_gt,
+							  ifence, fence,
+							  xe_vma_start(vma),
+							  xe_vma_end(vma),
+							  xe_vma_vm(vma)->usm.asid);
 			if (err) {
 				dma_fence_put(fence);
 				kfree(ifence);
@@ -1609,7 +1617,10 @@ __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queu
 			dma_fence_wait(fence, false);
 
 		/* TLB invalidation must be done before signaling unbind */
-		err = invalidation_fence_init(tile->primary_gt, ifence, fence, vma);
+		err = invalidation_fence_init(tile->primary_gt, ifence, fence,
+					      xe_vma_start(vma),
+					      xe_vma_end(vma),
+					      xe_vma_vm(vma)->usm.asid);
 		if (err) {
 			dma_fence_put(fence);
 			kfree(ifence);
Signature · 1 line
2.34.1
author maintainer
#12 lore raw

PATCH v3 12/22 drm/xe: Add some members to xe_vma_ops

In reply to Matthew Brost

This will help with moving to single jobs for many bind operations.

---
 drivers/gpu/drm/xe/xe_vm.c       | 13 ++++++++++---
 drivers/gpu/drm/xe/xe_vm_types.h |  8 ++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)
drivers/gpu/drm/xe/xe_vm.c +10 -3
index 72f6faf6f9ef..c66e8cff07f8 100644
@@ -1285,9 +1285,16 @@ static void xe_vm_free_scratch(struct xe_vm *vm)
 	}
 }
 
-static void xe_vma_ops_init(struct xe_vma_ops *vops)
+static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,
+			    struct xe_exec_queue *q,
+			    struct xe_sync_entry *syncs, u32 num_syncs)
 {
+	memset(vops, 0, sizeof(*vops));
 	INIT_LIST_HEAD(&vops->list);
+	vops->vm = vm;
+	vops->q = q;
+	vops->syncs = syncs;
+	vops->num_syncs = num_syncs;
 }
 
 struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
@@ -1310,7 +1317,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags)
 
 	init_rwsem(&vm->lock);
 
-	xe_vma_ops_init(&vm->dummy_ops.vops);
+	xe_vma_ops_init(&vm->dummy_ops.vops, vm, NULL, NULL, 0);
 	INIT_LIST_HEAD(&vm->dummy_ops.op.link);
 	list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
 
@@ -3089,7 +3096,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		goto free_syncs;
 	}
 
-	xe_vma_ops_init(&vops);
+	xe_vma_ops_init(&vops, vm, q, syncs, num_syncs);
 	for (i = 0; i < args->num_binds; ++i) {
 		u64 range = bind_ops[i].range;
 		u64 addr = bind_ops[i].addr;
drivers/gpu/drm/xe/xe_vm_types.h +8 -0
index 6655a0645a18..4ba05315476f 100644
@@ -201,6 +201,14 @@ struct xe_vma_op {
 struct xe_vma_ops {
 	/** @list: list of VMA operations */
 	struct list_head list;
+	/** @vm: VM */
+	struct xe_vm *vm;
+	/** @q: exec queue these operations */
+	struct xe_exec_queue *q;
+	/** @syncs: syncs these operation */
+	struct xe_sync_entry *syncs;
+	/** @num_syncs: number of syncs */
+	u32 num_syncs;
 };
 
 struct xe_vm {
Signature · 1 line
2.34.1
author maintainer
#13 lore raw

PATCH v3 13/22 drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops

In reply to Matthew Brost

Will help with the converstion to 1 job per VM bind IOCTL. Allocation only implemented in this patch.

---
 drivers/gpu/drm/xe/xe_pt_types.h | 12 ++++++
 drivers/gpu/drm/xe/xe_vm.c       | 67 ++++++++++++++++++++++++++++++--
 drivers/gpu/drm/xe/xe_vm_types.h |  8 ++++
 3 files changed, 83 insertions(+), 4 deletions(-)
drivers/gpu/drm/xe/xe_pt_types.h +12 -0
index cee70cb0f014..2093150f461e 100644
@@ -74,4 +74,16 @@ struct xe_vm_pgtable_update {
 	u32 flags;
 };
 
+/** struct xe_vm_pgtable_update_op - Page table update operation */
+struct xe_vm_pgtable_update_op {
+	/** @entries: entries to update for this operation */
+	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
+	/** @num_entries: number of entries for this update operation */
+	u32 num_entries;
+	/** @bind: is a bind */
+	bool bind;
+	/** @rebind: is a rebind */
+	bool rebind;
+};
+
 #endif
drivers/gpu/drm/xe/xe_vm.c +63 -4

135 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm_types.h +8 -0
index 4ba05315476f..1506c0d1338d 100644
@@ -21,6 +21,7 @@ struct xe_bo;
 struct xe_device;
 struct xe_sync_entry;
 struct xe_vm;
+struct xe_vm_pgtable_update_op;
 
 #define XE_VMA_READ_ONLY	DRM_GPUVA_USERBITS
 #define XE_VMA_DESTROYED	(DRM_GPUVA_USERBITS << 1)
@@ -209,6 +210,13 @@ struct xe_vma_ops {
 	struct xe_sync_entry *syncs;
 	/** @num_syncs: number of syncs */
 	u32 num_syncs;
+	/** @pt_update_ops: page table update operations */
+	struct {
+		/** @ops: operations */
+		struct xe_vm_pgtable_update_op *ops;
+		/** @num_ops: number of operations */
+		u32 num_ops;
+	} pt_update_ops[XE_MAX_TILES_PER_DEVICE];
 };
 
 struct xe_vm {
Signature · 1 line
2.34.1
author maintainer
#14 lore raw

PATCH v3 14/22 drm/xe: Convert multiple bind ops into single job

In reply to Matthew Brost

This aligns with the uAPI of an array of binds or single bind that results in multiple GPUVA ops to be considered a single atomic operations.

---
 drivers/gpu/drm/xe/xe_bo_types.h     |    2 +
 drivers/gpu/drm/xe/xe_exec.c         |   25 +-
 drivers/gpu/drm/xe/xe_gt_pagefault.c |    6 +-
 drivers/gpu/drm/xe/xe_migrate.c      |  298 ++++----
 drivers/gpu/drm/xe/xe_migrate.h      |   32 +-
 drivers/gpu/drm/xe/xe_pt.c           | 1006 ++++++++++++++++----------
 drivers/gpu/drm/xe/xe_pt.h           |    7 +
 drivers/gpu/drm/xe/xe_pt_types.h     |   34 +
 drivers/gpu/drm/xe/xe_vm.c           |  534 ++++----------
 drivers/gpu/drm/xe/xe_vm.h           |    5 +-
 drivers/gpu/drm/xe/xe_vm_types.h     |   29 +-
 11 files changed, 959 insertions(+), 1019 deletions(-)
drivers/gpu/drm/xe/xe_bo_types.h +2 -0
index 14ef13b7b421..1ff5b5a68adc 100644
@@ -77,6 +77,8 @@ struct xe_bo {
 	} props;
 	/** @freed: List node for delayed put. */
 	struct llist_node freed;
+	/** @update_index: Update index if PT BO */
+	int update_index;
 	/** @created: Whether the bo has passed initial creation */
 	bool created;
 
drivers/gpu/drm/xe/xe_exec.c +2 -23
index 952496c6260d..4feff67620c4 100644
@@ -294,30 +294,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 		err = PTR_ERR(rebind_fence);
 		goto err_put_job;
 	}
+	dma_fence_put(rebind_fence);
 
-	/*
-	 * We store the rebind_fence in the VM so subsequent execs don't get
-	 * scheduled before the rebinds of userptrs / evicted BOs is complete.
-	 */
-	if (rebind_fence) {
-		dma_fence_put(vm->rebind_fence);
-		vm->rebind_fence = rebind_fence;
-	}
-	if (vm->rebind_fence) {
-		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
-			     &vm->rebind_fence->flags)) {
-			dma_fence_put(vm->rebind_fence);
-			vm->rebind_fence = NULL;
-		} else {
-			dma_fence_get(vm->rebind_fence);
-			err = drm_sched_job_add_dependency(&job->drm,
-							   vm->rebind_fence);
-			if (err)
-				goto err_put_job;
-		}
-	}
-
-	/* Wait behind munmap style rebinds */
+	/* Wait for rebinds */
 	if (!xe_vm_in_lr_mode(vm)) {
 		err = drm_sched_job_add_resv_dependencies(&job->drm,
 							  xe_vm_resv(vm),
drivers/gpu/drm/xe/xe_gt_pagefault.c +3 -3
index 5a7bd63025b3..27dbbdc9127e 100644
@@ -206,9 +206,9 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
 
 	/* Bind VMA only to the GT that has faulted */
 	trace_xe_vma_pf_bind(vma);
-	xe_vm_populate_dummy_rebind(vm, vma);
-	vm->dummy_ops.op.tile_mask = BIT(tile->id);
-	vm->dummy_ops.op.q = xe_tile_migrate_exec_queue(tile);
+	xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+	vm->dummy_ops.vops.pt_update_ops[tile->id].q =
+		xe_tile_migrate_exec_queue(tile);
 	fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
 	if (IS_ERR(fence)) {
 		ret = PTR_ERR(fence);
drivers/gpu/drm/xe/xe_migrate.c +134 -164

409 lines not on this page · raw message

drivers/gpu/drm/xe/xe_migrate.h +20 -12
index 9935ce336bae..bd8eba1d3552 100644
@@ -47,6 +47,24 @@ struct xe_migrate_pt_update_ops {
 			 struct xe_tile *tile, struct iosys_map *map,
 			 void *pos, u32 ofs, u32 num_qwords,
 			 const struct xe_vm_pgtable_update *update);
+	/**
+	 * @clear: Clear a command buffer or page-table with ptes.
+	 * @pt_update: Embeddable callback argument.
+	 * @tile: The tile for the current operation.
+	 * @map: struct iosys_map into the memory to be populated.
+	 * @pos: If @map is NULL, map into the memory to be populated.
+	 * @ofs: qword offset into @map, unused if @map is NULL.
+	 * @num_qwords: Number of qwords to write.
+	 * @update: Information about the PTEs to be inserted.
+	 *
+	 * This interface is intended to be used as a callback into the
+	 * page-table system to populate command buffers or shared
+	 * page-tables with PTEs.
+	 */
+	void (*clear)(struct xe_migrate_pt_update *pt_update,
+		      struct xe_tile *tile, struct iosys_map *map,
+		      void *pos, u32 ofs, u32 num_qwords,
+		      const struct xe_vm_pgtable_update *update);
 
 	/**
 	 * @pre_commit: Callback to be called just before arming the
@@ -67,14 +85,10 @@ struct xe_migrate_pt_update_ops {
 struct xe_migrate_pt_update {
 	/** @ops: Pointer to the struct xe_migrate_pt_update_ops callbacks */
 	const struct xe_migrate_pt_update_ops *ops;
-	/** @vma: The vma we're updating the pagetable for. */
-	struct xe_vma *vma;
+	/** @vops: VMA operations */
+	struct xe_vma_ops *vops;
 	/** @job: The job if a GPU page-table update. NULL otherwise */
 	struct xe_sched_job *job;
-	/** @start: Start of update for the range fence */
-	u64 start;
-	/** @last: Last of update for the range fence */
-	u64 last;
 	/** @tile_id: Tile ID of the update */
 	u8 tile_id;
 };
@@ -96,12 +110,6 @@ struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
 
 struct dma_fence *
 xe_migrate_update_pgtables(struct xe_migrate *m,
-			   struct xe_vm *vm,
-			   struct xe_bo *bo,
-			   struct xe_exec_queue *q,
-			   const struct xe_vm_pgtable_update *updates,
-			   u32 num_updates,
-			   struct xe_sync_entry *syncs, u32 num_syncs,
 			   struct xe_migrate_pt_update *pt_update);
 
 void xe_migrate_wait(struct xe_migrate *m);
drivers/gpu/drm/xe/xe_pt.c +621 -385

1,147 lines not on this page · raw message

drivers/gpu/drm/xe/xe_pt.h +7 -0
index 71a4fbfcff43..cbf8170d89cc 100644
@@ -17,6 +17,7 @@ struct xe_sync_entry;
 struct xe_tile;
 struct xe_vm;
 struct xe_vma;
+struct xe_vma_ops;
 
 /* Largest huge pte is currently 1GiB. May become device dependent. */
 #define MAX_HUGEPTE_LEVEL 2
@@ -34,6 +35,12 @@ void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
 
 void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred);
 
+int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops);
+struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
+				       struct xe_vma_ops *vops);
+void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
+
 struct dma_fence *
 __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
 		 struct xe_sync_entry *syncs, u32 num_syncs,
drivers/gpu/drm/xe/xe_pt_types.h +34 -0
index 2093150f461e..16252f1be055 100644
@@ -86,4 +86,38 @@ struct xe_vm_pgtable_update_op {
 	bool rebind;
 };
 
+/** struct xe_vm_pgtable_update_ops: page table update operations */
+struct xe_vm_pgtable_update_ops {
+	/** @ops: operations */
+	struct xe_vm_pgtable_update_op *ops;
+	/** @deferred: deferred list to destroy PT entries */
+	struct llist_head deferred;
+	/** @q: exec queue for PT operations */
+	struct xe_exec_queue *q;
+	/** @start: start address of ops */
+	u64 start;
+	/** @last: last address of ops */
+	u64 last;
+	/** @num_ops: number of operations */
+	u32 num_ops;
+	/** @current_op: current operations */
+	u32 current_op;
+	/** @needs_userptr_lock: Needs userptr lock */
+	bool needs_userptr_lock;
+	/** @needs_invalidation: Needs invalidation */
+	bool needs_invalidation;
+	/**
+	 * @wait_vm_bookkeep: PT operations need to wait until VM is idle
+	 * (bookkeep dma-resv slots are idle) and stage all future VM activity
+	 * behind these operations (install PT operations into VM kernel
+	 * dma-resv slot).
+	 */
+	bool wait_vm_bookkeep;
+	/**
+	 * @wait_vm_kernel: PT operations need to wait until VM kernel dma-resv
+	 * slots are idle.
+	 */
+	bool wait_vm_kernel;
+};
+
 #endif
drivers/gpu/drm/xe/xe_vm.c +127 -407

643 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +4 -1
index 1d9a3b13aecc..b8134804fce9 100644
@@ -262,9 +262,12 @@ static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)
  */
 #define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))
 
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma);
+void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+				 u8 tile_mask);
 struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
 
+void xe_vm_kill(struct xe_vm *vm, bool unlocked);
+
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
 #define vm_dbg drm_dbg
 #else
drivers/gpu/drm/xe/xe_vm_types.h +5 -24
index 1506c0d1338d..7e639ed5ba4a 100644
@@ -156,29 +156,18 @@ struct xe_vma_op_prefetch {
 
 /** enum xe_vma_op_flags - flags for VMA operation */
 enum xe_vma_op_flags {
-	/** @XE_VMA_OP_FIRST: first VMA operation for a set of syncs */
-	XE_VMA_OP_FIRST			= BIT(0),
-	/** @XE_VMA_OP_LAST: last VMA operation for a set of syncs */
-	XE_VMA_OP_LAST			= BIT(1),
 	/** @XE_VMA_OP_COMMITTED: VMA operation committed */
-	XE_VMA_OP_COMMITTED		= BIT(2),
+	XE_VMA_OP_COMMITTED		= BIT(0),
 	/** @XE_VMA_OP_PREV_COMMITTED: Previous VMA operation committed */
-	XE_VMA_OP_PREV_COMMITTED	= BIT(3),
+	XE_VMA_OP_PREV_COMMITTED	= BIT(1),
 	/** @XE_VMA_OP_NEXT_COMMITTED: Next VMA operation committed */
-	XE_VMA_OP_NEXT_COMMITTED	= BIT(4),
+	XE_VMA_OP_NEXT_COMMITTED	= BIT(2),
 };
 
 /** struct xe_vma_op - VMA operation */
 struct xe_vma_op {
 	/** @base: GPUVA base operation */
 	struct drm_gpuva_op base;
-	/** @q: exec queue for this operation */
-	struct xe_exec_queue *q;
-	/**
-	 * @syncs: syncs for this operation, only used on first and last
-	 * operation
-	 */
-	struct xe_sync_entry *syncs;
 	/** @num_syncs: number of syncs */
 	u32 num_syncs;
 	/** @link: async operation link */
@@ -204,19 +193,14 @@ struct xe_vma_ops {
 	struct list_head list;
 	/** @vm: VM */
 	struct xe_vm *vm;
-	/** @q: exec queue these operations */
+	/** @q: exec queue for VMA operations */
 	struct xe_exec_queue *q;
 	/** @syncs: syncs these operation */
 	struct xe_sync_entry *syncs;
 	/** @num_syncs: number of syncs */
 	u32 num_syncs;
 	/** @pt_update_ops: page table update operations */
-	struct {
-		/** @ops: operations */
-		struct xe_vm_pgtable_update_op *ops;
-		/** @num_ops: number of operations */
-		u32 num_ops;
-	} pt_update_ops[XE_MAX_TILES_PER_DEVICE];
+	struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
 };
 
 struct xe_vm {
@@ -268,9 +252,6 @@ struct xe_vm {
 	 */
 	struct list_head rebind_list;
 
-	/** @rebind_fence: rebind fence from execbuf */
-	struct dma_fence *rebind_fence;
-
 	/**
 	 * @destroy_work: worker to destroy VM, needed as a dma_fence signaling
 	 * from an irq context can be last put and the destroy needs to be able
Signature · 1 line
2.34.1
author maintainer
#15 lore raw

PATCH v3 15/22 drm/xe: Remove old functions defs in xe_pt.h

In reply to Matthew Brost

__xe_pt_bind_vma and __xe_pt_unbind_vma are unused, remove these.

---
 drivers/gpu/drm/xe/xe_pt.h | 9 ---------
 1 file changed, 9 deletions(-)
drivers/gpu/drm/xe/xe_pt.h +0 -9
index cbf8170d89cc..9ab386431cad 100644
@@ -41,15 +41,6 @@ struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
 
-struct dma_fence *
-__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
-		 struct xe_sync_entry *syncs, u32 num_syncs,
-		 bool rebind);
-
-struct dma_fence *
-__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q,
-		   struct xe_sync_entry *syncs, u32 num_syncs);
-
 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);
 
 #endif
Signature · 1 line
2.34.1
author maintainer
#16 lore raw

PATCH v3 16/22 drm/xe: Update PT layer with better error handling

In reply to Matthew Brost

Update PT layer so if a memory allocation for a PTE fails the error can be propagated to the user without requiring to be killed.

---
 drivers/gpu/drm/xe/xe_pt.c       | 210 ++++++++++++++++++++++++-------
 drivers/gpu/drm/xe/xe_pt_types.h |   2 +
 2 files changed, 165 insertions(+), 47 deletions(-)
drivers/gpu/drm/xe/xe_pt.c +163 -47
index 7b7b3f99321d..8160a295dd84 100644
@@ -796,19 +796,27 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *t
 	}
 }
 
-static void xe_pt_abort_bind(struct xe_vma *vma,
-			     struct xe_vm_pgtable_update *entries,
-			     u32 num_entries)
+static void xe_pt_cancel_bind(struct xe_vma *vma,
+			      struct xe_vm_pgtable_update *entries,
+			      u32 num_entries)
 {
 	u32 i, j;
 
 	for (i = 0; i < num_entries; i++) {
-		if (!entries[i].pt_entries)
+		struct xe_pt *pt = entries[i].pt;
+
+		if (!pt)
 			continue;
 
-		for (j = 0; j < entries[i].qwords; j++)
-			xe_pt_destroy(entries[i].pt_entries[j].pt, xe_vma_vm(vma)->flags, NULL);
+		if (pt->level) {
+			for (j = 0; j < entries[i].qwords; j++)
+				xe_pt_destroy(entries[i].pt_entries[j].pt,
+					      xe_vma_vm(vma)->flags, NULL);
+		}
+
 		kfree(entries[i].pt_entries);
+		entries[i].pt_entries = NULL;
+		entries[i].qwords = 0;
 	}
 }
 
@@ -824,10 +832,61 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma)
 	xe_vm_assert_held(vm);
 }
 
-static void xe_pt_commit_bind(struct xe_vma *vma,
-			      struct xe_vm_pgtable_update *entries,
-			      u32 num_entries, bool rebind,
-			      struct llist_head *deferred)
+static void xe_pt_commit(struct xe_vma *vma,
+			 struct xe_vm_pgtable_update *entries,
+			 u32 num_entries, struct llist_head *deferred)
+{
+	u32 i, j;
+
+	xe_pt_commit_locks_assert(vma);
+
+	for (i = 0; i < num_entries; i++) {
+		struct xe_pt *pt = entries[i].pt;
+
+		if (!pt->level)
+			continue;
+
+		for (j = 0; j < entries[i].qwords; j++) {
+			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
+
+			xe_pt_destroy(oldpte, xe_vma_vm(vma)->flags, deferred);
+		}
+	}
+}
+
+static void xe_pt_abort_bind(struct xe_vma *vma,
+			     struct xe_vm_pgtable_update *entries,
+			     u32 num_entries, bool rebind)
+{
+	int i, j;
+
+	xe_pt_commit_locks_assert(vma);
+
+	for (i = num_entries - 1; i >= 0; --i) {
+		struct xe_pt *pt = entries[i].pt;
+		struct xe_pt_dir *pt_dir;
+
+		if (!rebind)
+			pt->num_live -= entries[i].qwords;
+
+		if (!pt->level)
+			continue;
+
+		pt_dir = as_xe_pt_dir(pt);
+		for (j = 0; j < entries[i].qwords; j++) {
+			u32 j_ = j + entries[i].ofs;
+			struct xe_pt *newpte = xe_pt_entry(pt_dir, j_);
+			struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
+
+			pt_dir->dir.entries[j_] = oldpte ? &oldpte->base : 0;
+			xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
+		}
+	}
+}
+
+static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
+				      struct xe_vm_pgtable_update *entries,
+				      u32 num_entries, bool rebind)
 {
 	u32 i, j;
 
@@ -847,12 +906,13 @@ static void xe_pt_commit_bind(struct xe_vma *vma,
 		for (j = 0; j < entries[i].qwords; j++) {
 			u32 j_ = j + entries[i].ofs;
 			struct xe_pt *newpte = entries[i].pt_entries[j].pt;
+			struct xe_pt *oldpte = NULL;
 
 			if (xe_pt_entry(pt_dir, j_))
-				xe_pt_destroy(xe_pt_entry(pt_dir, j_),
-					      xe_vma_vm(vma)->flags, deferred);
+				oldpte = xe_pt_entry(pt_dir, j_);
 
 			pt_dir->dir.entries[j_] = &newpte->base;
+			entries[i].pt_entries[j].pt = oldpte;
 		}
 	}
 }
@@ -876,8 +936,6 @@ xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
 	err = xe_pt_stage_bind(tile, vma, entries, num_entries);
 	if (!err)
 		xe_tile_assert(tile, *num_entries);
-	else /* abort! */
-		xe_pt_abort_bind(vma, entries, *num_entries);
 
 	return err;
 }
@@ -1366,7 +1424,7 @@ xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
 				     &end_offset))
 		return 0;
 
-	(void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false);
+	(void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
 	xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
 
 	return 0;
@@ -1434,32 +1492,58 @@ xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
 		memset64(ptr, empty, num_qwords);
 }
 
+static void xe_pt_abort_unbind(struct xe_vma *vma,
+			       struct xe_vm_pgtable_update *entries,
+			       u32 num_entries)
+{
+	int j, i;
+
+	xe_pt_commit_locks_assert(vma);
+
+	for (j = num_entries - 1; j >= 0; --j) {
+		struct xe_vm_pgtable_update *entry = &entries[j];
+		struct xe_pt *pt = entry->pt;
+		struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
+
+		pt->num_live += entry->qwords;
+
+		if (!pt->level)
+			continue;
+
+		for (i = entry->ofs; i < entry->ofs + entry->qwords; i++)
+			pt_dir->dir.entries[i] =
+				entries[j].pt_entries[i - entry->ofs].pt ?
+				&entries[j].pt_entries[i - entry->ofs].pt->base : 0;
+	}
+}
+
 static void
-xe_pt_commit_unbind(struct xe_vma *vma,
-		    struct xe_vm_pgtable_update *entries, u32 num_entries,
-		    struct llist_head *deferred)
+xe_pt_commit_prepare_unbind(struct xe_vma *vma,
+			    struct xe_vm_pgtable_update *entries,
+			    u32 num_entries)
 {
-	u32 j;
+	int j, i;
 
 	xe_pt_commit_locks_assert(vma);
 
 	for (j = 0; j < num_entries; ++j) {
 		struct xe_vm_pgtable_update *entry = &entries[j];
 		struct xe_pt *pt = entry->pt;
+		struct xe_pt_dir *pt_dir;
 
 		pt->num_live -= entry->qwords;
-		if (pt->level) {
-			struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
-			u32 i;
+		if (!pt->level)
+			continue;
 
-			for (i = entry->ofs; i < entry->ofs + entry->qwords;
-			     i++) {
-				if (xe_pt_entry(pt_dir, i))
-					xe_pt_destroy(xe_pt_entry(pt_dir, i),
-						      xe_vma_vm(vma)->flags, deferred);
+		pt_dir = as_xe_pt_dir(pt);
+		for (i = entry->ofs; i < entry->ofs + entry->qwords; i++) {
+			if (xe_pt_entry(pt_dir, i))
+				entries[j].pt_entries[i - entry->ofs].pt =
+					xe_pt_entry(pt_dir, i);
+			else
+				entries[j].pt_entries[i - entry->ofs].pt = NULL;
 
-				pt_dir->dir.entries[i] = NULL;
-			}
+			pt_dir->dir.entries[i] = NULL;
 		}
 	}
 }
@@ -1496,7 +1580,6 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 {
 	u32 current_op = pt_update_ops->current_op;
 	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
-	struct llist_head *deferred = &pt_update_ops->deferred;
 	int err;
 
 	xe_bo_assert_held(xe_vma_bo(vma));
@@ -1505,6 +1588,7 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 	       "Preparing bind, with range [%llx...%llx)\n",
 	       xe_vma_start(vma), xe_vma_end(vma) - 1);
 
+	pt_op->vma = NULL;
 	pt_op->bind = true;
 	pt_op->rebind = BIT(tile->id) & vma->tile_present;
 
@@ -1538,9 +1622,11 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
 			(!pt_op->rebind && vm->scratch_pt[tile->id] &&
 			 xe_vm_in_preempt_fence_mode(vm));
 
-		/* FIXME: Don't commit right away */
-		xe_pt_commit_bind(vma, pt_op->entries, pt_op->num_entries,
-				  pt_op->rebind, deferred);
+		pt_op->vma = vma;
+		xe_pt_commit_prepare_bind(vma, pt_op->entries,
+					  pt_op->num_entries, pt_op->rebind);
+	} else {
+		xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
 	}
 
 	return err;
@@ -1552,7 +1638,6 @@ static int unbind_op_prepare(struct xe_tile *tile,
 {
 	u32 current_op = pt_update_ops->current_op;
 	struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op];
-	struct llist_head *deferred = &pt_update_ops->deferred;
 
 	xe_bo_assert_held(xe_vma_bo(vma));
 
@@ -1560,6 +1645,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 	       "Preparing unbind, with range [%llx...%llx)\n",
 	       xe_vma_start(vma), xe_vma_end(vma) - 1);
 
+	pt_op->vma = vma;
 	pt_op->bind = false;
 	pt_op->rebind = false;
 
@@ -1570,9 +1656,7 @@ static int unbind_op_prepare(struct xe_tile *tile,
 	pt_update_ops->needs_userptr_lock |= xe_vma_is_userptr(vma);
 	pt_update_ops->needs_invalidation = true;
 
-	/* FIXME: Don't commit right away */
-	xe_pt_commit_unbind(vma, pt_op->entries, pt_op->num_entries,
-			    deferred);
+	xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
 
 	return 0;
 }
@@ -1782,7 +1866,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	struct invalidation_fence *ifence = NULL;
 	struct xe_range_fence *rfence;
 	struct xe_vma_op *op;
-	int err = 0;
+	int err = 0, i;
 	struct xe_migrate_pt_update update = {
 		.ops = pt_update_ops->needs_userptr_lock ?
 			&userptr_migrate_ops :
@@ -1796,8 +1880,10 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 
 	if (pt_update_ops->needs_invalidation) {
 		ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
-		if (!ifence)
-			return ERR_PTR(-ENOMEM);
+		if (!ifence) {
+			err = -ENOMEM;
+			goto kill_vm_tile1;
+		}
 	}
 
 	rfence = kzalloc(sizeof(*rfence), GFP_KERNEL);
@@ -1806,10 +1892,19 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 		goto free_ifence;
 	}
 
+	/* FIXME: Point of no return - VM killed if failure after this */
+	for (i = 0; i < pt_update_ops->num_ops; ++i) {
+		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
+
+		xe_pt_commit(pt_op->vma, pt_op->entries,
+			     pt_op->num_entries, &pt_update_ops->deferred);
+		pt_op->vma = NULL;	/* skip in xe_pt_update_ops_abort */
+	}
+
 	fence = xe_migrate_update_pgtables(tile->migrate, &update);
 	if (IS_ERR(fence)) {
 		err = PTR_ERR(fence);
-		goto free_rfence;
+		goto kill_vm_tile0;
 	}
 
 	err = xe_range_fence_insert(&vm->rftree[tile->id], rfence,
@@ -1847,10 +1942,15 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	if (pt_update_ops->needs_userptr_lock)
 		up_read(&vm->userptr.notifier_lock);
 	dma_fence_put(fence);
-free_rfence:
+kill_vm_tile0:
+	if (!tile->id)
+		xe_vm_kill(vops->vm, false);
 	kfree(rfence);
 free_ifence:
 	kfree(ifence);
+kill_vm_tile1:
+	if (tile->id)
+		xe_vm_kill(vops->vm, false);
 
 	return ERR_PTR(err);
 }
@@ -1871,12 +1971,10 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
-	/* FIXME: Not 100% correct */
 	for (i = 0; i < pt_update_ops->num_ops; ++i) {
 		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
 
-		if (pt_op->bind)
-			xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
 	}
 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
 }
@@ -1890,10 +1988,28 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
  */
 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
 {
+	struct xe_vm_pgtable_update_ops *pt_update_ops =
+		&vops->pt_update_ops[tile->id];
+	int i;
+
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
-	/* FIXME: Just kill VM for now + cleanup PTs */
+	for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
+		struct xe_vm_pgtable_update_op *pt_op =
+			&pt_update_ops->ops[i];
+
+		if (!pt_op->vma || i >= pt_update_ops->current_op)
+			continue;
+
+		if (pt_op->bind)
+			xe_pt_abort_bind(pt_op->vma, pt_op->entries,
+					 pt_op->num_entries,
+					 pt_op->rebind);
+		else
+			xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
+					   pt_op->num_entries);
+	}
+
 	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
-	xe_vm_kill(vops->vm, false);
 }
drivers/gpu/drm/xe/xe_pt_types.h +2 -0
index 16252f1be055..384cc04de719 100644
@@ -78,6 +78,8 @@ struct xe_vm_pgtable_update {
 struct xe_vm_pgtable_update_op {
 	/** @entries: entries to update for this operation */
 	struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1];
+	/** @vma: VMA for operation, operation not valid if NULL */
+	struct xe_vma *vma;
 	/** @num_entries: number of entries for this update operation */
 	u32 num_entries;
 	/** @bind: is a bind */
Signature · 1 line
2.34.1
author maintainer
#17 lore raw

PATCH v3 17/22 drm/xe: Update VM trace events

In reply to Matthew Brost

The trace events have changed moving to a single job per VM bind IOCTL, update the trace events align with old behavior as much as possible.

---
 drivers/gpu/drm/xe/xe_trace.h | 10 ++++-----
 drivers/gpu/drm/xe/xe_vm.c    | 42 +++++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 7 deletions(-)
drivers/gpu/drm/xe/xe_trace.h +5 -5
index e4e7262191ad..64f5f5680295 100644
@@ -400,11 +400,6 @@ DEFINE_EVENT(xe_vma, xe_vma_acc,
 	     TP_ARGS(vma)
 );
 
-DEFINE_EVENT(xe_vma, xe_vma_fail,
-	     TP_PROTO(struct xe_vma *vma),
-	     TP_ARGS(vma)
-);
-
 DEFINE_EVENT(xe_vma, xe_vma_bind,
 	     TP_PROTO(struct xe_vma *vma),
 	     TP_ARGS(vma)
@@ -518,6 +513,11 @@ DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_exit,
 	     TP_ARGS(vm)
 );
 
+DEFINE_EVENT(xe_vm, xe_vm_ops_fail,
+	     TP_PROTO(struct xe_vm *vm),
+	     TP_ARGS(vm)
+);
+
 /* GuC */
 DECLARE_EVENT_CLASS(xe_guc_ct_flow_control,
 		    TP_PROTO(u32 _head, u32 _tail, u32 size, u32 space, u32 len),
drivers/gpu/drm/xe/xe_vm.c +40 -2

66 lines not on this page · raw message

Signature · 1 line
2.34.1
author maintainer
#18 lore raw

PATCH v3 18/22 drm/xe: Update clear / populate arguments

In reply to Matthew Brost

This will help implement CPU binds in run_job() as 'struct xe_migrate_pt_update' is not available at the time of run_job().

---
 drivers/gpu/drm/xe/xe_migrate.c |  9 +++++----
 drivers/gpu/drm/xe/xe_migrate.h | 12 +++++-------
 drivers/gpu/drm/xe/xe_pt.c      | 12 +++++-------
 3 files changed, 15 insertions(+), 18 deletions(-)
drivers/gpu/drm/xe/xe_migrate.c +5 -4
index afad09591fbf..9ea7a6a221b5 100644
@@ -1110,6 +1110,7 @@ static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
 			  struct xe_migrate_pt_update *pt_update)
 {
 	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
+	struct xe_vm *vm = pt_update->vops->vm;
 	u32 chunk;
 	u32 ofs = update->ofs, size = update->qwords;
 
@@ -1141,10 +1142,10 @@ static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,
 		bb->cs[bb->len++] = lower_32_bits(addr);
 		bb->cs[bb->len++] = upper_32_bits(addr);
 		if (pt_op->bind)
-			ops->populate(pt_update, tile, NULL, bb->cs + bb->len,
+			ops->populate(tile, NULL, bb->cs + bb->len,
 				      ofs, chunk, update);
 		else
-			ops->clear(pt_update, tile, NULL, bb->cs + bb->len,
+			ops->clear(vm, tile, NULL, bb->cs + bb->len,
 				   ofs, chunk, update);
 
 		bb->len += chunk * 2;
@@ -1201,12 +1202,12 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
 				&pt_op->entries[i];
 
 			if (pt_op->bind)
-				ops->populate(pt_update, m->tile,
+				ops->populate(m->tile,
 					      &update->pt_bo->vmap, NULL,
 					      update->ofs, update->qwords,
 					      update);
 			else
-				ops->clear(pt_update, m->tile,
+				ops->clear(vm, m->tile,
 					   &update->pt_bo->vmap, NULL,
 					   update->ofs, update->qwords, update);
 		}
drivers/gpu/drm/xe/xe_migrate.h +5 -7
index bd8eba1d3552..18f5a8e40b5c 100644
@@ -31,7 +31,6 @@ struct xe_vma;
 struct xe_migrate_pt_update_ops {
 	/**
 	 * @populate: Populate a command buffer or page-table with ptes.
-	 * @pt_update: Embeddable callback argument.
 	 * @tile: The tile for the current operation.
 	 * @map: struct iosys_map into the memory to be populated.
 	 * @pos: If @map is NULL, map into the memory to be populated.
@@ -43,13 +42,12 @@ struct xe_migrate_pt_update_ops {
 	 * page-table system to populate command buffers or shared
 	 * page-tables with PTEs.
 	 */
-	void (*populate)(struct xe_migrate_pt_update *pt_update,
-			 struct xe_tile *tile, struct iosys_map *map,
+	void (*populate)(struct xe_tile *tile, struct iosys_map *map,
 			 void *pos, u32 ofs, u32 num_qwords,
 			 const struct xe_vm_pgtable_update *update);
 	/**
 	 * @clear: Clear a command buffer or page-table with ptes.
-	 * @pt_update: Embeddable callback argument.
+	 * @vm: VM being updated
 	 * @tile: The tile for the current operation.
 	 * @map: struct iosys_map into the memory to be populated.
 	 * @pos: If @map is NULL, map into the memory to be populated.
@@ -61,9 +59,9 @@ struct xe_migrate_pt_update_ops {
 	 * page-table system to populate command buffers or shared
 	 * page-tables with PTEs.
 	 */
-	void (*clear)(struct xe_migrate_pt_update *pt_update,
-		      struct xe_tile *tile, struct iosys_map *map,
-		      void *pos, u32 ofs, u32 num_qwords,
+	void (*clear)(struct xe_vm *vm, struct xe_tile *tile,
+		      struct iosys_map *map, void *pos, u32 ofs,
+		      u32 num_qwords,
 		      const struct xe_vm_pgtable_update *update);
 
 	/**
drivers/gpu/drm/xe/xe_pt.c +5 -7
index 8160a295dd84..4758ae8a5459 100644
@@ -778,9 +778,8 @@ bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
 }
 
 static void
-xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile,
-		       struct iosys_map *map, void *data,
-		       u32 qword_ofs, u32 num_qwords,
+xe_vm_populate_pgtable(struct xe_tile *tile, struct iosys_map *map,
+		       void *data, u32 qword_ofs, u32 num_qwords,
 		       const struct xe_vm_pgtable_update *update)
 {
 	struct xe_pt_entry *ptes = update->pt_entries;
@@ -1472,12 +1471,11 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma,
 }
 
 static void
-xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update,
-				  struct xe_tile *tile, struct iosys_map *map,
-				  void *ptr, u32 qword_ofs, u32 num_qwords,
+xe_migrate_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile,
+				  struct iosys_map *map, void *ptr,
+				  u32 qword_ofs, u32 num_qwords,
 				  const struct xe_vm_pgtable_update *update)
 {
-	struct xe_vm *vm = pt_update->vops->vm;
 	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
 	int i;
 
Signature · 1 line
2.34.1
author maintainer
#19 lore raw

PATCH v3 19/22 drm/xe: Add __xe_migrate_update_pgtables_cpu helper

In reply to Matthew Brost

This will help implement CPU binds as the submision backend can call this helper when a bind jobs dependencies are resolved.

---
 drivers/gpu/drm/xe/xe_migrate.c | 54 +++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 23 deletions(-)
drivers/gpu/drm/xe/xe_migrate.c +31 -23
index 9ea7a6a221b5..780e6eca2c40 100644
@@ -1169,6 +1169,34 @@ struct migrate_test_params {
 	container_of(_priv, struct migrate_test_params, base)
 #endif
 
+static void
+__xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,
+				 const struct xe_migrate_pt_update_ops *ops,
+				 struct xe_vm_pgtable_update_op *pt_op,
+				 int num_ops)
+{
+	u32 j, i;
+
+	for (j = 0; j < num_ops; ++j, ++pt_op) {
+		for (i = 0; i < pt_op->num_entries; i++) {
+			const struct xe_vm_pgtable_update *update =
+				&pt_op->entries[i];
+
+			if (pt_op->bind)
+				ops->populate(tile, &update->pt_bo->vmap,
+					      NULL, update->ofs, update->qwords,
+					      update);
+			else
+				ops->clear(vm, tile, &update->pt_bo->vmap,
+					   NULL, update->ofs, update->qwords,
+					   update);
+		}
+	}
+
+	trace_xe_vm_cpu_bind(vm);
+	xe_device_wmb(vm->xe);
+}
+
 static struct dma_fence *
 xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
 			       struct xe_migrate_pt_update *pt_update)
@@ -1181,7 +1209,6 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
 	struct xe_vm_pgtable_update_ops *pt_update_ops =
 		&pt_update->vops->pt_update_ops[pt_update->tile_id];
 	int err;
-	u32 j, i;
 
 	if (XE_TEST_ONLY(test && test->force_gpu))
 		return ERR_PTR(-ETIME);
@@ -1193,28 +1220,9 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
 			return ERR_PTR(err);
 	}
 
-	for (j = 0; j < pt_update_ops->num_ops; ++j) {
-		const struct xe_vm_pgtable_update_op *pt_op =
-			&pt_update_ops->ops[j];
-
-		for (i = 0; i < pt_op->num_entries; i++) {
-			const struct xe_vm_pgtable_update *update =
-				&pt_op->entries[i];
-
-			if (pt_op->bind)
-				ops->populate(m->tile,
-					      &update->pt_bo->vmap, NULL,
-					      update->ofs, update->qwords,
-					      update);
-			else
-				ops->clear(vm, m->tile,
-					   &update->pt_bo->vmap, NULL,
-					   update->ofs, update->qwords, update);
-		}
-	}
-
-	trace_xe_vm_cpu_bind(vm);
-	xe_device_wmb(vm->xe);
+	__xe_migrate_update_pgtables_cpu(vm, m->tile, ops,
+					 pt_update_ops->ops,
+					 pt_update_ops->num_ops);
 
 	return dma_fence_get_stub();
 }
Signature · 1 line
2.34.1
author maintainer
#20 lore raw

PATCH v3 20/22 drm/xe: CPU binds for jobs

In reply to Matthew Brost

No reason to use the GPU for binds. In run_job use the CPU to do binds once the bind job dependencies are resolved.

---
 drivers/gpu/drm/xe/xe_bo.c              |   7 +-
 drivers/gpu/drm/xe/xe_bo.h              |   4 +-
 drivers/gpu/drm/xe/xe_bo_types.h        |   2 -
 drivers/gpu/drm/xe/xe_device.c          |  35 +++++
 drivers/gpu/drm/xe/xe_device.h          |   2 +
 drivers/gpu/drm/xe/xe_device_types.h    |   4 +
 drivers/gpu/drm/xe/xe_gt_pagefault.c    |   5 +-
 drivers/gpu/drm/xe/xe_guc_submit.c      |  47 +++++-
 drivers/gpu/drm/xe/xe_migrate.c         | 198 +++---------------------
 drivers/gpu/drm/xe/xe_migrate.h         |   6 +
 drivers/gpu/drm/xe/xe_pt.c              |  36 +++--
 drivers/gpu/drm/xe/xe_pt.h              |   1 +
 drivers/gpu/drm/xe/xe_pt_types.h        |   5 +
 drivers/gpu/drm/xe/xe_sched_job.c       |  24 ++-
 drivers/gpu/drm/xe/xe_sched_job_types.h |  31 +++-
 drivers/gpu/drm/xe/xe_vm.c              | 104 ++++++-------
 drivers/gpu/drm/xe/xe_vm.h              |   5 +-
 17 files changed, 247 insertions(+), 269 deletions(-)
drivers/gpu/drm/xe/xe_bo.c +3 -4
index 686d716c5581..3f327c123bbc 100644
@@ -2237,16 +2237,16 @@ void __xe_bo_release_dummy(struct kref *kref)
 
 /**
  * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().
+ * @xe: Xe device
  * @deferred: The lockless list used for the call to xe_bo_put_deferred().
  *
  * Puts all bos whose put was deferred by xe_bo_put_deferred().
  * The @deferred list can be either an onstack local list or a global
  * shared list used by a workqueue.
  */
-void xe_bo_put_commit(struct llist_head *deferred)
+void xe_bo_put_commit(struct xe_device *xe, struct llist_head *deferred)
 {
 	struct llist_node *freed;
-	struct xe_bo *bo, *next;
 
 	if (!deferred)
 		return;
@@ -2255,8 +2255,7 @@ void xe_bo_put_commit(struct llist_head *deferred)
 	if (!freed)
 		return;
 
-	llist_for_each_entry_safe(bo, next, freed, freed)
-		drm_gem_object_free(&bo->ttm.base.refcount);
+	xe_device_put_deferred(xe, freed);
 }
 
 /**
drivers/gpu/drm/xe/xe_bo.h +2 -2
index db4b2db6b073..2a4bfa4fe6c4 100644
@@ -10,7 +10,6 @@
 
 #include "xe_bo_types.h"
 #include "xe_macros.h"
-#include "xe_vm_types.h"
 #include "xe_vm.h"
 
 /**
@@ -307,10 +306,11 @@ xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)
 	if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy))
 		return false;
 
+	xe_vm_get(bo->vm);
 	return llist_add(&bo->freed, deferred);
 }
 
-void xe_bo_put_commit(struct llist_head *deferred);
+void xe_bo_put_commit(struct xe_device *xe, struct llist_head *deferred);
 
 struct sg_table *xe_bo_sg(struct xe_bo *bo);
 
drivers/gpu/drm/xe/xe_bo_types.h +0 -2
index 1ff5b5a68adc..14ef13b7b421 100644
@@ -77,8 +77,6 @@ struct xe_bo {
 	} props;
 	/** @freed: List node for delayed put. */
 	struct llist_node freed;
-	/** @update_index: Update index if PT BO */
-	int update_index;
 	/** @created: Whether the bo has passed initial creation */
 	bool created;
 
drivers/gpu/drm/xe/xe_device.c +35 -0
index 5b84d7305520..2998c679f3bd 100644
@@ -198,6 +198,9 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)
 {
 	struct xe_device *xe = to_xe_device(dev);
 
+	flush_work(&xe->mem.deferred_work);
+	xe_assert(xe, !llist_del_all(&xe->mem.deferred));
+
 	if (xe->ordered_wq)
 		destroy_workqueue(xe->ordered_wq);
 
@@ -207,6 +210,35 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)
 	ttm_device_fini(&xe->ttm);
 }
 
+void xe_device_put_deferred(struct xe_device *xe, struct llist_node *deferred)
+{
+	struct xe_bo *bo, *next;
+
+	llist_for_each_entry_safe(bo, next, deferred, freed) {
+		init_llist_node(&bo->freed);
+		llist_add(&bo->freed, &xe->mem.deferred);
+	}
+	queue_work(system_wq, &xe->mem.deferred_work);
+}
+
+static void deferred_work(struct work_struct *w)
+{
+	struct xe_device *xe = container_of(w, struct xe_device,
+					    mem.deferred_work);
+	struct llist_node *freed = llist_del_all(&xe->mem.deferred);
+	struct xe_bo *bo, *next;
+
+	if (!freed)
+		return;
+
+	llist_for_each_entry_safe(bo, next, freed, freed) {
+		struct xe_vm *vm = bo->vm;
+
+		drm_gem_object_free(&bo->ttm.base.refcount);
+		xe_vm_put(vm);
+	}
+}
+
 struct xe_device *xe_device_create(struct pci_dev *pdev,
 				   const struct pci_device_id *ent)
 {
@@ -274,6 +306,9 @@ struct xe_device *xe_device_create(struct pci_dev *pdev,
 		goto err;
 	}
 
+	init_llist_head(&xe->mem.deferred);
+	INIT_WORK(&xe->mem.deferred_work, deferred_work);
+
 	err = xe_display_create(xe);
 	if (WARN_ON(err))
 		goto err;
drivers/gpu/drm/xe/xe_device.h +2 -0
index 462f59e902b1..8991d6a18368 100644
@@ -180,4 +180,6 @@ void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p);
 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address);
 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);
 
+void xe_device_put_deferred(struct xe_device *xe, struct llist_node *deferred);
+
 #endif
drivers/gpu/drm/xe/xe_device_types.h +4 -0
index eb2b806a1d23..3018f1d79177 100644
@@ -315,6 +315,10 @@ struct xe_device {
 		struct xe_mem_region vram;
 		/** @mem.sys_mgr: system TTM manager */
 		struct ttm_resource_manager sys_mgr;
+		/** @deferred: deferred list to destroy PT entries */
+		struct llist_head deferred;
+		/** @deferred_work: worker to destroy PT entries */
+		struct work_struct deferred_work;
 	} mem;
 
 	/** @sriov: device level virtualization data */
drivers/gpu/drm/xe/xe_gt_pagefault.c +4 -1
index 27dbbdc9127e..2beba426517e 100644
@@ -206,10 +206,13 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
 
 	/* Bind VMA only to the GT that has faulted */
 	trace_xe_vma_pf_bind(vma);
-	xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+	ret = xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+	if (ret)
+		goto unlock_dma_resv;
 	vm->dummy_ops.vops.pt_update_ops[tile->id].q =
 		xe_tile_migrate_exec_queue(tile);
 	fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
+	xe_vma_ops_free(&vm->dummy_ops.vops);
 	if (IS_ERR(fence)) {
 		ret = PTR_ERR(fence);
 		goto unlock_dma_resv;
drivers/gpu/drm/xe/xe_guc_submit.c +40 -7

74 lines not on this page · raw message

drivers/gpu/drm/xe/xe_migrate.c +19 -179

241 lines not on this page · raw message

drivers/gpu/drm/xe/xe_migrate.h +6 -0
index 18f5a8e40b5c..f80c94bb8f4c 100644
@@ -22,6 +22,7 @@ struct xe_pt;
 struct xe_tile;
 struct xe_vm;
 struct xe_vm_pgtable_update;
+struct xe_vm_pgtable_update_op;
 struct xe_vma;
 
 /**
@@ -106,6 +107,11 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m,
 
 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
 
+void __xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,
+				      const struct xe_migrate_pt_update_ops *ops,
+				      struct xe_vm_pgtable_update_op *pt_op,
+				      int num_ops);
+
 struct dma_fence *
 xe_migrate_update_pgtables(struct xe_migrate *m,
 			   struct xe_migrate_pt_update *pt_update);
drivers/gpu/drm/xe/xe_pt.c +25 -11
index 4758ae8a5459..26a00af6c4a6 100644
@@ -315,7 +315,7 @@ xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
 	entry->pt = parent;
 	entry->flags = 0;
 	entry->qwords = 0;
-	entry->pt_bo->update_index = -1;
+	entry->level = parent->level;
 
 	if (alloc_entries) {
 		entry->pt_entries = kmalloc_array(XE_PDES,
@@ -1476,7 +1476,7 @@ xe_migrate_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile,
 				  u32 qword_ofs, u32 num_qwords,
 				  const struct xe_vm_pgtable_update *update)
 {
-	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
+	u64 empty = __xe_pt_empty_pte(tile, vm, update->level);
 	int i;
 
 	if (map && map->is_iomem)
@@ -1890,7 +1890,7 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 		goto free_ifence;
 	}
 
-	/* FIXME: Point of no return - VM killed if failure after this */
+	/* Point of no return - VM killed if failure after this */
 	for (i = 0; i < pt_update_ops->num_ops; ++i) {
 		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
 
@@ -1953,6 +1953,21 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	return ERR_PTR(err);
 }
 
+/**
+ * xe_pt_update_ops_free() - Free PT update operations
+ * @pt_op: Array of PT update operations
+ * @num_ops: Number of PT update operations
+ *
+ * Free PT update operations
+ */
+void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op, u32 num_ops)
+{
+	u32 i;
+
+	for (i = 0; i < num_ops; ++i, ++pt_op)
+		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+}
+
 /**
  * xe_pt_update_ops_fini() - Finish PT update operations
  * @tile: Tile of PT update operations
@@ -1964,17 +1979,16 @@ void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops)
 {
 	struct xe_vm_pgtable_update_ops *pt_update_ops =
 		&vops->pt_update_ops[tile->id];
-	int i;
 
 	lockdep_assert_held(&vops->vm->lock);
 	xe_vm_assert_held(vops->vm);
 
-	for (i = 0; i < pt_update_ops->num_ops; ++i) {
-		struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];
-
-		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
-	}
-	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+	xe_bo_put_commit(tile_to_xe(tile), &pt_update_ops->deferred);
+	if (!pt_update_ops->skip_free)
+		xe_pt_update_ops_free(pt_update_ops->ops,
+				      pt_update_ops->num_ops);
+	else
+		pt_update_ops->ops = NULL;
 }
 
 /**
@@ -2009,5 +2023,5 @@ void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops)
 					   pt_op->num_entries);
 	}
 
-	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+	xe_pt_update_ops_fini(tile, vops);
 }
drivers/gpu/drm/xe/xe_pt.h +1 -0
index 9ab386431cad..989c9b190fa0 100644
@@ -40,6 +40,7 @@ struct dma_fence *xe_pt_update_ops_run(struct xe_tile *tile,
 				       struct xe_vma_ops *vops);
 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op, u32 num_ops);
 
 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);
 
drivers/gpu/drm/xe/xe_pt_types.h +5 -0
index 384cc04de719..cfd0d35408a5 100644
@@ -70,6 +70,9 @@ struct xe_vm_pgtable_update {
 	/** @pt_entries: Newly added pagetable entries */
 	struct xe_pt_entry *pt_entries;
 
+	/** @level: level of update */
+	unsigned int level;
+
 	/** @flags: Target flags */
 	u32 flags;
 };
@@ -120,6 +123,8 @@ struct xe_vm_pgtable_update_ops {
 	 * slots are idle.
 	 */
 	bool wait_vm_kernel;
+	/** @skip_free: Free @ops in submission backend rather than in IOCTL */
+	bool skip_free;
 };
 
 #endif
drivers/gpu/drm/xe/xe_sched_job.c +17 -7
index 8151ddafb940..5930f03a81ad 100644
@@ -34,8 +34,10 @@ int __init xe_sched_job_module_init(void)
 	xe_sched_job_parallel_slab =
 		kmem_cache_create("xe_sched_job_parallel",
 				  sizeof(struct xe_sched_job) +
+				  max_t(size_t,
 				  sizeof(u64) *
-				  XE_HW_ENGINE_MAX_INSTANCE, 0,
+				  XE_HW_ENGINE_MAX_INSTANCE,
+				  sizeof(struct pt_update_args)), 0,
 				  SLAB_HWCACHE_ALIGN, NULL);
 	if (!xe_sched_job_parallel_slab) {
 		kmem_cache_destroy(xe_sched_job_slab);
@@ -108,7 +110,13 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
 	if (err)
 		goto err_free;
 
-	if (!xe_exec_queue_is_parallel(q)) {
+	if (!batch_addr) {
+		job->fence = dma_fence_allocate_private_stub(ktime_get());
+		if (!job->fence) {
+			err = -ENOMEM;
+			goto err_sched_job;
+		}
+	} else if (!xe_exec_queue_is_parallel(q)) {
 		job->fence = xe_lrc_create_seqno_fence(q->lrc);
 		if (IS_ERR(job->fence)) {
 			err = PTR_ERR(job->fence);
@@ -148,12 +156,14 @@ struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q,
 		job->fence = &cf->base;
 	}
 
-	width = q->width;
-	if (is_migration)
-		width = 2;
+	if (batch_addr) {
+		width = q->width;
+		if (is_migration)
+			width = 2;
 
-	for (i = 0; i < width; ++i)
-		job->batch_addr[i] = batch_addr[i];
+		for (i = 0; i < width; ++i)
+			job->batch_addr[i] = batch_addr[i];
+	}
 
 	/* All other jobs require a VM to be open which has a ref */
 	if (unlikely(q->flags & EXEC_QUEUE_FLAG_KERNEL))
drivers/gpu/drm/xe/xe_sched_job_types.h +29 -2
index b1d83da50a53..623d6dcc8ae5 100644
@@ -11,6 +11,28 @@
 #include <drm/gpu_scheduler.h>
 
 struct xe_exec_queue;
+struct xe_migrate_pt_update_ops;
+struct xe_tile;
+struct xe_vm;
+struct xe_vm_pgtable_update_op;
+
+/**
+ * struct pt_update_args - PT update arguments
+ */
+struct pt_update_args {
+	/** @vm: VM */
+	struct xe_vm *vm;
+	/** @tile: Tile */
+	struct xe_tile *tile;
+	/** @ops: Migrate PT update ops */
+	const struct xe_migrate_pt_update_ops *ops;
+	/** @pt_op: PT update ops */
+	struct xe_vm_pgtable_update_op *pt_op;
+	/** @deferred: deferred list to destroy PT entries */
+	struct llist_head deferred;
+	/** @num_ops: number of PT update ops */
+	int num_ops;
+};
 
 /**
  * struct xe_sched_job - XE schedule job (batch buffer tracking)
@@ -27,6 +49,7 @@ struct xe_sched_job {
 	 * can safely reference fence, fence cannot safely reference job.
 	 */
 #define JOB_FLAG_SUBMIT		DMA_FENCE_FLAG_USER_BITS
+#define JOB_FLAG_PT		(DMA_FENCE_FLAG_USER_BITS << 1)
 	struct dma_fence *fence;
 	/** @user_fence: write back value when BB is complete */
 	struct {
@@ -39,8 +62,12 @@ struct xe_sched_job {
 	} user_fence;
 	/** @migrate_flush_flags: Additional flush flags for migration jobs */
 	u32 migrate_flush_flags;
-	/** @batch_addr: batch buffer address of job */
-	u64 batch_addr[];
+	union {
+		/** @batch_addr: batch buffer address of job */
+		u64 batch_addr[0];
+		/** @pt_update: PT update arguments */
+		struct pt_update_args pt_update[0];
+	};
 };
 
 struct xe_sched_job_snapshot {
drivers/gpu/drm/xe/xe_vm.c +52 -52

166 lines not on this page · raw message

drivers/gpu/drm/xe/xe_vm.h +3 -2
index b8134804fce9..58e7490f7401 100644
@@ -262,8 +262,9 @@ static inline struct dma_resv *xe_vm_resv(struct xe_vm *vm)
  */
 #define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))
 
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
-				 u8 tile_mask);
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,
+				u8 tile_mask);
+void xe_vma_ops_free(struct xe_vma_ops *vops);
 struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);
 
 void xe_vm_kill(struct xe_vm *vm, bool unlocked);
Signature · 1 line
2.34.1
#21 lore raw

Re: [PATCH v3 20/22] drm/xe: CPU binds for jobs

Hi, Matthew,

Quoted 3 lines · Matthew Brost

No reason to use the GPU for binds. In run_job use the CPU to do binds
once the bind job dependencies are resolved.

I think there are two good reasons to keep the GPU job functionality.

One which we've discussed previously is for small aperture situations. I'm not sure we actually will see much of this moving forward, but I don't think in that case we'd want to fill whatever small aperture we have with page-table bos.

The other is if we rely on cpu page-table binding only we're forced to accept the latency bubbles between finishing a migration job, signaling the fence and scheduling the bind job for run. I'm thinking of a situation where you allocate and clear a number of small bos. The ideal situation here would be clearing and binding on the same exec queue. You can then pipeline them all in the exec queue with little or no latency between the jobs.

In any case, I understand that this simplifies the refactoring, but I don't think at least we should *rely* on us doing cpu binding for all future, and we need a better commit message discussing why we, given the above, choose to sacrifice gpu binding.

/Thomas

Quoted 1,115 lines

---
 drivers/gpu/drm/xe/xe_bo.c              |   7 +- drivers/gpu/drm/xe/xe_bo.h              |   4 +- drivers/gpu/drm/xe/xe_bo_types.h        |   2 - drivers/gpu/drm/xe/xe_device.c          |  35 +++++ drivers/gpu/drm/xe/xe_device.h          |   2 + drivers/gpu/drm/xe/xe_device_types.h    |   4 + drivers/gpu/drm/xe/xe_gt_pagefault.c    |   5 +- drivers/gpu/drm/xe/xe_guc_submit.c      |  47 +++++- drivers/gpu/drm/xe/xe_migrate.c         | 198 +++------------------- --
 drivers/gpu/drm/xe/xe_migrate.h         |   6 + drivers/gpu/drm/xe/xe_pt.c              |  36 +++-- drivers/gpu/drm/xe/xe_pt.h              |   1 + drivers/gpu/drm/xe/xe_pt_types.h        |   5 + drivers/gpu/drm/xe/xe_sched_job.c       |  24 ++- drivers/gpu/drm/xe/xe_sched_job_types.h |  31 +++- drivers/gpu/drm/xe/xe_vm.c              | 104 ++++++------- drivers/gpu/drm/xe/xe_vm.h              |   5 +-

 17 files changed, 247 insertions(+), 269 deletions(-)
drivers/gpu/drm/xe/xe_bo.c +0 -0
index 686d716c5581..3f327c123bbc 100644
@@ -2237,16 +2237,16 @@ void __xe_bo_release_dummy(struct kref *kref)


 /**
  * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred().
+ * @xe: Xe device
  * @deferred: The lockless list used for the call to xe_bo_put_deferred().
  *
  * Puts all bos whose put was deferred by xe_bo_put_deferred().
  * The @deferred list can be either an onstack local list or a global
  * shared list used by a workqueue.
  */

drivers/gpu/drm/xe/xe_bo.c +1 -1
-void xe_bo_put_commit(struct llist_head *deferred)
+void xe_bo_put_commit(struct xe_device *xe, struct llist_head

*deferred)
 {
  struct llist_node *freed;

drivers/gpu/drm/xe/xe_bo.c +0 -1
-	struct xe_bo *bo, *next;


  if (!deferred)
  return;

drivers/gpu/drm/xe/xe_bo.c +0 -0
@@ -2255,8 +2255,7 @@ void xe_bo_put_commit(struct llist_head
 *deferred)
  	if (!freed)
  		return;

drivers/gpu/drm/xe/xe_bo.c +1 -2
-	llist_for_each_entry_safe(bo, next, freed, freed)
-		drm_gem_object_free(&bo->ttm.base.refcount);
+	xe_device_put_deferred(xe, freed);

 }

 /**

drivers/gpu/drm/xe/xe_bo.h +0 -0
index db4b2db6b073..2a4bfa4fe6c4 100644
@@ -10,7 +10,6 @@


 #include "xe_bo_types.h"
 #include "xe_macros.h"
-#include "xe_vm_types.h"
 #include "xe_vm.h"

 /**

drivers/gpu/drm/xe/xe_bo.h +0 -0
@@ -307,10 +306,11 @@ xe_bo_put_deferred(struct xe_bo *bo, struct
 llist_head *deferred)
  	if (!kref_put(&bo->ttm.base.refcount,
 __xe_bo_release_dummy))
  		return false;

drivers/gpu/drm/xe/xe_bo.h +1 -0
+	xe_vm_get(bo->vm);

  return llist_add(&bo->freed, deferred);
 }

drivers/gpu/drm/xe/xe_bo.h +1 -1
-void xe_bo_put_commit(struct llist_head *deferred);
+void xe_bo_put_commit(struct xe_device *xe, struct llist_head

*deferred);

 struct sg_table *xe_bo_sg(struct xe_bo *bo);

drivers/gpu/drm/xe/xe_bo_types.h +0 -2
index 1ff5b5a68adc..14ef13b7b421 100644
@@ -77,8 +77,6 @@ struct xe_bo {
  	} props;
  	/** @freed: List node for delayed put. */
  	struct llist_node freed;
-	/** @update_index: Update index if PT BO */
-	int update_index;
  	/** @created: Whether the bo has passed initial creation */
  	bool created;

drivers/gpu/drm/xe/xe_device.c +0 -0
index 5b84d7305520..2998c679f3bd 100644
@@ -198,6 +198,9 @@ static void xe_device_destroy(struct drm_device
 *dev, void *dummy)
  {
  	struct xe_device *xe = to_xe_device(dev);

drivers/gpu/drm/xe/xe_device.c +3 -0
+	flush_work(&xe->mem.deferred_work);
+	xe_assert(xe, !llist_del_all(&xe->mem.deferred));
+

  if (xe->ordered_wq) destroy_workqueue(xe->ordered_wq);

drivers/gpu/drm/xe/xe_device.c +0 -0
@@ -207,6 +210,35 @@ static void xe_device_destroy(struct drm_device
 *dev, void *dummy)
  	ttm_device_fini(&xe->ttm);
  }


+void xe_device_put_deferred(struct xe_device *xe, struct llist_node *deferred)

drivers/gpu/drm/xe/xe_device.c +28 -0
+{
+	struct xe_bo *bo, *next;
+
+	llist_for_each_entry_safe(bo, next, deferred, freed) {
+		init_llist_node(&bo->freed);
+		llist_add(&bo->freed, &xe->mem.deferred);
+	}
+	queue_work(system_wq, &xe->mem.deferred_work);
+}
+
+static void deferred_work(struct work_struct *w)
+{
+	struct xe_device *xe = container_of(w, struct xe_device,
+					    mem.deferred_work);
+	struct llist_node *freed = llist_del_all(&xe->mem.deferred);
+	struct xe_bo *bo, *next;
+
+	if (!freed)
+		return;
+
+	llist_for_each_entry_safe(bo, next, freed, freed) {
+		struct xe_vm *vm = bo->vm;
+
+		drm_gem_object_free(&bo->ttm.base.refcount);
+		xe_vm_put(vm);
+	}
+}
+

 struct xe_device *xe_device_create(struct pci_dev *pdev, const struct pci_device_id *ent) {

drivers/gpu/drm/xe/xe_device.c +0 -0
@@ -274,6 +306,9 @@ struct xe_device *xe_device_create(struct pci_dev
 *pdev,
  		goto err;
  	}

drivers/gpu/drm/xe/xe_device.c +3 -0
+	init_llist_head(&xe->mem.deferred);
+	INIT_WORK(&xe->mem.deferred_work, deferred_work);
+

  err = xe_display_create(xe);
  if (WARN_ON(err))
  goto err;

drivers/gpu/drm/xe/xe_device.h +0 -0
index 462f59e902b1..8991d6a18368 100644
@@ -180,4 +180,6 @@ void xe_device_snapshot_print(struct xe_device
 *xe, struct drm_printer *p);
  u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address);

 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address);

+void xe_device_put_deferred(struct xe_device *xe, struct llist_node *deferred);
+
 #endif

drivers/gpu/drm/xe/xe_device_types.h +4 -0
index eb2b806a1d23..3018f1d79177 100644
@@ -315,6 +315,10 @@ struct xe_device {
  		struct xe_mem_region vram;
  		/** @mem.sys_mgr: system TTM manager */
  		struct ttm_resource_manager sys_mgr;
+		/** @deferred: deferred list to destroy PT entries
 */
+		struct llist_head deferred;
+		/** @deferred_work: worker to destroy PT entries */
+		struct work_struct deferred_work;
  	} mem;


  /** @sriov: device level virtualization data */

drivers/gpu/drm/xe/xe_gt_pagefault.c +0 -0
index 27dbbdc9127e..2beba426517e 100644
@@ -206,10 +206,13 @@ static int handle_pagefault(struct xe_gt *gt,
 struct pagefault *pf)


  /* Bind VMA only to the GT that has faulted */ trace_xe_vma_pf_bind(vma);

drivers/gpu/drm/xe/xe_gt_pagefault.c +3 -1
-	xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+	ret = xe_vm_populate_dummy_rebind(vm, vma, BIT(tile->id));
+	if (ret)
+		goto unlock_dma_resv;

  vm->dummy_ops.vops.pt_update_ops[tile->id].q = xe_tile_migrate_exec_queue(tile); fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);

drivers/gpu/drm/xe/xe_gt_pagefault.c +1 -0
+	xe_vma_ops_free(&vm->dummy_ops.vops);

  if (IS_ERR(fence)) {
  ret = PTR_ERR(fence);
  goto unlock_dma_resv;

drivers/gpu/drm/xe/xe_guc_submit.c +3 -0
index 4744668ef60a..bcf0563b76ea 100644
@@ -17,6 +17,7 @@
  #include "abi/guc_klvs_abi.h"
  #include "regs/xe_lrc_layout.h"
  #include "xe_assert.h"
+#include "xe_bo.h"
  #include "xe_devcoredump.h"
  #include "xe_device.h"
  #include "xe_exec_queue.h"
@@ -33,7 +34,9 @@
  #include "xe_lrc.h"
  #include "xe_macros.h"
  #include "xe_map.h"
+#include "xe_migrate.h"
  #include "xe_mocs.h"
+#include "xe_pt.h"
  #include "xe_ring_ops_types.h"
  #include "xe_sched_job.h"
  #include "xe_trace.h"
@@ -719,6 +722,29 @@ static void submit_exec_queue(struct
 xe_exec_queue *q)
  	}
  }

drivers/gpu/drm/xe/xe_guc_submit.c +6 -0
+static bool is_pt_job(struct xe_sched_job *job)
+{
+	return test_bit(JOB_FLAG_PT, &job->fence->flags);
+}
+
+static void cleanup_pt_job(struct xe_device *xe, struct xe_sched_job

*job)

drivers/gpu/drm/xe/xe_guc_submit.c +8 -0
+{
+	xe_pt_update_ops_free(job->pt_update[0].pt_op,
+			      job->pt_update[0].num_ops);
+	xe_bo_put_commit(xe, &job->pt_update[0].deferred);
+	kfree(job->pt_update[0].pt_op);
+}
+
+static void run_pt_job(struct xe_device *xe, struct xe_sched_job

*job)

drivers/gpu/drm/xe/xe_guc_submit.c +9 -0
+{
+	__xe_migrate_update_pgtables_cpu(job->pt_update[0].vm,
+					 job->pt_update[0].tile,
+					 job->pt_update[0].ops,
+					 job->pt_update[0].pt_op,
+					 job->pt_update[0].num_ops);
+	cleanup_pt_job(xe, job);
+}
+

 static struct dma_fence * guc_exec_queue_run_job(struct drm_sched_job *drm_job) {

drivers/gpu/drm/xe/xe_guc_submit.c +0 -0
@@ -734,15 +760,22 @@ guc_exec_queue_run_job(struct drm_sched_job
 *drm_job)
  	trace_xe_sched_job_run(job);


  if (!exec_queue_killed_or_banned(q) &&
!xe_sched_job_is_error(job)) {

drivers/gpu/drm/xe/xe_guc_submit.c +0 -3
-		if (!exec_queue_registered(q))
-			register_engine(q);
-		if (!lr)	/* LR jobs are emitted in the exec

IOCTL */

drivers/gpu/drm/xe/xe_guc_submit.c +6 -2
-			q->ring_ops->emit_job(job);
-		submit_exec_queue(q);
+		if (is_pt_job(job)) {
+			run_pt_job(xe, job);
+		} else {
+			if (!exec_queue_registered(q))
+				register_engine(q);
+			if (!lr)	/* LR jobs are emitted in

the exec IOCTL */

drivers/gpu/drm/xe/xe_guc_submit.c +5 -0
+				q->ring_ops->emit_job(job);
+			submit_exec_queue(q);
+		}
+	} else if (is_pt_job(job)) {
+		cleanup_pt_job(xe, job);

  }

drivers/gpu/drm/xe/xe_guc_submit.c +3 -2
-	if (lr) {
-		xe_sched_job_set_error(job, -EOPNOTSUPP);
+	if (lr || is_pt_job(job)) {
+		if (lr)
+			xe_sched_job_set_error(job, -EOPNOTSUPP);

  return NULL;
  } else if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence-

Quoted 1 line

flags)) {

  return job->fence;

drivers/gpu/drm/xe/xe_migrate.c +0 -0
index 780e6eca2c40..7d752e424b3b 100644
@@ -1104,56 +1104,6 @@ struct dma_fence *xe_migrate_clear(struct
 xe_migrate *m,
  	return fence;
  }


-static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs,

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-			  const struct xe_vm_pgtable_update_op

*pt_op,

drivers/gpu/drm/xe/xe_migrate.c +0 -10
-			  const struct xe_vm_pgtable_update *update,
-			  struct xe_migrate_pt_update *pt_update)
-{
-	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
-	struct xe_vm *vm = pt_update->vops->vm;
-	u32 chunk;
-	u32 ofs = update->ofs, size = update->qwords;
-
-	/*
-	 * If we have 512 entries (max), we would populate it

ourselves,

drivers/gpu/drm/xe/xe_migrate.c +0 -2
-	 * and update the PDE above it to the new pointer.
-	 * The only time this can only happen if we have to update

the top

drivers/gpu/drm/xe/xe_migrate.c +0 -3
-	 * PDE. This requires a BO that is almost vm->size big.
-	 *
-	 * This shouldn't be possible in practice.. might change

when 16K

drivers/gpu/drm/xe/xe_migrate.c +0 -6
-	 * pages are used. Hence the assert.
-	 */
-	xe_tile_assert(tile, update->qwords < MAX_NUM_PTE);
-	if (!ppgtt_ofs)
-		ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile),
-						xe_bo_addr(update-
Quoted 1 line

pt_bo, 0,

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-							  

XE_PAGE_SIZE));

drivers/gpu/drm/xe/xe_migrate.c +0 -7
-
-	do {
-		u64 addr = ppgtt_ofs + ofs * 8;
-
-		chunk = min(size, MAX_PTE_PER_SDI);
-
-		/* Ensure populatefn can do memset64 by aligning bb-
Quoted 1 line

cs */

drivers/gpu/drm/xe/xe_migrate.c +0 -4
-		if (!(bb->len & 1))
-			bb->cs[bb->len++] = MI_NOOP;
-
-		bb->cs[bb->len++] = MI_STORE_DATA_IMM |

MI_SDI_NUM_QW(chunk);

drivers/gpu/drm/xe/xe_migrate.c +0 -15
-		bb->cs[bb->len++] = lower_32_bits(addr);
-		bb->cs[bb->len++] = upper_32_bits(addr);
-		if (pt_op->bind)
-			ops->populate(tile, NULL, bb->cs + bb->len,
-				      ofs, chunk, update);
-		else
-			ops->clear(vm, tile, NULL, bb->cs + bb->len,
-				   ofs, chunk, update);
-
-		bb->len += chunk * 2;
-		ofs += chunk;
-		size -= chunk;
-	} while (size);
-}
-

 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) {
  return xe_vm_get(m->q->vm);

drivers/gpu/drm/xe/xe_migrate.c +0 -0
@@ -1169,11 +1119,10 @@ struct migrate_test_params {
  	container_of(_priv, struct migrate_test_params, base)
  #endif

drivers/gpu/drm/xe/xe_migrate.c +0 -2
-static void
-__xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile

*tile,

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-				 const struct

xe_migrate_pt_update_ops *ops,

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-				 struct xe_vm_pgtable_update_op

*pt_op,

drivers/gpu/drm/xe/xe_migrate.c +1 -1
-				 int num_ops)
+void __xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct

xe_tile *tile,

drivers/gpu/drm/xe/xe_migrate.c +1 -0
+				      const struct

xe_migrate_pt_update_ops *ops,

drivers/gpu/drm/xe/xe_migrate.c +1 -0
+				      struct xe_vm_pgtable_update_op

*pt_op,

drivers/gpu/drm/xe/xe_migrate.c +1 -0
+				      int num_ops)

 {
  u32 j, i;

drivers/gpu/drm/xe/xe_migrate.c +0 -42
@@ -1234,131 +1183,15 @@ __xe_migrate_update_pgtables(struct
 xe_migrate *m,
  {
  	const struct xe_migrate_pt_update_ops *ops = pt_update->ops;
  	struct xe_tile *tile = m->tile;
-	struct xe_gt *gt = tile->primary_gt;
-	struct xe_device *xe = tile_to_xe(tile);
  	struct xe_sched_job *job;
  	struct dma_fence *fence;
-	struct drm_suballoc *sa_bo = NULL;
-	struct xe_bb *bb;
-	u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs =
 0;
-	u32 num_updates = 0, current_update = 0;
-	u64 addr;
-	int err = 0;
  	bool is_migrate = pt_update_ops->q == m->q;
-	bool usm = is_migrate && xe->info.has_usm;
-
-	for (i = 0; i < pt_update_ops->num_ops; ++i) {
-		struct xe_vm_pgtable_update_op *pt_op =
 &pt_update_ops->ops[i];
-		struct xe_vm_pgtable_update *updates = pt_op-
 >entries;
-
-		num_updates += pt_op->num_entries;
-		for (j = 0; j < pt_op->num_entries; ++j) {
-			u32 num_cmds =
 DIV_ROUND_UP(updates[j].qwords, 0x1ff);
-
-			/* align noop + MI_STORE_DATA_IMM cmd prefix
 */
-			batch_size += 4 * num_cmds +
 updates[j].qwords * 2;
-		}
-	}
-
-	/* fixed + PTE entries */
-	if (IS_DGFX(xe))
-		batch_size += 2;
-	else
-		batch_size += 6 + num_updates * 2;
-
-	bb = xe_bb_new(gt, batch_size, usm);
-	if (IS_ERR(bb))
-		return ERR_CAST(bb);
-
-	/* For sysmem PTE's, need to map them in our hole.. */
-	if (!IS_DGFX(xe)) {
-		ppgtt_ofs = NUM_KERNEL_PDE - 1;
-		if (!is_migrate) {
-			u32 num_units = DIV_ROUND_UP(num_updates,
-						    
 NUM_VMUSA_WRITES_PER_UNIT);
-
-			sa_bo = drm_suballoc_new(&m->vm_update_sa,
 num_units,
-						 GFP_KERNEL, true,

0);

drivers/gpu/drm/xe/xe_migrate.c +0 -15
-			if (IS_ERR(sa_bo)) {
-				err = PTR_ERR(sa_bo);
-				goto err;
-			}
-
-			ppgtt_ofs = NUM_KERNEL_PDE +
-				(drm_suballoc_soffset(sa_bo) /
-				 NUM_VMUSA_UNIT_PER_PAGE);
-			page_ofs = (drm_suballoc_soffset(sa_bo) %
-				    NUM_VMUSA_UNIT_PER_PAGE) *
-				VM_SA_UPDATE_UNIT_SIZE;
-		}
-
-		/* Map our PT's to gtt */
-		bb->cs[bb->len++] = MI_STORE_DATA_IMM |

MI_SDI_NUM_QW(num_updates);

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-		bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE +

page_ofs;

drivers/gpu/drm/xe/xe_migrate.c +0 -6
-		bb->cs[bb->len++] = 0; /* upper_32_bits */
-
-		for (i = 0; i < pt_update_ops->num_ops; ++i) {
-			struct xe_vm_pgtable_update_op *pt_op =
-				&pt_update_ops->ops[i];
-			struct xe_vm_pgtable_update *updates =

pt_op->entries;

drivers/gpu/drm/xe/xe_migrate.c +1 -3
-
-			for (j = 0; j < pt_op->num_entries; ++j,
++current_update) {
-				struct xe_vm *vm = pt_update->vops-
Quoted 1 line

vm;

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-				struct xe_bo *pt_bo =

updates[j].pt_bo;

drivers/gpu/drm/xe/xe_migrate.c +0 -2
-
-				xe_tile_assert(tile, pt_bo->size ==

SZ_4K);

drivers/gpu/drm/xe/xe_migrate.c +0 -4
-
-				/* Map a PT at most once */
-				if (pt_bo->update_index < 0)
-					pt_bo->update_index =

current_update;

drivers/gpu/drm/xe/xe_migrate.c +0 -2
-
-				addr = vm->pt_ops-
Quoted 1 line

pte_encode_bo(pt_bo, 0,

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-								

XE_CACHE_WB, 0);

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-				bb->cs[bb->len++] =

lower_32_bits(addr);

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-				bb->cs[bb->len++] =

upper_32_bits(addr);

drivers/gpu/drm/xe/xe_migrate.c +0 -12
-			}
-		}
-
-		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
-		update_idx = bb->len;
-
-		addr = xe_migrate_vm_addr(ppgtt_ofs, 0) +
-			(page_ofs / sizeof(u64)) * XE_PAGE_SIZE;
-		for (i = 0; i < pt_update_ops->num_ops; ++i) {
-			struct xe_vm_pgtable_update_op *pt_op =
-				&pt_update_ops->ops[i];
-			struct xe_vm_pgtable_update *updates =

pt_op->entries;

drivers/gpu/drm/xe/xe_migrate.c +0 -3
-
-			for (j = 0; j < pt_op->num_entries; ++j) {
-				struct xe_bo *pt_bo =

updates[j].pt_bo;

drivers/gpu/drm/xe/xe_migrate.c +0 -3
-
-				write_pgtable(tile, bb, addr +
-					      pt_bo->update_index *

XE_PAGE_SIZE,

drivers/gpu/drm/xe/xe_migrate.c +0 -1
-					      pt_op, &updates[j],

pt_update);

drivers/gpu/drm/xe/xe_migrate.c +0 -11
-			}
-		}
-	} else {
-		/* phys pages, no preamble required */
-		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
-		update_idx = bb->len;
-
-		for (i = 0; i < pt_update_ops->num_ops; ++i) {
-			struct xe_vm_pgtable_update_op *pt_op =
-				&pt_update_ops->ops[i];
-			struct xe_vm_pgtable_update *updates =

pt_op->entries;

drivers/gpu/drm/xe/xe_migrate.c +0 -3
-
-			for (j = 0; j < pt_op->num_entries; ++j)
-				write_pgtable(tile, bb, 0, pt_op,

&updates[j],

drivers/gpu/drm/xe/xe_migrate.c +1 -3
-					      pt_update);
-		}
-	}
+	int err;


  if (is_migrate)
  mutex_lock(&m->job_mutex);

drivers/gpu/drm/xe/xe_migrate.c +0 -2
-	job = xe_bb_create_migration_job(pt_update_ops->q, bb,
-					 xe_migrate_batch_base(m,

usm),

drivers/gpu/drm/xe/xe_migrate.c +1 -1
-					 update_idx);
+	job = xe_sched_job_create(pt_update_ops->q, NULL);

  if (IS_ERR(job)) {
  err = PTR_ERR(job);
  goto err_bb;

drivers/gpu/drm/xe/xe_migrate.c +13 -0
@@ -1370,6 +1203,19 @@ __xe_migrate_update_pgtables(struct xe_migrate
 *m,
  		if (err)
  			goto err_job;
  	}
+
+	set_bit(JOB_FLAG_PT, &job->fence->flags);
+	job->pt_update[0].vm = pt_update->vops->vm;
+	job->pt_update[0].tile = tile;
+	job->pt_update[0].ops = ops;
+	job->pt_update[0].pt_op = pt_update_ops->ops;
+	job->pt_update[0].num_ops = pt_update_ops->num_ops;
+	job->pt_update[0].deferred = pt_update_ops->deferred;
+
+	/* Submission backend now owns freeing of pt_update_ops->ops
 */
+	init_llist_head(&pt_update_ops->deferred);
+	pt_update_ops->skip_free = true;
+
  	xe_sched_job_arm(job);

  fence = dma_fence_get(&job->drm.s_fence->finished); xe_sched_job_push(job);

drivers/gpu/drm/xe/xe_migrate.c +0 -0
@@ -1377,9 +1223,6 @@ __xe_migrate_update_pgtables(struct xe_migrate
 *m,
  	if (is_migrate)
  		mutex_unlock(&m->job_mutex);

drivers/gpu/drm/xe/xe_migrate.c +0 -3
-	xe_bb_free(bb, fence);
-	drm_suballoc_free(sa_bo, fence);
-

  return fence;

 err_job:

drivers/gpu/drm/xe/xe_migrate.c +0 -0
@@ -1387,9 +1230,6 @@ __xe_migrate_update_pgtables(struct xe_migrate
 *m,

 err_bb:
  if (is_migrate)
  mutex_unlock(&m->job_mutex);

drivers/gpu/drm/xe/xe_migrate.c +0 -3
-	xe_bb_free(bb, NULL);
-err:
-	drm_suballoc_free(sa_bo, NULL);

  return ERR_PTR(err);
 }

drivers/gpu/drm/xe/xe_migrate.h +1 -0
index 18f5a8e40b5c..f80c94bb8f4c 100644
@@ -22,6 +22,7 @@ struct xe_pt;
  struct xe_tile;
  struct xe_vm;
  struct xe_vm_pgtable_update;
+struct xe_vm_pgtable_update_op;
  struct xe_vma;


 /**

drivers/gpu/drm/xe/xe_migrate.h +0 -0
@@ -106,6 +107,11 @@ struct dma_fence *xe_migrate_clear(struct
 xe_migrate *m,


 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m);
+void __xe_migrate_update_pgtables_cpu(struct xe_vm *vm, struct xe_tile *tile,

drivers/gpu/drm/xe/xe_migrate.h +1 -0
+				      const struct

xe_migrate_pt_update_ops *ops,

drivers/gpu/drm/xe/xe_migrate.h +1 -0
+				      struct xe_vm_pgtable_update_op

*pt_op,

drivers/gpu/drm/xe/xe_migrate.h +2 -0
+				      int num_ops);
+

 struct dma_fence *
 xe_migrate_update_pgtables(struct xe_migrate *m,
     struct xe_migrate_pt_update *pt_update);

drivers/gpu/drm/xe/xe_pt.c +1 -1
index 4758ae8a5459..26a00af6c4a6 100644
@@ -315,7 +315,7 @@ xe_pt_new_shared(struct xe_walk_update *wupd,
 struct xe_pt *parent,
  	entry->pt = parent;
  	entry->flags = 0;
  	entry->qwords = 0;
-	entry->pt_bo->update_index = -1;
+	entry->level = parent->level;


  if (alloc_entries) {
  entry->pt_entries = kmalloc_array(XE_PDES,

drivers/gpu/drm/xe/xe_pt.c +0 -0
@@ -1476,7 +1476,7 @@ xe_migrate_clear_pgtable_callback(struct xe_vm
 *vm, struct xe_tile *tile,
  				  u32 qword_ofs, u32 num_qwords,

    const struct xe_vm_pgtable_update *update)
 {

drivers/gpu/drm/xe/xe_pt.c +1 -1
-	u64 empty = __xe_pt_empty_pte(tile, vm, update->pt->level);
+	u64 empty = __xe_pt_empty_pte(tile, vm, update->level);

  int i;

  if (map && map->is_iomem)

drivers/gpu/drm/xe/xe_pt.c +0 -0
@@ -1890,7 +1890,7 @@ xe_pt_update_ops_run(struct xe_tile *tile,
 struct xe_vma_ops *vops)
  		goto free_ifence;
  	}

drivers/gpu/drm/xe/xe_pt.c +0 -1
-	/* FIXME: Point of no return - VM killed if failure after

this */

drivers/gpu/drm/xe/xe_pt.c +1 -0
+	/* Point of no return - VM killed if failure after this */

  for (i = 0; i < pt_update_ops->num_ops; ++i) { struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i];

drivers/gpu/drm/xe/xe_pt.c +0 -0
@@ -1953,6 +1953,21 @@ xe_pt_update_ops_run(struct xe_tile *tile,
 struct xe_vma_ops *vops)
  	return ERR_PTR(err);
  }

drivers/gpu/drm/xe/xe_pt.c +8 -0
+/**
+ * xe_pt_update_ops_free() - Free PT update operations
+ * @pt_op: Array of PT update operations
+ * @num_ops: Number of PT update operations
+ *
+ * Free PT update operations
+ */
+void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op,

u32 num_ops)

drivers/gpu/drm/xe/xe_pt.c +7 -0
+{
+	u32 i;
+
+	for (i = 0; i < num_ops; ++i, ++pt_op)
+		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
+}
+

 /**
  * xe_pt_update_ops_fini() - Finish PT update operations
  * @tile: Tile of PT update operations

drivers/gpu/drm/xe/xe_pt.c +0 -0
@@ -1964,17 +1979,16 @@ void xe_pt_update_ops_fini(struct xe_tile
 *tile, struct xe_vma_ops *vops)
  {

  struct xe_vm_pgtable_update_ops *pt_update_ops = &vops->pt_update_ops[tile->id];

drivers/gpu/drm/xe/xe_pt.c +0 -1
-	int i;


  lockdep_assert_held(&vops->vm->lock);
  xe_vm_assert_held(vops->vm);

drivers/gpu/drm/xe/xe_pt.c +0 -2
-	for (i = 0; i < pt_update_ops->num_ops; ++i) {
-		struct xe_vm_pgtable_update_op *pt_op =

&pt_update_ops->ops[i];

drivers/gpu/drm/xe/xe_pt.c +1 -4
-
-		xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
-	}
-	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+	xe_bo_put_commit(tile_to_xe(tile), &pt_update_ops-
Quoted 1 line

deferred);

drivers/gpu/drm/xe/xe_pt.c +5 -0
+	if (!pt_update_ops->skip_free)
+		xe_pt_update_ops_free(pt_update_ops->ops,
+				      pt_update_ops->num_ops);
+	else
+		pt_update_ops->ops = NULL;

 }

 /**

drivers/gpu/drm/xe/xe_pt.c +0 -0
@@ -2009,5 +2023,5 @@ void xe_pt_update_ops_abort(struct xe_tile
 *tile, struct xe_vma_ops *vops)
  					   pt_op->num_entries);
  	}

drivers/gpu/drm/xe/xe_pt.c +1 -1
-	xe_bo_put_commit(&vops->pt_update_ops[tile->id].deferred);
+	xe_pt_update_ops_fini(tile, vops);

 }

drivers/gpu/drm/xe/xe_pt.h +0 -0
index 9ab386431cad..989c9b190fa0 100644
@@ -40,6 +40,7 @@ struct dma_fence *xe_pt_update_ops_run(struct
 xe_tile *tile,
  				       struct xe_vma_ops *vops);

 void xe_pt_update_ops_fini(struct xe_tile *tile, struct xe_vma_ops *vops);
 void xe_pt_update_ops_abort(struct xe_tile *tile, struct xe_vma_ops *vops);
+void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op, u32 num_ops);

 bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma);

drivers/gpu/drm/xe/xe_pt_types.h +0 -0
index 384cc04de719..cfd0d35408a5 100644
@@ -70,6 +70,9 @@ struct xe_vm_pgtable_update {
  	/** @pt_entries: Newly added pagetable entries */
  	struct xe_pt_entry *pt_entries;

drivers/gpu/drm/xe/xe_pt_types.h +3 -0
+	/** @level: level of update */
+	unsigned int level;
+

  /** @flags: Target flags */
  u32 flags;
 };

drivers/gpu/drm/xe/xe_pt_types.h +1 -0
@@ -120,6 +123,8 @@ struct xe_vm_pgtable_update_ops {
  	 * slots are idle.
  	 */
  	bool wait_vm_kernel;
+	/** @skip_free: Free @ops in submission backend rather than

in IOCTL */

drivers/gpu/drm/xe/xe_pt_types.h +1 -0
+	bool skip_free;

 };

 #endif

drivers/gpu/drm/xe/xe_sched_job.c +0 -0
index 8151ddafb940..5930f03a81ad 100644
@@ -34,8 +34,10 @@ int __init xe_sched_job_module_init(void)

  xe_sched_job_parallel_slab = kmem_cache_create("xe_sched_job_parallel", sizeof(struct xe_sched_job) +

drivers/gpu/drm/xe/xe_sched_job.c +1 -0
+				  max_t(size_t,

    sizeof(u64) *

drivers/gpu/drm/xe/xe_sched_job.c +2 -1
-				  XE_HW_ENGINE_MAX_INSTANCE, 0,
+				  XE_HW_ENGINE_MAX_INSTANCE,
+				  sizeof(struct pt_update_args)), 0,

    SLAB_HWCACHE_ALIGN, NULL); if (!xe_sched_job_parallel_slab) { kmem_cache_destroy(xe_sched_job_slab);

drivers/gpu/drm/xe/xe_sched_job.c +0 -0
@@ -108,7 +110,13 @@ struct xe_sched_job *xe_sched_job_create(struct
 xe_exec_queue *q,
  	if (err)
  		goto err_free;

drivers/gpu/drm/xe/xe_sched_job.c +2 -1
-	if (!xe_exec_queue_is_parallel(q)) {
+	if (!batch_addr) {
+		job->fence =

dma_fence_allocate_private_stub(ktime_get());

drivers/gpu/drm/xe/xe_sched_job.c +5 -0
+		if (!job->fence) {
+			err = -ENOMEM;
+			goto err_sched_job;
+		}
+	} else if (!xe_exec_queue_is_parallel(q)) {

  job->fence = xe_lrc_create_seqno_fence(q->lrc); if (IS_ERR(job->fence)) {
  err = PTR_ERR(job->fence);

drivers/gpu/drm/xe/xe_sched_job.c +0 -0
@@ -148,12 +156,14 @@ struct xe_sched_job *xe_sched_job_create(struct
 xe_exec_queue *q,
  		job->fence = &cf->base;
  	}

drivers/gpu/drm/xe/xe_sched_job.c +4 -3
-	width = q->width;
-	if (is_migration)
-		width = 2;
+	if (batch_addr) {
+		width = q->width;
+		if (is_migration)
+			width = 2;

drivers/gpu/drm/xe/xe_sched_job.c +3 -2
-	for (i = 0; i < width; ++i)
-		job->batch_addr[i] = batch_addr[i];
+		for (i = 0; i < width; ++i)
+			job->batch_addr[i] = batch_addr[i];
+	}


  /* All other jobs require a VM to be open which has a ref */ if (unlikely(q->flags & EXEC_QUEUE_FLAG_KERNEL))

drivers/gpu/drm/xe/xe_sched_job_types.h +0 -0
index b1d83da50a53..623d6dcc8ae5 100644
@@ -11,6 +11,28 @@
  #include <drm/gpu_scheduler.h>


 struct xe_exec_queue;

drivers/gpu/drm/xe/xe_sched_job_types.h +22 -0
+struct xe_migrate_pt_update_ops;
+struct xe_tile;
+struct xe_vm;
+struct xe_vm_pgtable_update_op;
+
+/**
+ * struct pt_update_args - PT update arguments
+ */
+struct pt_update_args {
+	/** @vm: VM */
+	struct xe_vm *vm;
+	/** @tile: Tile */
+	struct xe_tile *tile;
+	/** @ops: Migrate PT update ops */
+	const struct xe_migrate_pt_update_ops *ops;
+	/** @pt_op: PT update ops */
+	struct xe_vm_pgtable_update_op *pt_op;
+	/** @deferred: deferred list to destroy PT entries */
+	struct llist_head deferred;
+	/** @num_ops: number of PT update ops */
+	int num_ops;
+};


 /**
  * struct xe_sched_job - XE schedule job (batch buffer tracking)

drivers/gpu/drm/xe/xe_sched_job_types.h +0 -0
@@ -27,6 +49,7 @@ struct xe_sched_job {
  	 * can safely reference fence, fence cannot safely reference

job.
  */
 #define JOB_FLAG_SUBMIT DMA_FENCE_FLAG_USER_BITS
+#define JOB_FLAG_PT (DMA_FENCE_FLAG_USER_BITS << 1)
  struct dma_fence *fence;
  /** @user_fence: write back value when BB is complete */ struct {

drivers/gpu/drm/xe/xe_sched_job_types.h +0 -0
@@ -39,8 +62,12 @@ struct xe_sched_job {
  	} user_fence;
  	/** @migrate_flush_flags: Additional flush flags for

migration jobs */
  u32 migrate_flush_flags;

drivers/gpu/drm/xe/xe_sched_job_types.h +6 -2
-	/** @batch_addr: batch buffer address of job */
-	u64 batch_addr[];
+	union {
+		/** @batch_addr: batch buffer address of job */
+		u64 batch_addr[0];
+		/** @pt_update: PT update arguments */
+		struct pt_update_args pt_update[0];
+	};

 };

 struct xe_sched_job_snapshot {

drivers/gpu/drm/xe/xe_vm.c +0 -0
index 9d2d1c262445..3b3136045327 100644
@@ -749,6 +749,45 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)

  list_empty_careful(&vm->userptr.invalidated)) ? 0 :
-EAGAIN;
 }

+static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,

drivers/gpu/drm/xe/xe_vm.c +2 -0
+			    struct xe_exec_queue *q,
+			    struct xe_sync_entry *syncs, u32

num_syncs)

drivers/gpu/drm/xe/xe_vm.c +19 -0
+{
+	memset(vops, 0, sizeof(*vops));
+	INIT_LIST_HEAD(&vops->list);
+	vops->vm = vm;
+	vops->q = q;
+	vops->syncs = syncs;
+	vops->num_syncs = num_syncs;
+}
+
+static int xe_vma_ops_alloc(struct xe_vma_ops *vops)
+{
+	int i;
+
+	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
+		if (!vops->pt_update_ops[i].num_ops)
+			continue;
+
+		vops->pt_update_ops[i].ops =
+			kmalloc_array(vops-
Quoted 1 line

pt_update_ops[i].num_ops,

drivers/gpu/drm/xe/xe_vm.c +1 -0
+				      sizeof(*vops-
Quoted 1 line

pt_update_ops[i].ops),

drivers/gpu/drm/xe/xe_vm.c +16 -0
+				      GFP_KERNEL);
+		if (!vops->pt_update_ops[i].ops)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void xe_vma_ops_free(struct xe_vma_ops *vops)
+{
+	int i;
+
+	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
+		kfree(vops->pt_update_ops[i].ops);
+}
+

 /**
  * xe_vm_populate_dummy_rebind() - Populate dummy rebind VMA ops
  * @vm: The VM.

drivers/gpu/drm/xe/xe_vm.c +0 -0
@@ -756,9 +795,11 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm)
   * @tile_mask: tile mask for VMA ops
   *
   * Populate dummy VMA ops which can be used to issue a rebind for

the VMA
+ *
+ * Return: 0 on success, -ENOMEM on failure
  */
-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,

drivers/gpu/drm/xe/xe_vm.c +1 -1
-				 u8 tile_mask)
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma

*vma,

drivers/gpu/drm/xe/xe_vm.c +1 -0
+				u8 tile_mask)

 {
  int i;

drivers/gpu/drm/xe/xe_vm.c +2 -0
@@ -792,12 +833,15 @@ void xe_vm_populate_dummy_rebind(struct xe_vm
 *vm, struct xe_vma *vma,
  	vm->dummy_ops.op.map.immediate = true;
  	vm->dummy_ops.op.map.read_only = xe_vma_read_only(vma);
  	vm->dummy_ops.op.map.is_null = xe_vma_is_null(vma);
+
+	return xe_vma_ops_alloc(&vm->dummy_ops.vops);
  }


 struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) {
  struct dma_fence *fence = NULL;
  struct xe_vma *vma, *next;

drivers/gpu/drm/xe/xe_vm.c +1 -0
+	int err;


  lockdep_assert_held(&vm->lock);
  if (xe_vm_in_lr_mode(vm) && !rebind_worker)

drivers/gpu/drm/xe/xe_vm.c +0 -0
@@ -815,8 +859,12 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm,
 bool rebind_worker)

  else trace_xe_vma_rebind_exec(vma);

drivers/gpu/drm/xe/xe_vm.c +0 -1
-		xe_vm_populate_dummy_rebind(vm, vma, vma-
Quoted 1 line

tile_present);

drivers/gpu/drm/xe/xe_vm.c +1 -0
+		err = xe_vm_populate_dummy_rebind(vm, vma, vma-
Quoted 1 line

tile_present);

drivers/gpu/drm/xe/xe_vm.c +3 -0
+		if (err)
+			return ERR_PTR(err);
+

  fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);

drivers/gpu/drm/xe/xe_vm.c +1 -0
+		xe_vma_ops_free(&vm->dummy_ops.vops);

  if (IS_ERR(fence))
  return fence;
  }

drivers/gpu/drm/xe/xe_vm.c +0 -0
@@ -1305,48 +1353,6 @@ static void xe_vm_free_scratch(struct xe_vm
 *vm)
  	}
  }


-static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm,

drivers/gpu/drm/xe/xe_vm.c +0 -2
-			    struct xe_exec_queue *q,
-			    struct xe_sync_entry *syncs, u32

num_syncs)

drivers/gpu/drm/xe/xe_vm.c +0 -19
-{
-	memset(vops, 0, sizeof(*vops));
-	INIT_LIST_HEAD(&vops->list);
-	vops->vm = vm;
-	vops->q = q;
-	vops->syncs = syncs;
-	vops->num_syncs = num_syncs;
-}
-
-static int xe_vma_ops_alloc(struct xe_vma_ops *vops)
-{
-	int i, j;
-
-	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i) {
-		if (!vops->pt_update_ops[i].num_ops)
-			continue;
-
-		vops->pt_update_ops[i].ops =
-			kmalloc_array(vops-
Quoted 1 line

pt_update_ops[i].num_ops,

drivers/gpu/drm/xe/xe_vm.c +0 -1
-				      sizeof(*vops-
Quoted 1 line

pt_update_ops[i].ops),

drivers/gpu/drm/xe/xe_vm.c +0 -6
-				      GFP_KERNEL);
-		if (!vops->pt_update_ops[i].ops)
-			return -ENOMEM;
-
-		for (j = 0; j < vops->pt_update_ops[i].num_ops; ++j)
-			vops->pt_update_ops[i].ops[j].num_entries =

0;

drivers/gpu/drm/xe/xe_vm.c +0 -13
-	}
-
-	return 0;
-}
-
-static void xe_vma_ops_fini(struct xe_vma_ops *vops)
-{
-	int i;
-
-	for (i = 0; i < XE_MAX_TILES_PER_DEVICE; ++i)
-		kfree(vops->pt_update_ops[i].ops);
-}
-

 static void xe_vma_ops_incr_pt_update_ops(struct xe_vma_ops *vops, u8 tile_mask)
 {
  int i;

drivers/gpu/drm/xe/xe_vm.c +0 -3
@@ -1381,9 +1387,6 @@ struct xe_vm *xe_vm_create(struct xe_device
 *xe, u32 flags)
  	list_add(&vm->dummy_ops.op.link, &vm->dummy_ops.vops.list);
  	for (id = 0; id < XE_MAX_TILES_PER_DEVICE; ++id)
  		vm->dummy_ops.vops.pt_update_ops[id].num_ops = 1;
-	err = xe_vma_ops_alloc(&vm->dummy_ops.vops);
-	if (err)
-		goto err_free;


  INIT_LIST_HEAD(&vm->rebind_list);

drivers/gpu/drm/xe/xe_vm.c +0 -2
@@ -1513,8 +1516,6 @@ struct xe_vm *xe_vm_create(struct xe_device
 *xe, u32 flags)
  		xe_device_mem_access_put(xe);
  	for_each_tile(tile, xe, id)
  		xe_range_fence_tree_fini(&vm->rftree[id]);
-err_free:
-	xe_vma_ops_fini(&vm->dummy_ops.vops);
  	kfree(vm);
  	return ERR_PTR(err);

 }

drivers/gpu/drm/xe/xe_vm.c +0 -0
@@ -1650,7 +1651,6 @@ static void vm_destroy_work_func(struct
 work_struct *w)
  		XE_WARN_ON(vm->pt_root[id]);


  trace_xe_vm_free(vm);

drivers/gpu/drm/xe/xe_vm.c +0 -1
-	xe_vma_ops_fini(&vm->dummy_ops.vops);

  kfree(vm);
 }

drivers/gpu/drm/xe/xe_vm.c +0 -0
@@ -2948,7 +2948,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev,
 void *data, struct drm_file *file)

 unwind_ops:
  if (err && err != -ENODATA) vm_bind_ioctl_ops_unwind(vm, ops, args->num_binds);

drivers/gpu/drm/xe/xe_vm.c +1 -1
-	xe_vma_ops_fini(&vops);
+	xe_vma_ops_free(&vops);

  for (i = args->num_binds - 1; i >= 0; --i)
  if (ops[i]) drm_gpuva_ops_free(&vm->gpuvm, ops[i]);

drivers/gpu/drm/xe/xe_vm.h +0 -0
index b8134804fce9..58e7490f7401 100644
@@ -262,8 +262,9 @@ static inline struct dma_resv *xe_vm_resv(struct
 xe_vm *vm)
   */
  #define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))


-void xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma *vma,

drivers/gpu/drm/xe/xe_vm.h +1 -1
-				 u8 tile_mask);
+int xe_vm_populate_dummy_rebind(struct xe_vm *vm, struct xe_vma

*vma,

drivers/gpu/drm/xe/xe_vm.h +2 -0
+				u8 tile_mask);
+void xe_vma_ops_free(struct xe_vma_ops *vops);

 struct dma_fence *xe_vm_ops_execute(struct xe_vm *vm, struct xe_vma_ops *vops);

 void xe_vm_kill(struct xe_vm *vm, bool unlocked);

author maintainer
#22 lore raw

PATCH v3 21/22 drm/xe: Don't use migrate exec queue for page fault binds

In reply to Matthew Brost

Now that the CPU is always used for binds even in jobs, CPU bind jobs can pass GPU jobs in the same exec queue resulting dma-fences signaling out-of-order. Use a dedicated exec queue for binds issued from page faults to avoid ordering issues.

---
 drivers/gpu/drm/xe/xe_gt_pagefault.c |  2 +-
 drivers/gpu/drm/xe/xe_migrate.c      | 22 +++++++++++++++++++++-
 drivers/gpu/drm/xe/xe_migrate.h      |  1 +
 3 files changed, 23 insertions(+), 2 deletions(-)
drivers/gpu/drm/xe/xe_gt_pagefault.c +1 -1
index 2beba426517e..a30bcf314589 100644
@@ -210,7 +210,7 @@ static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf)
 	if (ret)
 		goto unlock_dma_resv;
 	vm->dummy_ops.vops.pt_update_ops[tile->id].q =
-		xe_tile_migrate_exec_queue(tile);
+		xe_tile_migrate_bind_exec_queue(tile);
 	fence = xe_vm_ops_execute(vm, &vm->dummy_ops.vops);
 	xe_vma_ops_free(&vm->dummy_ops.vops);
 	if (IS_ERR(fence)) {
drivers/gpu/drm/xe/xe_migrate.c +21 -1
index 7d752e424b3b..1dd73e2117ea 100644
@@ -40,6 +40,8 @@
 struct xe_migrate {
 	/** @q: Default exec queue used for migration */
 	struct xe_exec_queue *q;
+	/** @bind_q: Default exec queue used for binds */
+	struct xe_exec_queue *bind_q;
 	/** @tile: Backpointer to the tile this struct xe_migrate belongs to. */
 	struct xe_tile *tile;
 	/** @job_mutex: Timeline mutex for @eng. */
@@ -96,6 +98,11 @@ struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile)
 	return tile->migrate->q;
 }
 
+struct xe_exec_queue *xe_tile_migrate_bind_exec_queue(struct xe_tile *tile)
+{
+	return tile->migrate->bind_q;
+}
+
 static void xe_migrate_fini(struct drm_device *dev, void *arg)
 {
 	struct xe_migrate *m = arg;
@@ -110,6 +117,8 @@ static void xe_migrate_fini(struct drm_device *dev, void *arg)
 	mutex_destroy(&m->job_mutex);
 	xe_vm_close_and_put(m->q->vm);
 	xe_exec_queue_put(m->q);
+	if (m->bind_q)
+		xe_exec_queue_put(m->bind_q);
 }
 
 static u64 xe_migrate_vm_addr(u64 slot, u32 level)
@@ -367,6 +376,15 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
 		if (!hwe || !logical_mask)
 			return ERR_PTR(-EINVAL);
 
+		m->bind_q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
+						 EXEC_QUEUE_FLAG_KERNEL |
+						 EXEC_QUEUE_FLAG_PERMANENT |
+						 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0);
+		if (IS_ERR(m->bind_q)) {
+			xe_vm_close_and_put(vm);
+			return ERR_CAST(m->bind_q);
+		}
+
 		m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe,
 					    EXEC_QUEUE_FLAG_KERNEL |
 					    EXEC_QUEUE_FLAG_PERMANENT |
@@ -378,6 +396,8 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile)
 						  EXEC_QUEUE_FLAG_PERMANENT);
 	}
 	if (IS_ERR(m->q)) {
+		if (m->bind_q)
+			xe_exec_queue_put(m->bind_q);
 		xe_vm_close_and_put(vm);
 		return ERR_CAST(m->q);
 	}
@@ -1185,7 +1205,7 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
 	struct xe_tile *tile = m->tile;
 	struct xe_sched_job *job;
 	struct dma_fence *fence;
-	bool is_migrate = pt_update_ops->q == m->q;
+	bool is_migrate = pt_update_ops->q == m->bind_q;
 	int err;
 
 	if (is_migrate)
drivers/gpu/drm/xe/xe_migrate.h +1 -0
index f80c94bb8f4c..701bb27349b0 100644
@@ -119,5 +119,6 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
 void xe_migrate_wait(struct xe_migrate *m);
 
 struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile);
+struct xe_exec_queue *xe_tile_migrate_bind_exec_queue(struct xe_tile *tile);
 
 #endif
Signature · 1 line
2.34.1
author maintainer
#23 lore raw

PATCH v3 22/22 drm/xe: Add VM bind IOCTL error injection

In reply to Matthew Brost

Add VM bind IOCTL error injection which steals MSB of the bind flags field which if set injects errors at various points in the VM bind IOCTL. Intended to validate error paths. Enabled by CONFIG_DRM_XE_DEBUG.

---
 drivers/gpu/drm/xe/xe_device_types.h | 12 ++++++++++++
 drivers/gpu/drm/xe/xe_pt.c           | 12 ++++++++++++
 drivers/gpu/drm/xe/xe_vm.c           | 21 +++++++++++++++++++++
 drivers/gpu/drm/xe/xe_vm_types.h     | 14 ++++++++++++++
 4 files changed, 59 insertions(+)
drivers/gpu/drm/xe/xe_device_types.h +12 -0
index 3018f1d79177..16bec6d7e724 100644
@@ -22,6 +22,10 @@
 #include "xe_sriov_types.h"
 #include "xe_step_types.h"
 
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
+#define TEST_VM_OPS_ERROR
+#endif
+
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 #include "soc/intel_pch.h"
 #include "intel_display_core.h"
@@ -467,6 +471,14 @@ struct xe_device {
 	/** @needs_flr_on_fini: requests function-reset on fini */
 	bool needs_flr_on_fini;
 
+#ifdef TEST_VM_OPS_ERROR
+	/**
+	 * @vm_inject_error_position: inject errors at different places in VM
+	 * bind IOCTL based on this value
+	 */
+	u8 vm_inject_error_position;
+#endif
+
 	/* private: */
 
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
drivers/gpu/drm/xe/xe_pt.c +12 -0
index 26a00af6c4a6..818546cc216d 100644
@@ -1748,6 +1748,12 @@ int xe_pt_update_ops_prepare(struct xe_tile *tile, struct xe_vma_ops *vops)
 	xe_tile_assert(tile, pt_update_ops->current_op ==
 		       pt_update_ops->num_ops);
 
+#ifdef TEST_VM_OPS_ERROR
+	if (vops->inject_error &&
+	    vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
+	       return -ENOSPC;
+#endif
+
 	return 0;
 }
 
@@ -1876,6 +1882,12 @@ xe_pt_update_ops_run(struct xe_tile *tile, struct xe_vma_ops *vops)
 	lockdep_assert_held(&vm->lock);
 	xe_vm_assert_held(vm);
 
+#ifdef TEST_VM_OPS_ERROR
+	if (vops->inject_error &&
+	    vm->xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
+	       return ERR_PTR(-ENOSPC);
+#endif
+
 	if (pt_update_ops->needs_invalidation) {
 		ifence = kzalloc(sizeof(*ifence), GFP_KERNEL);
 		if (!ifence) {
drivers/gpu/drm/xe/xe_vm.c +21 -0
index 3b3136045327..bd36512ef3d5 100644
@@ -2394,6 +2394,12 @@ static int vm_bind_ioctl_ops_lock(struct drm_exec *exec,
 			return err;
 	}
 
+#ifdef TEST_VM_OPS_ERROR
+	if (vops->inject_error &&
+	    vm->xe->vm_inject_error_position == FORCE_OP_ERROR_LOCK)
+	       return -ENOSPC;
+#endif
+
 	return 0;
 }
 
@@ -2601,9 +2607,15 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm,
 	return err;
 }
 
+#ifdef TEST_VM_OPS_ERROR
+#define SUPPORTED_FLAGS	\
+	(FORCE_OP_ERROR | DRM_XE_VM_BIND_FLAG_READONLY | \
+	 DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL)
+#else
 #define SUPPORTED_FLAGS	\
 	(DRM_XE_VM_BIND_FLAG_READONLY | \
 	 DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL)
+#endif
 #define XE_64K_PAGE_MASK 0xffffull
 #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP)
 
@@ -2931,6 +2943,15 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 					      &vops, i == args->num_binds - 1);
 		if (err)
 			goto unwind_ops;
+
+#ifdef TEST_VM_OPS_ERROR
+		if (flags & FORCE_OP_ERROR) {
+			vops.inject_error = true;
+			vm->xe->vm_inject_error_position =
+				(vm->xe->vm_inject_error_position + 1) %
+				FORCE_OP_ERROR_COUNT;
+		}
+#endif
 	}
 
 	/* Nothing to do */
drivers/gpu/drm/xe/xe_vm_types.h +14 -0
index 7e639ed5ba4a..b7094fc69053 100644
@@ -23,6 +23,16 @@ struct xe_sync_entry;
 struct xe_vm;
 struct xe_vm_pgtable_update_op;
 
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG)
+#define TEST_VM_OPS_ERROR
+#define FORCE_OP_ERROR	BIT(31)
+
+#define FORCE_OP_ERROR_LOCK	0
+#define FORCE_OP_ERROR_PREPARE	1
+#define FORCE_OP_ERROR_RUN	2
+#define FORCE_OP_ERROR_COUNT	3
+#endif
+
 #define XE_VMA_READ_ONLY	DRM_GPUVA_USERBITS
 #define XE_VMA_DESTROYED	(DRM_GPUVA_USERBITS << 1)
 #define XE_VMA_ATOMIC_PTE_BIT	(DRM_GPUVA_USERBITS << 2)
@@ -201,6 +211,10 @@ struct xe_vma_ops {
 	u32 num_syncs;
 	/** @pt_update_ops: page table update operations */
 	struct xe_vm_pgtable_update_ops pt_update_ops[XE_MAX_TILES_PER_DEVICE];
+#ifdef TEST_VM_OPS_ERROR
+	/** @inject_error: inject error to test error handling */
+	bool inject_error;
+#endif
 };
 
 struct xe_vm {
Signature · 1 line
2.34.1
#24 lore raw

✗ CI.Patch_applied: failure for Refactor VM bind code (rev3)

In reply to Matthew Brost
== Series Details ==

== Series Details ==

Series: Refactor VM bind code (rev3)
URL : https://patchwork.freedesktop.org/series/125608/
State : failure

== Summary ==

=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: 8a42fafc7 drm-tip: 2024y-02m-06d-13h-49m-55s UTC integration manifest === git am output follows ===
error: patch failed: drivers/gpu/drm/xe/xe_vm.c:2682
error: drivers/gpu/drm/xe/xe_vm.c: patch does not apply
hint: Use 'git am --show-current-patch' to see the failed patch
Applying: drm/xe: Lock all gpuva ops during VM bind IOCTL
Patch failed at 0001 drm/xe: Lock all gpuva ops during VM bind IOCTL When you have resolved this problem, run "git am --continue".
If you prefer to skip this patch, run "git am --skip" instead.
To restore the original branch and stop patching, run "git am --abort".

#25 lore raw

✓ CI.Patch_applied: success for Refactor VM bind code (rev4)

In reply to Matthew Brost
== Series Details ==

== Series Details ==

Series: Refactor VM bind code (rev4)
URL : https://patchwork.freedesktop.org/series/125608/
State : success

== Summary ==

=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: eb1e13ab1 drm-tip: 2024y-02m-07d-18h-19m-40s UTC integration manifest === git am output follows ===
Applying: drm/xe: Lock all gpuva ops during VM bind IOCTL
Applying: drm/xe: Add ops_execute function which returns a fence
Applying: drm/xe: Move migrate to prefetch to op_lock funtion
Applying: drm/xe: Add struct xe_vma_ops abstraction
Applying: drm/xe: Update xe_vm_rebind to use dummy VMA operations
Applying: drm/xe: Simplify VM bind IOCTL error handling and cleanup
Applying: drm/xe: Update pagefaults to use dummy VMA operations
Applying: drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue
Applying: drm/xe: Add vm_bind_ioctl_ops_install_fences helper
Applying: drm/xe: Move setting last fence to vm_bind_ioctl_ops_install_fences
Applying: drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this
Applying: drm/xe: Add some members to xe_vma_ops
Applying: drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
Applying: drm/xe: Convert multiple bind ops into single job
Applying: drm/xe: Remove old functions defs in xe_pt.h
Applying: drm/xe: Update PT layer with better error handling
Applying: drm/xe: Update VM trace events
Applying: drm/xe: Update clear / populate arguments
Applying: drm/xe: Add __xe_migrate_update_pgtables_cpu helper
Applying: drm/xe: CPU binds for jobs
Applying: drm/xe: Don't use migrate exec queue for page fault binds
Applying: drm/xe: Add VM bind IOCTL error injection

#26 lore raw

✗ CI.checkpatch: warning for Refactor VM bind code (rev4)

In reply to Matthew Brost
== Series Details ==

== Series Details ==

Series: Refactor VM bind code (rev4)
URL : https://patchwork.freedesktop.org/series/125608/
State : warning

== Summary ==

+ KERNEL=/kernel
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools mt Cloning into 'mt'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ git -C mt rev-list -n1 origin/master 35591fb8b4d5305b37ce31483f85ac0956eaa536
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ git log -n1
commit 819cd1163cc0ed5bd7b8f428abf8e64ee023e3c8

Author: Matthew Brost <[email protected]>
Date:   Tue Feb 6 15:37:29 2024 -0800

drm/xe: Add VM bind IOCTL error injection

Add VM bind IOCTL error injection which steals MSB of the bind flags field which if set injects errors at various points in the VM bind IOCTL. Intended to validate error paths. Enabled by CONFIG_DRM_XE_DEBUG.

+ /mt/dim checkpatch eb1e13ab1374d4785d5f2b2584d88e6e05311229 drm-intel 8ba74fe16 drm/xe: Lock all gpuva ops during VM bind IOCTL 87653de6b drm/xe: Add ops_execute function which returns a fence 883439553 drm/xe: Move migrate to prefetch to op_lock funtion -:4: WARNING:TYPO_SPELLING: 'funtion' may be misspelled - perhaps 'function'? #4:
Subject: [PATCH] drm/xe: Move migrate to prefetch to op_lock funtion

                                                             ^^^^^^^

total: 0 errors, 1 warnings, 0 checks, 49 lines checked
456930f69 drm/xe: Add struct xe_vma_ops abstraction
6200a4c05 drm/xe: Update xe_vm_rebind to use dummy VMA operations
5080c77b5 drm/xe: Simplify VM bind IOCTL error handling and cleanup
76fbc63eb drm/xe: Update pagefaults to use dummy VMA operations
dacb99ec3 drm/xe: s/xe_tile_migrate_engine/xe_tile_migrate_exec_queue 4d1c00496 drm/xe: Add vm_bind_ioctl_ops_install_fences helper
-:9: ERROR:BAD_SIGN_OFF: Unrecognized email address: 'Matthew Brost <[email protected]' #9:

-:134: ERROR:NO_AUTHOR_SIGN_OFF: Missing Signed-off-by: line by nominal patch author 'Matthew Brost <[email protected]>'

total: 2 errors, 0 warnings, 0 checks, 111 lines checked
af253508d drm/xe: Move setting last fence to vm_bind_ioctl_ops_install_fences 4f783081a drm/xe: Add xe_gt_tlb_invalidation_range and convert PT layer to use this -:107: CHECK:BRACES: Blank lines aren't necessary before a close brace '}' #107: FILE: drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c:306:
+
+}

total: 0 errors, 0 warnings, 1 checks, 186 lines checked
75c69c5e8 drm/xe: Add some members to xe_vma_ops
645be5ccb drm/xe: Add xe_vm_pgtable_update_op to xe_vma_ops
06c167151 drm/xe: Convert multiple bind ops into single job
-:1501: WARNING:TYPO_SPELLING: 'commiting' may be misspelled - perhaps 'committing'? #1501: FILE: drivers/gpu/drm/xe/xe_pt.c:1769:
+ * Run PT update operations which includes commiting internal PT state changes,

                                            ^^^^^^^^^

-:1684: WARNING:TYPO_SPELLING: 'commiting' may be misspelled - perhaps 'committing'? #1684: FILE: drivers/gpu/drm/xe/xe_pt.c:1863:
+ * Finish PT update operations by commiting to destroy page table memory

                                   ^^^^^^^^^

total: 0 errors, 2 warnings, 0 checks, 2435 lines checked a3d3d269d drm/xe: Remove old functions defs in xe_pt.h f2b277f1e drm/xe: Update PT layer with better error handling 5b5bab2d6 drm/xe: Update VM trace events
-:61: ERROR:SPACING: space prohibited after that open parenthesis '(' #61: FILE: drivers/gpu/drm/xe/xe_vm.c:2414:
+ trace_xe_vma_unbind( gpuva_to_vma(op->base.unmap.va));

total: 1 errors, 0 warnings, 0 checks, 88 lines checked 79ef9178a drm/xe: Update clear / populate arguments
7d9dc94ff drm/xe: Add __xe_migrate_update_pgtables_cpu helper 0e0adaf83 drm/xe: CPU binds for jobs
-:672: CHECK:PARENTHESIS_ALIGNMENT: Alignment should match open parenthesis #672: FILE: drivers/gpu/drm/xe/xe_sched_job.c:38:
+ max_t(size_t,
sizeof(u64) *

-:766: ERROR:FLEXIBLE_ARRAY: Use C99 flexible arrays - see https://docs.kernel.org/process/deprecated.html#zero-length-and-one-element-arrays
#766: FILE: drivers/gpu/drm/xe/xe_sched_job_types.h:70:
+ struct pt_update_args pt_update[0];
+ };

total: 1 errors, 0 warnings, 1 checks, 838 lines checked
63fa61f34 drm/xe: Don't use migrate exec queue for page fault binds 819cd1163 drm/xe: Add VM bind IOCTL error injection
-:51: WARNING:SUSPECT_CODE_INDENT: suspect code indent for conditional statements (8, 15) #51: FILE: drivers/gpu/drm/xe/xe_pt.c:1752:
+ if (vops->inject_error &&
[...]
+ return -ENOSPC;

-:53: WARNING:TABSTOP: Statements should start on a tabstop #53: FILE: drivers/gpu/drm/xe/xe_pt.c:1754:
+ return -ENOSPC;

-:64: WARNING:SUSPECT_CODE_INDENT: suspect code indent for conditional statements (8, 15) #64: FILE: drivers/gpu/drm/xe/xe_pt.c:1886:
+ if (vops->inject_error &&
[...]
+ return ERR_PTR(-ENOSPC);

-:66: WARNING:TABSTOP: Statements should start on a tabstop #66: FILE: drivers/gpu/drm/xe/xe_pt.c:1888:
+ return ERR_PTR(-ENOSPC);

-:81: WARNING:SUSPECT_CODE_INDENT: suspect code indent for conditional statements (8, 15) #81: FILE: drivers/gpu/drm/xe/xe_vm.c:2398:
+ if (vops->inject_error &&
[...]
+ return -ENOSPC;

-:83: WARNING:TABSTOP: Statements should start on a tabstop #83: FILE: drivers/gpu/drm/xe/xe_vm.c:2400:
+ return -ENOSPC;

total: 0 errors, 6 warnings, 0 checks, 116 lines checked

#27 lore raw

✗ CI.KUnit: failure for Refactor VM bind code (rev4)

In reply to Matthew Brost
== Series Details ==

== Series Details ==

Series: Refactor VM bind code (rev4)
URL : https://patchwork.freedesktop.org/series/125608/
State : failure

== Summary ==

+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig ERROR:root:../arch/x86/um/user-offsets.c:17:6: warning: no previous prototype for ‘foo’ [-Wmissing-prototypes] 17 | void foo(void)

      |      ^~~

In file included from ../arch/um/kernel/asm-offsets.c:1:
../arch/x86/um/shared/sysdep/kernel-offsets.h:9:6: warning: no previous prototype for ‘foo’ [-Wmissing-prototypes]
9 | void foo(void)
| ^~~
../arch/x86/um/fault.c:18:5: warning: no previous prototype for ‘arch_fixup’ [-Wmissing-prototypes]
18 | int arch_fixup(unsigned long address, struct uml_pt_regs *regs)

      |     ^~~~~~~~~~

../arch/x86/um/bugs_64.c:9:6: warning: no previous prototype for ‘arch_check_bugs’ [-Wmissing-prototypes]
9 | void arch_check_bugs(void)
| ^~~~~~~~~~~~~~~
../arch/x86/um/bugs_64.c:13:6: warning: no previous prototype for ‘arch_examine_signal’ [-Wmissing-prototypes]
13 | void arch_examine_signal(int sig, struct uml_pt_regs *regs)

      |      ^~~~~~~~~~~~~~~~~~~

../arch/x86/um/os-Linux/mcontext.c:7:6: warning: no previous prototype for ‘get_regs_from_mc’ [-Wmissing-prototypes]
7 | void get_regs_from_mc(struct uml_pt_regs *regs, mcontext_t *mc)
| ^~~~~~~~~~~~~~~~
../arch/x86/um/os-Linux/registers.c:146:15: warning: no previous prototype for ‘get_thread_reg’ [-Wmissing-prototypes]
146 | unsigned long get_thread_reg(int reg, jmp_buf *buf)

      |               ^~~~~~~~~~~~~~
../arch/x86/um/vdso/um_vdso.c:16:5: warning: no previous prototype for ‘__vdso_clock_gettime’ [-Wmissing-prototypes]
   16 | int __vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
      |     ^~~~~~~~~~~~~~~~~~~~
../arch/x86/um/vdso/um_vdso.c:30:5: warning: no previous prototype for ‘__vdso_gettimeofday’ [-Wmissing-prototypes]
   30 | int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
      |     ^~~~~~~~~~~~~~~~~~~
../arch/x86/um/vdso/um_vdso.c:44:21: warning: no previous prototype for ‘__vdso_time’ [-Wmissing-prototypes]
   44 | __kernel_old_time_t __vdso_time(__kernel_old_time_t *t)
      |                     ^~~~~~~~~~~
../arch/x86/um/vdso/um_vdso.c:57:1: warning: no previous prototype for ‘__vdso_getcpu’ [-Wmissing-prototypes]
   57 | __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused)
      | ^~~~~~~~~~~~~
../arch/um/os-Linux/skas/process.c:107:6: warning: no previous prototype for ‘wait_stub_done’ [-Wmissing-prototypes]
  107 | void wait_stub_done(int pid)
      |      ^~~~~~~~~~~~~~
../arch/um/os-Linux/skas/process.c:683:6: warning: no previous prototype for ‘__switch_mm’ [-Wmissing-prototypes]
  683 | void __switch_mm(struct mm_id *mm_idp)
      |      ^~~~~~~~~~~
../arch/um/kernel/skas/process.c:36:12: warning: no previous prototype for ‘start_uml’ [-Wmissing-prototypes]
   36 | int __init start_uml(void)
      |            ^~~~~~~~~
../arch/x86/um/ptrace_64.c:111:5: warning: no previous prototype for ‘poke_user’ [-Wmissing-prototypes]
  111 | int poke_user(struct task_struct *child, long addr, long data)
      |     ^~~~~~~~~
../arch/x86/um/ptrace_64.c:171:5: warning: no previous prototype for ‘peek_user’ [-Wmissing-prototypes]
  171 | int peek_user(struct task_struct *child, long addr, long data)
      |     ^~~~~~~~~
../arch/x86/um/signal.c:560:6: warning: no previous prototype for ‘sys_rt_sigreturn’ [-Wmissing-prototypes]
  560 | long sys_rt_sigreturn(void)
      |      ^~~~~~~~~~~~~~~~
../arch/um/kernel/skas/mmu.c:17:5: warning: no previous prototype for ‘init_new_context’ [-Wmissing-prototypes]
   17 | int init_new_context(struct task_struct *task, struct mm_struct *mm)
      |     ^~~~~~~~~~~~~~~~
../arch/um/kernel/skas/mmu.c:60:6: warning: no previous prototype for ‘destroy_context’ [-Wmissing-prototypes]
   60 | void destroy_context(struct mm_struct *mm)
      |      ^~~~~~~~~~~~~~~
../arch/um/os-Linux/main.c:187:7: warning: no previous prototype for ‘__wrap_malloc’ [-Wmissing-prototypes]
  187 | void *__wrap_malloc(int size)
      |       ^~~~~~~~~~~~~
../arch/um/os-Linux/main.c:208:7: warning: no previous prototype for ‘__wrap_calloc’ [-Wmissing-prototypes]
  208 | void *__wrap_calloc(int n, int size)
      |       ^~~~~~~~~~~~~
../arch/um/os-Linux/main.c:222:6: warning: no previous prototype for ‘__wrap_free’ [-Wmissing-prototypes]
  222 | void __wrap_free(void *ptr)
      |      ^~~~~~~~~~~
../arch/um/os-Linux/mem.c:28:6: warning: no previous prototype for ‘kasan_map_memory’ [-Wmissing-prototypes]
   28 | void kasan_map_memory(void *start, size_t len)
      |      ^~~~~~~~~~~~~~~~
../arch/um/os-Linux/mem.c:212:13: warning: no previous prototype for ‘check_tmpexec’ [-Wmissing-prototypes]
  212 | void __init check_tmpexec(void)
      |             ^~~~~~~~~~~~~
../arch/um/os-Linux/signal.c:75:6: warning: no previous prototype for ‘sig_handler’ [-Wmissing-prototypes]
   75 | void sig_handler(int sig, struct siginfo *si, mcontext_t *mc)
      |      ^~~~~~~~~~~
../arch/um/os-Linux/signal.c:111:6: warning: no previous prototype for ‘timer_alarm_handler’ [-Wmissing-prototypes]
  111 | void timer_alarm_handler(int sig, struct siginfo *unused_si, mcontext_t *mc)
      |      ^~~~~~~~~~~~~~~~~~~
../arch/um/os-Linux/start_up.c:301:12: warning: no previous prototype for ‘parse_iomem’ [-Wmissing-prototypes]
  301 | int __init parse_iomem(char *str, int *add)
      |            ^~~~~~~~~~~
../arch/x86/um/syscalls_64.c:48:6: warning: no previous prototype for ‘arch_switch_to’ [-Wmissing-prototypes]
   48 | void arch_switch_to(struct task_struct *to)
      |      ^~~~~~~~~~~~~~
../arch/um/kernel/mem.c:202:8: warning: no previous prototype for ‘pgd_alloc’ [-Wmissing-prototypes]
  202 | pgd_t *pgd_alloc(struct mm_struct *mm)
      |        ^~~~~~~~~
../arch/um/kernel/mem.c:215:7: warning: no previous prototype for ‘uml_kmalloc’ [-Wmissing-prototypes]
  215 | void *uml_kmalloc(int size, int flags)
      |       ^~~~~~~~~~~
../arch/um/kernel/process.c:51:5: warning: no previous prototype for ‘pid_to_processor_id’ [-Wmissing-prototypes]
   51 | int pid_to_processor_id(int pid)
      |     ^~~~~~~~~~~~~~~~~~~
../arch/um/kernel/process.c:87:7: warning: no previous prototype for ‘__switch_to’ [-Wmissing-prototypes]
   87 | void *__switch_to(struct task_struct *from, struct task_struct *to)
      |       ^~~~~~~~~~~
../arch/um/kernel/process.c:140:6: warning: no previous prototype for ‘fork_handler’ [-Wmissing-prototypes]
  140 | void fork_handler(void)
      |      ^~~~~~~~~~~~
../arch/um/kernel/process.c:217:6: warning: no previous prototype for ‘arch_cpu_idle’ [-Wmissing-prototypes]
  217 | void arch_cpu_idle(void)
      |      ^~~~~~~~~~~~~
../arch/um/kernel/process.c:253:5: warning: no previous prototype for ‘copy_to_user_proc’ [-Wmissing-prototypes]
  253 | int copy_to_user_proc(void __user *to, void *from, int size)
      |     ^~~~~~~~~~~~~~~~~
../arch/um/kernel/process.c:263:5: warning: no previous prototype for ‘clear_user_proc’ [-Wmissing-prototypes]
  263 | int clear_user_proc(void __user *buf, int size)
      |     ^~~~~~~~~~~~~~~
../arch/um/kernel/process.c:271:6: warning: no previous prototype for ‘set_using_sysemu’ [-Wmissing-prototypes]
  271 | void set_using_sysemu(int value)
      |      ^~~~~~~~~~~~~~~~
../arch/um/kernel/process.c:278:5: warning: no previous prototype for ‘get_using_sysemu’ [-Wmissing-prototypes]
  278 | int get_using_sysemu(void)
      |     ^~~~~~~~~~~~~~~~
../arch/um/kernel/process.c:316:12: warning: no previous prototype for ‘make_proc_sysemu’ [-Wmissing-prototypes]
  316 | int __init make_proc_sysemu(void)
      |            ^~~~~~~~~~~~~~~~
../arch/um/kernel/process.c:348:15: warning: no previous prototype for ‘arch_align_stack’ [-Wmissing-prototypes]
  348 | unsigned long arch_align_stack(unsigned long sp)
      |               ^~~~~~~~~~~~~~~~
../arch/um/kernel/reboot.c:45:6: warning: no previous prototype for ‘machine_restart’ [-Wmissing-prototypes]
   45 | void machine_restart(char * __unused)
      |      ^~~~~~~~~~~~~~~
../arch/um/kernel/reboot.c:51:6: warning: no previous prototype for ‘machine_power_off’ [-Wmissing-prototypes]
   51 | void machine_power_off(void)
      |      ^~~~~~~~~~~~~~~~~
../arch/um/kernel/reboot.c:57:6: warning: no previous prototype for ‘machine_halt’ [-Wmissing-prototypes]
   57 | void machine_halt(void)
      |      ^~~~~~~~~~~~
../arch/um/kernel/tlb.c:579:6: warning: no previous prototype for ‘flush_tlb_mm_range’ [-Wmissing-prototypes]
  579 | void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
      |      ^~~~~~~~~~~~~~~~~~
../arch/um/kernel/tlb.c:594:6: warning: no previous prototype for ‘force_flush_all’ [-Wmissing-prototypes]
  594 | void force_flush_all(void)
      |      ^~~~~~~~~~~~~~~
../arch/um/kernel/kmsg_dump.c:60:12: warning: no previous prototype for ‘kmsg_dumper_stdout_init’ [-Wmissing-prototypes]
   60 | int __init kmsg_dumper_stdout_init(void)
      |            ^~~~~~~~~~~~~~~~~~~~~~~
../arch/um/kernel/um_arch.c:408:19: warning: no previous prototype for ‘read_initrd’ [-Wmissing-prototypes]
  408 | int __init __weak read_initrd(void)
      |                   ^~~~~~~~~~~
../arch/um/kernel/um_arch.c:461:7: warning: no previous prototype for ‘text_poke’ [-Wmissing-prototypes]
  461 | void *text_poke(void *addr, const void *opcode, size_t len)
      |       ^~~~~~~~~
../arch/um/kernel/um_arch.c:473:6: warning: no previous prototype for ‘text_poke_sync’ [-Wmissing-prototypes]
  473 | void text_poke_sync(void)
      |      ^~~~~~~~~~~~~~
In file included from ../drivers/gpu/drm/xe/xe_migrate.c:1303:
../drivers/gpu/drm/xe/tests/xe_migrate.c:91:14: error: initialization of ‘void (*)(struct xe_tile *, struct iosys_map *, void *, u32,  u32,  const struct xe_vm_pgtable_update *)’ {aka ‘void (*)(struct xe_tile *, struct iosys_map *, void *, unsigned int,  unsigned int,  const struct xe_vm_pgtable_update *)’} from incompatible pointer type ‘void (*)(struct xe_migrate_pt_update *, struct xe_tile *, struct iosys_map *, void *, u32,  u32,  const struct xe_vm_pgtable_update *)’ {aka ‘void (*)(struct xe_migrate_pt_update *, struct xe_tile *, struct iosys_map *, void *, unsigned int,  unsigned int,  const struct xe_vm_pgtable_update *)’} [-Werror=incompatible-pointer-types]
   91 |  .populate = sanity_populate_cb,
      |              ^~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/tests/xe_migrate.c:91:14: note: (near initialization for ‘sanity_ops.populate’)
../drivers/gpu/drm/xe/tests/xe_migrate.c: In function ‘test_pt_update’:
../drivers/gpu/drm/xe/tests/xe_migrate.c:239:44: error: passing argument 2 of ‘xe_migrate_update_pgtables’ from incompatible pointer type [-Werror=incompatible-pointer-types]
  239 |  fence = xe_migrate_update_pgtables(m, m->q->vm, NULL, m->q, &update, 1,
      |                                        ~~~~^~~~
      |                                            |
      |                                            struct xe_vm *
../drivers/gpu/drm/xe/xe_migrate.c:1274:36: note: expected ‘struct xe_migrate_pt_update *’ but argument is of type ‘struct xe_vm *’
 1274 |       struct xe_migrate_pt_update *pt_update)
      |       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~
In file included from ../drivers/gpu/drm/xe/xe_migrate.c:1303:
../drivers/gpu/drm/xe/tests/xe_migrate.c:239:10: error: too many arguments to function ‘xe_migrate_update_pgtables’
  239 |  fence = xe_migrate_update_pgtables(m, m->q->vm, NULL, m->q, &update, 1,
      |          ^~~~~~~~~~~~~~~~~~~~~~~~~~
../drivers/gpu/drm/xe/xe_migrate.c:1273:1: note: declared here
 1273 | xe_migrate_update_pgtables(struct xe_migrate *m,
      | ^~~~~~~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors
make[7]: *** [../scripts/Makefile.build:243: drivers/gpu/drm/xe/xe_migrate.o] Error 1 make[7]: *** Waiting for unfinished jobs....
../lib/iomap.c:156:5: warning: no previous prototype for ‘ioread64_lo_hi’ [-Wmissing-prototypes]
156 | u64 ioread64_lo_hi(const void __iomem *addr)

      |     ^~~~~~~~~~~~~~
../lib/iomap.c:163:5: warning: no previous prototype for ‘ioread64_hi_lo’ [-Wmissing-prototypes]
  163 | u64 ioread64_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~
../lib/iomap.c:170:5: warning: no previous prototype for ‘ioread64be_lo_hi’ [-Wmissing-prototypes]
  170 | u64 ioread64be_lo_hi(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:178:5: warning: no previous prototype for ‘ioread64be_hi_lo’ [-Wmissing-prototypes]
  178 | u64 ioread64be_hi_lo(const void __iomem *addr)
      |     ^~~~~~~~~~~~~~~~
../lib/iomap.c:264:6: warning: no previous prototype for ‘iowrite64_lo_hi’ [-Wmissing-prototypes]
  264 | void iowrite64_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:272:6: warning: no previous prototype for ‘iowrite64_hi_lo’ [-Wmissing-prototypes]
  272 | void iowrite64_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~
../lib/iomap.c:280:6: warning: no previous prototype for ‘iowrite64be_lo_hi’ [-Wmissing-prototypes]
  280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~
../lib/iomap.c:288:6: warning: no previous prototype for ‘iowrite64be_hi_lo’ [-Wmissing-prototypes]
  288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr)
      |      ^~~~~~~~~~~~~~~~~

make[6]: *** [../scripts/Makefile.build:481: drivers/gpu/drm/xe] Error 2 make[6]: *** Waiting for unfinished jobs....
make[5]: *** [../scripts/Makefile.build:481: drivers/gpu/drm] Error 2 make[4]: *** [../scripts/Makefile.build:481: drivers/gpu] Error 2 make[3]: *** [../scripts/Makefile.build:481: drivers] Error 2 make[2]: *** [/kernel/Makefile:1921: .] Error 2
make[1]: *** [/kernel/Makefile:240: __sub-make] Error 2
make: *** [Makefile:240: __sub-make] Error 2

[22:12:35] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:

$ make ARCH=um O=.kunit olddefconfig
[22:12:39] Building KUnit Kernel ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig
Building with:
$ make ARCH=um O=.kunit --jobs=48
+ cleanup
++ stat -c %u:%g /kernel
+ chown -R 1003:1003 /kernel