drm/xe: CPU binds
Subsystems (2)
Matthew Brost
maintainerRFC PATCH 1/1 drm/xe: CPU binds
Debug for isse #799, do not review.
Signed-off-by Matthew Brost <[email protected]>
--- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_bo.c | 7 +- drivers/gpu/drm/xe/xe_bo.h | 4 +- drivers/gpu/drm/xe/xe_device.c | 35 + drivers/gpu/drm/xe/xe_device.h | 2 + drivers/gpu/drm/xe/xe_device_types.h | 4 + drivers/gpu/drm/xe/xe_exec.c | 2 +- drivers/gpu/drm/xe/xe_exec_queue.c | 112 +- drivers/gpu/drm/xe/xe_exec_queue_types.h | 20 +- drivers/gpu/drm/xe/xe_guc_submit.c | 24 +- drivers/gpu/drm/xe/xe_migrate.c | 385 ++---- drivers/gpu/drm/xe/xe_migrate.h | 46 +- drivers/gpu/drm/xe/xe_pt.c | 1359 +++++++++++++++------- drivers/gpu/drm/xe/xe_pt.h | 17 +- drivers/gpu/drm/xe/xe_pt_exec_queue.c | 170 +++ drivers/gpu/drm/xe/xe_pt_exec_queue.h | 14 + drivers/gpu/drm/xe/xe_pt_types.h | 67 ++ drivers/gpu/drm/xe/xe_sched_job.c | 104 +- drivers/gpu/drm/xe/xe_sched_job_types.h | 30 +- drivers/gpu/drm/xe/xe_trace.h | 23 +- drivers/gpu/drm/xe/xe_vm.c | 692 ++++------- drivers/gpu/drm/xe/xe_vm.h | 4 + drivers/gpu/drm/xe/xe_vm_types.h | 43 +- 23 files changed, 1766 insertions(+), 1399 deletions(-) create mode 100644 drivers/gpu/drm/xe/xe_pt_exec_queue.c create mode 100644 drivers/gpu/drm/xe/xe_pt_exec_queue.h
drivers/gpu/drm/xe/Makefile +1 -0
@@ -121,6 +121,7 @@ xe-y += xe_bb.o \ xe_pm.o \ xe_preempt_fence.o \ xe_pt.o \ + xe_pt_exec_queue.o \ xe_pt_walk.o \ xe_query.o \ xe_range_fence.o \
drivers/gpu/drm/xe/xe_bo.c +3 -4
@@ -2256,16 +2256,16 @@ void __xe_bo_release_dummy(struct kref *kref)/** * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred(). +* @xe: Xe device * @deferred: The lockless list used for the call to xe_bo_put_deferred(). * * Puts all bos whose put was deferred by xe_bo_put_deferred(). * The @deferred list can be either an onstack local list or a global * shared list used by a workqueue. */ -void xe_bo_put_commit (struct llist_head *deferred) +void xe_bo_put_commit (struct xe_device *xe,struct llist_head *deferred) {struct llist_node *freed; -struct xe_bo *bo, *next;if (!deferred)return ;
@@ -2274,8 +2274,7 @@ void xe_bo_put_commit(struct llist_head *deferred)if (!freed)return ; -llist_for_each_entry_safe (bo, next, freed, freed) -drm_gem_object_free (&bo->ttm.base.refcount); +xe_device_put_deferred (xe, freed); }/**
drivers/gpu/drm/xe/xe_bo.h +2 -2
@@ -10,7 +10,6 @@#include "xe_bo_types.h" #include "xe_macros.h" -#include "xe_vm_types.h" #include "xe_vm.h" #define XE_DEFAULT_GTT_SIZE_MB 3072ULL/* 3GB by default */
@@ -282,10 +281,11 @@ xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred)if (!kref_put (&bo->ttm.base.refcount, __xe_bo_release_dummy))return false; +xe_vm_get (bo->vm);return llist_add (&bo->freed, deferred); } -void xe_bo_put_commit (struct llist_head *deferred); +void xe_bo_put_commit (struct xe_device *xe,struct llist_head *deferred);struct sg_table *xe_bo_sg (struct xe_bo *bo);
drivers/gpu/drm/xe/xe_device.c +35 -0
@@ -232,6 +232,9 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)if (xe->preempt_fence_wq)destroy_workqueue (xe->preempt_fence_wq); +flush_work (&xe->mem.deferred_work); +xe_assert (xe, !llist_del_all (&xe->mem.deferred)); +if (xe->ordered_wq)destroy_workqueue (xe->ordered_wq);
@@ -241,6 +244,35 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)ttm_device_fini (&xe->ttm); } +void xe_device_put_deferred (struct xe_device *xe,struct llist_node *deferred) +{ +struct xe_bo *bo, *next; + +llist_for_each_entry_safe (bo, next, deferred, freed) { +init_llist_node (&bo->freed); +llist_add (&bo->freed, &xe->mem.deferred); + } +queue_work (system_wq, &xe->mem.deferred_work); +} + +static void deferred_work (struct work_struct *w) +{ + struct xe_device *xe = container_of(w, struct xe_device, + mem.deferred_work); +struct llist_node *freed =llist_del_all (&xe->mem.deferred); +struct xe_bo *bo, *next; + +if (!freed) +return ; + +llist_for_each_entry_safe (bo, next, freed, freed) { +struct xe_vm *vm = bo->vm; + +drm_gem_object_free (&bo->ttm.base.refcount); +xe_vm_put (vm); + } +} +struct xe_device *xe_device_create (struct pci_dev *pdev,const struct pci_device_id *ent) {
@@ -314,6 +346,9 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, goto err; } +init_llist_head (&xe->mem.deferred); +INIT_WORK (&xe->mem.deferred_work, deferred_work); + err =xe_display_create (xe);if (WARN_ON (err)) goto err;
drivers/gpu/drm/xe/xe_device.h +2 -0
@@ -168,4 +168,6 @@ static inline bool xe_device_wedged(struct xe_device *xe)void xe_device_declare_wedged (struct xe_device *xe); +void xe_device_put_deferred (struct xe_device *xe,struct llist_node *deferred); +#endif
drivers/gpu/drm/xe/xe_device_types.h +4 -0
@@ -324,6 +324,10 @@ struct xe_device {struct xe_mem_region vram;/** @mem.sys_mgr: system TTM manager */ struct ttm_resource_manager sys_mgr; +/** @mem.deferred: deferred list to destroy PT entries */ +struct llist_head deferred; +/** @mem.deferred_work: worker to destroy PT entries */ +struct work_struct deferred_work; } mem;/** @sriov: device level virtualization data */
drivers/gpu/drm/xe/xe_exec.c +1 -1
@@ -134,7 +134,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)if (XE_IOCTL_DBG (xe, !q))return -ENOENT ; -if (XE_IOCTL_DBG (xe, q->flags &EXEC_QUEUE_FLAG_VM )) +if (XE_IOCTL_DBG (xe, q->flags &EXEC_QUEUE_FLAG_PT ))return -EINVAL ; if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
drivers/gpu/drm/xe/xe_exec_queue.c +21 -91
188 lines not on this page · raw message
drivers/gpu/drm/xe/xe_exec_queue_types.h +8 -12
@@ -19,6 +19,7 @@ struct xe_execlist_exec_queue;struct xe_gt ;struct xe_guc_exec_queue ;struct xe_hw_engine ; +struct xe_pt_exec_queue ;struct xe_vm ;enum xe_exec_queue_priority {
@@ -38,6 +39,8 @@ enum xe_exec_queue_priority {* a kernel object. */ struct xe_exec_queue { +/** @xe: Xe device */ +struct xe_device *xe;/** @gt: graphics tile this exec queue can submit to */ struct xe_gt *gt;/**
@@ -76,12 +79,10 @@ struct xe_exec_queue {#define EXEC_QUEUE_FLAG_KERNEL BIT(1)/* kernel engine only destroyed at driver unload */ #define EXEC_QUEUE_FLAG_PERMANENT BIT(2) -/* for VM jobs. Caller needs to hold rpm ref when creating queue with this flag */ -#define EXEC_QUEUE_FLAG_VM BIT(3) -/* child of VM queue for multi-tile VM jobs */ -#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(4) +/* for PT jobs. Caller needs to hold rpm ref when creating queue with this flag */ +#define EXEC_QUEUE_FLAG_PT BIT(3)/* kernel exec_queue only, set priority to highest level */ -#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(5) +#define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(4)/** * @flags: flags for this exec queue, should statically setup aside from ban
@@ -89,18 +90,13 @@ struct xe_exec_queue {*/ unsigned long flags; -union { -/** @multi_gt_list: list head for VM bind engines if multi-GT */ -struct list_head multi_gt_list; -/** @multi_gt_link: link for VM bind engines if multi-GT */ -struct list_head multi_gt_link; - }; -union {/** @execlist: execlist backend specific state for exec queue */ struct xe_execlist_exec_queue *execlist;/** @guc: GuC backend specific state for exec queue */ struct xe_guc_exec_queue *guc; +/** @pt: PT backend specific state for exec queue */ +struct xe_pt_exec_queue *pt; };/** @sched_props: scheduling properties */
drivers/gpu/drm/xe/xe_guc_submit.c +15 -9
@@ -1,4 +1,4 @@ -// SPDX-License-Identifier: MIT +// SPDX-License-Identifier: MIT /* * Copyright © 2022 Intel Corporation */
@@ -17,6 +17,7 @@#include "abi/guc_klvs_abi.h" #include "regs/xe_lrc_layout.h" #include "xe_assert.h" +#include "xe_bo.h" #include "xe_devcoredump.h" #include "xe_device.h" #include "xe_exec_queue.h"
@@ -727,6 +728,11 @@ static void submit_exec_queue(struct xe_exec_queue *q) } } +static bool is_pt_job (struct xe_sched_job *job) +{ +return test_bit (JOB_FLAG_PT , &job->fence->flags); +} +static struct dma_fence *guc_exec_queue_run_job (struct drm_sched_job *drm_job) {
@@ -736,6 +742,8 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job)struct xe_device *xe =guc_to_xe (guc);bool lr =xe_exec_queue_is_lr (q); +xe_assert (xe, !is_pt_job (job)); +xe_assert (xe, !(q->flags &EXEC_QUEUE_FLAG_PT ));xe_assert (xe, !(exec_queue_destroyed (q) ||exec_queue_pending_disable (q)) ||exec_queue_banned (q) ||exec_queue_suspended (q));
@@ -930,6 +938,8 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)int i =0 ;bool wedged; +xe_assert (xe, !(q->flags &EXEC_QUEUE_FLAG_PT )); +/* * TDR has fired before free job worker. Common if exec queue * immediately closed after last fence signaled.
@@ -945,8 +955,6 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) q->guc->id, q->flags);xe_gt_WARN (q->gt, q->flags &EXEC_QUEUE_FLAG_KERNEL ,"Kernel-submitted job timed out\n" ); -xe_gt_WARN (q->gt, q->flags &EXEC_QUEUE_FLAG_VM && !exec_queue_killed (q), -"VM job timed out on non-killed execqueue\n" );if (!exec_queue_killed (q))xe_devcoredump (job);
@@ -962,8 +970,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)* Kernel jobs should never fail, nor should VM jobs if they do * somethings has gone wrong and the GT needs a reset */ -if (!wedged && (q->flags &EXEC_QUEUE_FLAG_KERNEL || - (q->flags &EXEC_QUEUE_FLAG_VM && !exec_queue_killed (q)))) { +if (!wedged && q->flags &EXEC_QUEUE_FLAG_KERNEL ) {if (!xe_sched_invalidate_job (job,2 )) {xe_sched_add_pending_job (sched, job);xe_sched_submission_start (sched);
@@ -1448,11 +1455,10 @@ static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)trace_xe_exec_queue_stop (q);/* -* Ban any engine (aside from kernel and engines used for VM ops) with a -* started but not complete job or if a job has gone through a GT reset -* more than twice. +* Ban any engine (aside from kernel) with a started but not complete +* job or if a job has gone through a GT reset more than twice. */ -if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL |EXEC_QUEUE_FLAG_VM ))) { +if (!(q->flags &EXEC_QUEUE_FLAG_KERNEL )) {struct xe_sched_job *job =xe_sched_first_pending_job (sched);bool ban = false;
drivers/gpu/drm/xe/xe_migrate.c +110 -275
486 lines not on this page · raw message
drivers/gpu/drm/xe/xe_migrate.h +28 -18
85 lines not on this page · raw message
drivers/gpu/drm/xe/xe_pt.c +914 -445
1,582 lines not on this page · raw message
drivers/gpu/drm/xe/xe_pt.h +9 -8
@@ -17,6 +17,7 @@ struct xe_sync_entry;struct xe_tile ;struct xe_vm ;struct xe_vma ; +struct xe_vma_ops ;/* Largest huge pte is currently 1GiB. May become device dependent. */ #define MAX_HUGEPTE_LEVEL 2
@@ -34,15 +35,15 @@ void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,void xe_pt_destroy (struct xe_pt *pt,u32 flags,struct llist_head *deferred); -struct dma_fence * -__xe_pt_bind_vma (struct xe_tile *tile,struct xe_vma *vma,struct xe_exec_queue *q, -struct xe_sync_entry *syncs,u32 num_syncs, -bool rebind); - -struct dma_fence * -__xe_pt_unbind_vma (struct xe_tile *tile,struct xe_vma *vma,struct xe_exec_queue *q, -struct xe_sync_entry *syncs,u32 num_syncs); +int xe_pt_update_ops_prepare (struct xe_tile *tile,struct xe_vma_ops *vops); +struct dma_fence *xe_pt_update_ops_run (struct xe_tile *tile, +struct xe_vma_ops *vops); +void xe_pt_update_ops_abort (struct xe_tile *tile,struct xe_vma_ops *vops);bool xe_pt_zap_ptes (struct xe_tile *tile,struct xe_vma *vma); +struct xe_pt_job_ops *xe_pt_job_ops_alloc (struct xe_device *xe,u32 num_ops); +struct xe_pt_job_ops *xe_pt_job_ops_get (struct xe_pt_job_ops *pt_job_ops); +void xe_pt_job_ops_put (struct xe_pt_job_ops *pt_job_ops); +#endif
drivers/gpu/drm/xe/xe_pt_exec_queue.c new file +170 -0
170 lines not on this page · raw message
drivers/gpu/drm/xe/xe_pt_exec_queue.h new file +14 -0
@@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* +* Copyright © 2024 Intel Corporation +*/ + +#ifndef _XE_PT_EXEC_QUEUE_H_ +#define _XE_PT_EXEC_QUEUE_H_ + +struct xe_device ; +struct xe_exec_queue ; + +struct xe_exec_queue *xe_pt_exec_queue_create (struct xe_device *xe); + +#endif
drivers/gpu/drm/xe/xe_pt_types.h +67 -0
75 lines not on this page · raw message
drivers/gpu/drm/xe/xe_sched_job.c +64 -40
120 lines not on this page · raw message
drivers/gpu/drm/xe/xe_sched_job_types.h +27 -3
Lines of this file are not shown. View the raw message.
drivers/gpu/drm/xe/xe_vm.c +237 -455
692 lines not on this page · raw message
drivers/gpu/drm/xe/xe_vm_types.h +17 -26
43 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.34.1
Part of the diff is not shown here. View the raw message.