amdgpu: Allow explicitly synchronized submissions.
Subsystems (4)
v2 cover amdgpu: Allow explicitly synchronized submissions.
This adds a context option to use DMA_RESV_USAGE_BOOKKEEP for userspace submissions. This is a respin of [1] but rebased on top of the newly introduced drm_exec helpers.
Disabling implicit sync is something we've wanted in RADV for a while for resolving some corner cases. A more immediate thing that would be solved here is avoiding a bunch of implicit sync on GPU map/unmap operations as well, which helps with stutter around sparse maps/unmaps.
This has seen a significant improvement in stutter in Forza Horizon 5 and Forza Horizon 4. (As games that had significant issues in sparse binding related stutter). I've been able to pass a full Vulkan CTS run on Navi 10 with this.
(Note that the drm_exec patchset has introduced new CTS failures with duplicate BOs, but we will be tracking that separately.)
Userspace code for this is available at [2] and a branch for the kernel code is available at [3].
[1]: https://patchwork.freedesktop.org/series/107233/
[2]: https://gitlab.freedesktop.org/ishitatsuyuki/mesa/-/commits/single-sparse-queue-explicit-rebase
[3]: https://github.com/ishitatsuyuki/linux/tree/explicit-sync-drm-misc-next
Bas Nieuwenhuizen (4):
drm/amdgpu: Add separate mode for syncing DMA_RESV_USAGE_BOOKKEEP. drm/amdgpu: Allow explicit sync for VM ops.
drm/amdgpu: Add option to disable implicit sync for a context. drm/amdgpu: Bump amdgpu driver version.
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 18 +++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 30 +++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 ++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 11 +++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h | 4 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 3 +- include/uapi/drm/amdgpu_drm.h | 4 +++ 12 files changed, 74 insertions(+), 19 deletions(-)
Signature · 1 line
2.41.0
PATCH v2 1/4 drm/amdgpu: Add separate mode for syncing DMA_RESV_USAGE_BOOKKEEP.
From: Bas Nieuwenhuizen <[email protected]>
To prep for allowing different sync modes in a follow-up patch.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 +++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 11 ++++++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 2 +- 8 files changed, 23 insertions(+), 14 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +1 -1
@@ -1287,7 +1287,7 @@ static int process_sync_pds_resv(struct amdkfd_process_info *process_info,struct amdgpu_bo *pd = peer_vm->root.bo; ret =amdgpu_sync_resv (NULL , sync, pd->tbo.base.resv, -AMDGPU_SYNC_NE_OWNER , +AMDGPU_SYNC_NE_OWNER ,AMDGPU_SYNC_NE_OWNER ,AMDGPU_FENCE_OWNER_KFD );if (ret)return ret;
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +1 -1
@@ -1203,7 +1203,7 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) sync_mode =amdgpu_bo_explicit_sync (bo) ?AMDGPU_SYNC_EXPLICIT :AMDGPU_SYNC_NE_OWNER ; r =amdgpu_sync_resv (p->adev, &p->sync, resv, sync_mode, - &fpriv->vm); +AMDGPU_SYNC_EXPLICIT , &fpriv->vm);if (r)return r; }
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +7 -4
@@ -1450,7 +1450,8 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,* * @adev: amdgpu device pointer * @resv: reservation object to sync to -* @sync_mode: synchronization mode +* @implicit_sync_mode: synchronization mode for usage <= DMA_RESV_USAGE_READ +* @explicit_sync_mode: synchronization mode for usage DMA_RESV_USAGE_BOOKKEEP * @owner: fence owner * @intr: Whether the wait is interruptible *
@@ -1460,14 +1461,15 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,* 0 on success, errno otherwise. */ int amdgpu_bo_sync_wait_resv (struct amdgpu_device *adev,struct dma_resv *resv, -enum amdgpu_sync_mode sync_mode,void *owner, +enum amdgpu_sync_mode implicit_sync_mode, +enum amdgpu_sync_mode explicit_sync_mode,void *owner,bool intr) {struct amdgpu_sync sync;int r;amdgpu_sync_create (&sync); -amdgpu_sync_resv (adev, &sync, resv, sync_mode, owner); +amdgpu_sync_resv (adev, &sync, resv, implicit_sync_mode, explicit_sync_mode, owner); r =amdgpu_sync_wait (&sync, intr);amdgpu_sync_free (&sync);return r;
@@ -1488,7 +1490,8 @@ int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)struct amdgpu_device *adev =amdgpu_ttm_adev (bo->tbo.bdev);return amdgpu_bo_sync_wait_resv (adev, bo->tbo.base.resv, -AMDGPU_SYNC_NE_OWNER , owner, intr); +AMDGPU_SYNC_NE_OWNER ,AMDGPU_SYNC_EXPLICIT , + owner, intr); }/**
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +2 -1
@@ -350,7 +350,8 @@ vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo);void amdgpu_bo_fence (struct amdgpu_bo *bo,struct dma_fence *fence,bool shared);int amdgpu_bo_sync_wait_resv (struct amdgpu_device *adev,struct dma_resv *resv, -enum amdgpu_sync_mode sync_mode,void *owner, +enum amdgpu_sync_mode implicit_sync_mode, +enum amdgpu_sync_mode explicit_sync_mode,void *owner,bool intr);int amdgpu_bo_sync_wait (struct amdgpu_bo *bo,void *owner,bool intr);u64 amdgpu_bo_gpu_offset (struct amdgpu_bo *bo);
drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +8 -3
@@ -226,14 +226,15 @@ static bool amdgpu_sync_test_fence(struct amdgpu_device *adev,* @adev: amdgpu device * @sync: sync object to add fences from reservation object to * @resv: reservation object with embedded fence -* @mode: how owner affects which fences we sync to +* @implicit_mode: how owner affects which fences with usage <= DMA_RESV_USAGE_READ we sync to +* @explicit_mode: how owner affects which fences with usage DMA_RESV_USAGE_BOOKKEEP we sync to * @owner: owner of the planned job submission * * Sync to the fence */ int amdgpu_sync_resv (struct amdgpu_device *adev,struct amdgpu_sync *sync, -struct dma_resv *resv,enum amdgpu_sync_mode mode, -void *owner) +struct dma_resv *resv,enum amdgpu_sync_mode implicit_mode, +enum amdgpu_sync_mode explicit_mode,void *owner) {struct dma_resv_iter cursor;struct dma_fence *f;
@@ -246,6 +247,10 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,dma_resv_for_each_fence (&cursor, resv,DMA_RESV_USAGE_BOOKKEEP , f) {dma_fence_chain_for_each (f, f) {struct dma_fence *tmp =dma_fence_chain_contained (f); +enum amdgpu_sync_mode mode = implicit_mode; + +if (dma_resv_iter_usage (&cursor) >=DMA_RESV_USAGE_BOOKKEEP ) + mode = explicit_mode;if (amdgpu_sync_test_fence (adev, mode, owner, tmp)) { r =amdgpu_sync_fence (sync, f);
drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h +2 -2
@@ -49,8 +49,8 @@ struct amdgpu_sync {void amdgpu_sync_create (struct amdgpu_sync *sync);int amdgpu_sync_fence (struct amdgpu_sync *sync,struct dma_fence *f);int amdgpu_sync_resv (struct amdgpu_device *adev,struct amdgpu_sync *sync, -struct dma_resv *resv,enum amdgpu_sync_mode mode, -void *owner); +struct dma_resv *resv,enum amdgpu_sync_mode implicit_mode, +enum amdgpu_sync_mode explicit_mode,void *owner);struct dma_fence *amdgpu_sync_peek_fence (struct amdgpu_sync *sync,struct amdgpu_ring *ring);struct dma_fence *amdgpu_sync_get_fence (struct amdgpu_sync *sync);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c +1 -1
@@ -51,7 +51,7 @@ static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p,if (!resv)return 0 ; -return amdgpu_bo_sync_wait_resv (p->adev, resv, sync_mode, p->vm, true); +return amdgpu_bo_sync_wait_resv (p->adev, resv, sync_mode, sync_mode, p->vm, true); }/**
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +1 -1
@@ -98,7 +98,7 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p,return 0 ;amdgpu_sync_create (&sync); - r =amdgpu_sync_resv (p->adev, &sync, resv, sync_mode, p->vm); + r =amdgpu_sync_resv (p->adev, &sync, resv, sync_mode, sync_mode, p->vm);if (!r) r =amdgpu_sync_push_to_job (&sync, p->job);amdgpu_sync_free (&sync);
Signature · 1 line
2.41.0
PATCH v2 2/4 drm/amdgpu: Allow explicit sync for VM ops.
In reply to Tatsuyuki IshiFrom: Bas Nieuwenhuizen <[email protected]>
This should be okay because moves themselves use KERNEL usage and hence still sync with BOOKKEEP usage. Then any later submits still wait on any pending VM operations.
(i.e. we only made VM ops not wait on BOOKKEEP submits, not the other
way around)
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c +2 -1
@@ -51,7 +51,8 @@ static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p,if (!resv)return 0 ; -return amdgpu_bo_sync_wait_resv (p->adev, resv, sync_mode, sync_mode, p->vm, true); +return amdgpu_bo_sync_wait_resv (p->adev, resv, sync_mode, +AMDGPU_SYNC_EXPLICIT , p->vm, true); }/**
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2 -1
@@ -98,7 +98,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p,return 0 ;amdgpu_sync_create (&sync); - r =amdgpu_sync_resv (p->adev, &sync, resv, sync_mode, sync_mode, p->vm); + r =amdgpu_sync_resv (p->adev, &sync, resv, sync_mode, +AMDGPU_SYNC_EXPLICIT , p->vm);if (!r) r =amdgpu_sync_push_to_job (&sync, p->job);amdgpu_sync_free (&sync);
Signature · 1 line
2.41.0
Christian König
maintainerRe: [PATCH v2 2/4] drm/amdgpu: Allow explicit sync for VM ops.
Am 21.08.23 um 08:20 schrieb Tatsuyuki Ishi:
Quoted 8 lines
From: Bas Nieuwenhuizen <[email protected]>
This should be okay because moves themselves use KERNEL usage and hence still sync with BOOKKEEP usage. Then any later submits still wait on any pending VM operations.
(i.e. we only made VM ops not wait on BOOKKEEP submits, not the other
way around)
Well that approach won't work like this.
Basically the whole approach with the implicit vs. explicit handling sounds like a really bad idea to me.
What you need to do instead is to give amdgpu_vm_sdma_prepare() and amdgpu_vm_cpu_prepare() the information if this is an update triggered by the kernel (because the BO was evicted) or triggered by userspace.
Take a look where the sync_mode parameter is determined instead.
Regards,
Christian.
Quoted 36 lines
Signed-off-by Bas Nieuwenhuizen <[email protected]>Signed-off-by Tatsuyuki Ishi <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-)drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c +2
-1@@ -51,7 +51,8 @@ static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p,if (!resv)return 0 ; -return amdgpu_bo_sync_wait_resv (p->adev, resv, sync_mode, sync_mode, p->vm, true); +return amdgpu_bo_sync_wait_resv (p->adev, resv, sync_mode, +AMDGPU_SYNC_EXPLICIT , p->vm, true); }/** drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2
-1@@ -98,7 +98,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p,return 0 ;amdgpu_sync_create (&sync); - r =amdgpu_sync_resv (p->adev, &sync, resv, sync_mode, sync_mode, p->vm); + r =amdgpu_sync_resv (p->adev, &sync, resv, sync_mode, +AMDGPU_SYNC_EXPLICIT , p->vm);if (!r) r =amdgpu_sync_push_to_job (&sync, p->job);amdgpu_sync_free (&sync);
PATCH v2 3/4 drm/amdgpu: Add option to disable implicit sync for a context.
In reply to Tatsuyuki IshiFrom: Bas Nieuwenhuizen <[email protected]>
This changes all BO usages in a submit to BOOKKEEP instead of READ, which effectively disables implicit sync for these submits.
This is configured at a context level using the existing IOCTL.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 16 +++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 30 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 1 + include/uapi/drm/amdgpu_drm.h | 4 ++++ 4 files changed, 47 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +12 -4
@@ -1200,8 +1200,11 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)struct dma_resv *resv = bo->tbo.base.resv;enum amdgpu_sync_mode sync_mode; - sync_mode =amdgpu_bo_explicit_sync (bo) ? -AMDGPU_SYNC_EXPLICIT :AMDGPU_SYNC_NE_OWNER ; +if (amdgpu_bo_explicit_sync (bo) || p->ctx->disable_implicit_sync) + sync_mode =AMDGPU_SYNC_EXPLICIT ; +else + sync_mode =AMDGPU_SYNC_NE_OWNER ; + r =amdgpu_sync_resv (p->adev, &p->sync, resv, sync_mode,AMDGPU_SYNC_EXPLICIT , &fpriv->vm);if (r)
@@ -1322,11 +1325,16 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,dma_resv_add_fence (gobj->resv, &p->jobs[i]->base.s_fence->finished, -DMA_RESV_USAGE_READ ); + p->ctx->disable_implicit_sync ? +DMA_RESV_USAGE_BOOKKEEP : +DMA_RESV_USAGE_READ ); }/* The gang leader as remembered as writer */ -dma_resv_add_fence (gobj->resv, p->fence,DMA_RESV_USAGE_WRITE ); +dma_resv_add_fence (gobj->resv, p->fence, + p->ctx->disable_implicit_sync ? +DMA_RESV_USAGE_BOOKKEEP : +DMA_RESV_USAGE_WRITE ); } seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx],
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +30 -0
@@ -661,6 +661,30 @@ static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,return r; } +static int amdgpu_ctx_set_implicit_sync (struct amdgpu_device *adev, +struct amdgpu_fpriv *fpriv,uint32_t id, +bool enable) +{ +struct amdgpu_ctx *ctx; +struct amdgpu_ctx_mgr *mgr; + +if (!fpriv) +return -EINVAL ; + + mgr = &fpriv->ctx_mgr; +mutex_lock (&mgr->lock); + ctx =idr_find (&mgr->ctx_handles, id); +if (!ctx) { +mutex_unlock (&mgr->lock); +return -EINVAL ; + } + + ctx->disable_implicit_sync = !enable; + +mutex_unlock (&mgr->lock); +return 0 ; +} +int amdgpu_ctx_ioctl (struct drm_device *dev,void *data,struct drm_file *filp) {
@@ -709,6 +733,12 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,return -EINVAL ; r =amdgpu_ctx_stable_pstate (adev, fpriv, id, true, &stable_pstate);break ; +case AMDGPU_CTX_OP_SET_IMPLICIT_SYNC : +if ((args->in.flags & ~AMDGPU_CTX_IMPLICIT_SYNC_ENABLED ) || args->in.priority) +return -EINVAL ; + r =amdgpu_ctx_set_implicit_sync (adev, fpriv, id, + args->in.flags & ~AMDGPU_CTX_IMPLICIT_SYNC_ENABLED ); +break ;default :return -EINVAL ; }
drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h +1 -0
@@ -58,6 +58,7 @@ struct amdgpu_ctx {unsigned long ras_counter_ue;uint32_t stable_pstate;struct amdgpu_ctx_mgr *ctx_mgr; +bool disable_implicit_sync; };struct amdgpu_ctx_mgr {
include/uapi/drm/amdgpu_drm.h +4 -0
@@ -226,6 +226,7 @@ union drm_amdgpu_bo_list {#define AMDGPU_CTX_OP_QUERY_STATE2 4#define AMDGPU_CTX_OP_GET_STABLE_PSTATE 5#define AMDGPU_CTX_OP_SET_STABLE_PSTATE 6 +#define AMDGPU_CTX_OP_SET_IMPLICIT_SYNC 7/* GPU reset status */ #define AMDGPU_CTX_NO_RESET 0
@@ -268,6 +269,9 @@ union drm_amdgpu_bo_list {#define AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK 3#define AMDGPU_CTX_STABLE_PSTATE_PEAK 4 +/* opt-out of implicit sync */ +#define AMDGPU_CTX_IMPLICIT_SYNC_ENABLED 1 +struct drm_amdgpu_ctx_in {/** AMDGPU_CTX_OP_* */ __u32 op;
Signature · 1 line
2.41.0
PATCH v2 4/4 drm/amdgpu: Bump amdgpu driver version.
In reply to Tatsuyuki IshiFrom: Bas Nieuwenhuizen <[email protected]>
For detection of the new explicit sync functionality without having to try the ioctl.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +2 -1
@@ -113,9 +113,10 @@* gl1c_cache_size, gl2c_cache_size, mall_size, enabled_rb_pipes_mask_hi * 3.53.0 - Support for GFX11 CP GFX shadowing * 3.54.0 - Add AMDGPU_CTX_QUERY2_FLAGS_RESET_IN_PROGRESS support +* - 3.55.0 - Add AMDGPU_CTX_OP_SET_IMPLICIT_SYNC context operation. */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 54 +#define KMS_DRIVER_MINOR 55#define KMS_DRIVER_PATCHLEVEL 0unsigned int amdgpu_vram_limit =UINT_MAX ;
Signature · 1 line
2.41.0