drm//amdgpu: Add job fence to resv conditionally
Subsystems (2)
PATCH 1/2 drm//amdgpu: Add job fence to resv conditionally
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 5 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 9 +++++++++ 2 files changed, 14 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5 -0
@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + +/** +* @resv: sync the resv and add job fence to it conditionally. +*/ +struct dma_resv *resv; };struct amdgpu_vm_update_funcs {
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +9 -0
@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; + p->resv = resv; +if (!resv)return 0 ;
@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,swap (p->vm->last_delayed, tmp);dma_fence_put (tmp); +/* add job fence to resv. +* MM notifier path is an exception as we can not grab the +* resv lock. +*/ +if (!p->direct && p->resv) +dma_resv_add_shared_fence (p->resv, f); +if (fence && !p->direct)swap (*fence, f);dma_fence_put (f);
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/2 drm/amdgpu: unref the bo after job submit
Otherwise we might free BOs before job has completed. We add the fence to BO after commit, so free BOs after that.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 29 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +19 -10
@@ -942,13 +942,17 @@ static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,* * @entry: PDE to free */ -static void amdgpu_vm_free_table (struct amdgpu_vm_pt *entry) +static void amdgpu_vm_free_table (struct amdgpu_vm_pt *entry, +struct list_head *head) {if (entry->base.bo) { entry->base.bo->vm_bo =NULL ;list_del (&entry->base.vm_status); -amdgpu_bo_unref (&entry->base.bo->shadow); -amdgpu_bo_unref (&entry->base.bo); +if (!head) { +amdgpu_bo_unref (&entry->base.bo->shadow); +amdgpu_bo_unref (&entry->base.bo); + }else +list_add (&entry->base.vm_status, head); }kvfree (entry->entries); entry->entries =NULL ;
@@ -965,7 +969,8 @@ static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)*/ static void amdgpu_vm_free_pts (struct amdgpu_device *adev,struct amdgpu_vm *vm, -struct amdgpu_vm_pt_cursor *start) +struct amdgpu_vm_pt_cursor *start, +struct list_head *head) {struct amdgpu_vm_pt_cursor cursor;struct amdgpu_vm_pt *entry;
@@ -973,10 +978,10 @@ static void amdgpu_vm_free_pts(struct amdgpu_device *adev, vm->bulk_moveable = false;for_each_amdgpu_vm_pt_dfs_safe (adev, vm, start, cursor, entry) -amdgpu_vm_free_table (entry); +amdgpu_vm_free_table (entry, head);if (start) -amdgpu_vm_free_table (start->entry); +amdgpu_vm_free_table (start->entry, head); }/**
@@ -1428,7 +1433,8 @@ static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params,*/ static int amdgpu_vm_update_ptes (struct amdgpu_vm_update_params *params,uint64_t start,uint64_t end, -uint64_t dst,uint64_t flags) +uint64_t dst,uint64_t flags, +struct list_head *head) {struct amdgpu_device *adev = params->adev;struct amdgpu_vm_pt_cursor cursor;
@@ -1539,7 +1545,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,* completely covered by the range and so potentially still in use. */ while (cursor.pfn < frag_start) { -amdgpu_vm_free_pts (adev, params->vm, &cursor); +amdgpu_vm_free_pts (adev, params->vm, &cursor, head);amdgpu_vm_pt_next (adev, &cursor); }
@@ -1582,6 +1588,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,struct amdgpu_vm_update_params params;enum amdgpu_sync_mode sync_mode;int r; +struct list_head head;memset (¶ms,0 ,sizeof (params)); params.adev = adev;
@@ -1589,6 +1596,8 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, params.direct = direct; params.pages_addr = pages_addr; +INIT_LIST_HEAD (&head); +/* Implicitly sync to command submissions in the same VM before * unmapping. Sync to moving fences before mapping. */
@@ -1617,7 +1626,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,if (r) goto error_unlock; - r =amdgpu_vm_update_ptes (¶ms, start, last +1 , addr, flags); + r =amdgpu_vm_update_ptes (¶ms, start, last +1 , addr, flags, &head);if (r) goto error_unlock;
@@ -3118,7 +3127,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)amdgpu_vm_free_mapping (adev, vm, mapping,NULL ); } -amdgpu_vm_free_pts (adev, vm,NULL ); +amdgpu_vm_free_pts (adev, vm,NULL ,NULL );amdgpu_bo_unreserve (root);amdgpu_bo_unref (&root);WARN_ON (vm->root.base.bo);
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
wait, this causes test case in Sl+ state. need take a look.
Quoted 61 lines
2020年3月13日 19:53,Pan, Xinhui <[email protected]> 写道:
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
Cc Christian König <[email protected]>Cc Alex Deucher <[email protected]>Cc Felix Kuehling <[email protected]>Suggested-by Christian König <[email protected]>Signed-off-by xinhui pan <[email protected]>---
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h| 5 +++++drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c| 9 +++++++++2 files changed, 14 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5
-0@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + +/** +* @resv: sync the resv and add job fence to it conditionally. +*/ +struct dma_resv *resv; };struct amdgpu_vm_update_funcs {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +9
-0@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; + p->resv = resv; +if (!resv)return 0 ;@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,swap (p->vm->last_delayed, tmp);dma_fence_put (tmp); +/* add job fence to resv. +* MM notifier path is an exception as we can not grab the +* resv lock. +*/ +if (!p->direct && p->resv) +dma_resv_add_shared_fence (p->resv, f); +if (fence && !p->direct)swap (*fence, f);dma_fence_put (f);Signature · 1 line
2.17.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Christian König
maintainerAm 13.03.20 um 12:53 schrieb xinhui pan:
Quoted 2 lines
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
That won't work because this is the wrong resv object :)
You added the fence to the mapped BO and not the page table.
No wonder that this doesn't work,
Christian.
Quoted 44 lines
Cc Christian König <[email protected]>Cc Alex Deucher <[email protected]>Cc Felix Kuehling <[email protected]>Suggested-by Christian König <[email protected]>Signed-off-by xinhui pan <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 5 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 9 +++++++++ 2 files changed, 14 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5
-0@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + +/** +* @resv: sync the resv and add job fence to it conditionally. +*/ +struct dma_resv *resv; };struct amdgpu_vm_update_funcs {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2
-0@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; + p->resv = resv; +if (!resv)return 0 ;@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,swap (p->vm->last_delayed, tmp);dma_fence_put (tmp);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, · line 125
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +7
-0+/* add job fence to resv. +* MM notifier path is an exception as we can not grab the +* resv lock. +*/ +if (!p->direct && p->resv) +dma_resv_add_shared_fence (p->resv, f); +if (fence && !p->direct)swap (*fence, f);dma_fence_put (f);
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
page table BOs share same resv.It should be ok using any of them, root bo resv or bo resv. I forgot to unref bos which cause problems. not good at rebasing...
________________________________
From: Koenig, Christian <[email protected]>
Sent: Friday, March 13, 2020 9:34:42 PM
To: Pan, Xinhui <[email protected]>; [email protected] <[email protected]>
Cc: Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>
Subject: Re: [PATCH 1/2] drm//amdgpu: Add job fence to resv conditionally
Am 13.03.20 um 12:53 schrieb xinhui pan:
Quoted 2 lines
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
That won't work because this is the wrong resv object :)
You added the fence to the mapped BO and not the page table.
No wonder that this doesn't work,
Christian.
Quoted 44 lines
Cc Christian König <[email protected]>Cc Alex Deucher <[email protected]>Cc Felix Kuehling <[email protected]>Suggested-by Christian König <[email protected]>Signed-off-by xinhui pan <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 5 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 9 +++++++++ 2 files changed, 14 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5
-0@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + +/** +* @resv: sync the resv and add job fence to it conditionally. +*/ +struct dma_resv *resv; };struct amdgpu_vm_update_funcs {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2
-0@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; + p->resv = resv; +if (!resv)return 0 ;@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,swap (p->vm->last_delayed, tmp);dma_fence_put (tmp);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, · line 125
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +7
-0+/* add job fence to resv. +* MM notifier path is an exception as we can not grab the +* resv lock. +*/ +if (!p->direct && p->resv) +dma_resv_add_shared_fence (p->resv, f); +if (fence && !p->direct)swap (*fence, f);dma_fence_put (f);
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Christian König
maintainerYeah, but this is still the wrong resv object :)
See the object passed to amdgpu_vm_sdma_prepare() is the one of the BO which is mapped into the page tables and NOT the one of the page tables.
You need to use p->vm->root.base.bo->tbo.base.resv here.
Regards,
Christian.
Am 13.03.20 um 14:43 schrieb Pan, Xinhui:
Quoted 63 lines
[AMD Official Use Only - Internal Distribution Only]
page table BOs share same resv.It should be ok using any of them, root bo resv or bo resv. I forgot to unref bos which cause problems. not good at rebasing...
------------------------------------------------------------------------
*From:* Koenig, Christian <[email protected]>
*Sent:* Friday, March 13, 2020 9:34:42 PM
*To:* Pan, Xinhui <[email protected]>; [email protected] <[email protected]> *Cc:* Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]> *Subject:* Re: [PATCH 1/2] drm//amdgpu: Add job fence to resv conditionally
Am 13.03.20 um 12:53 schrieb xinhui pan:Quoted 2 lines
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
That won't work because this is the wrong resv object :)
You added the fence to the mapped BO and not the page table.
No wonder that this doesn't work,
Christian.Quoted 6 lines
Cc Christian König <[email protected]>Cc Alex Deucher <[email protected]>Cc Felix Kuehling <[email protected]>Suggested-by Christian König <[email protected]>Signed-off-by xinhui pan <[email protected]>Review
Quoted 6 lines
---
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 5 +++++drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c| 9 +++++++++2 files changed, 14 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.hReview
Quoted 16 lines
index
b5705fcfc935..ca6021b4200b100644drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5
-0@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + + /** + * @resv: sync the resv and add job fence to it conditionally. + */ + struct dma_resv *resv; }; struct amdgpu_vm_update_funcs {(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.cdrivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c static int amdgpu_vm_sdma_prepare(struct
Quoted 4 lines
index
4cc7881f438c..0cfac59bff36100644drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +0
-0@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct
amdgpu_vm_update_params*p,Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +0
-0p->num_dw_left = ndw;
Quoted 21 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c static int amdgpu_vm_sdma_commit(struct
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2
-0+ p->resv = resv; + if (!resv) return 0;@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct
amdgpu_vm_update_params*p,Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +7
-0swap(p->vm->last_delayed, tmp); dma_fence_put(tmp); + /* add job fence to resv. + * MM notifier path is an exception as we can not grab the + * resv lock. + */ + if (!p->direct && p->resv) + dma_resv_add_shared_fence(p->resv, f); + if (fence && !p->direct) swap(*fence, f); dma_fence_put(f);
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Yep, will send v3.
Thanks
xinhui
发件人: "Koenig, Christian" <[email protected]>
日期: 2020年3月13日 星期五 21:46
收件人: "Pan, Xinhui" <[email protected]>, "[email protected]" <[email protected]> 抄送: "Deucher, Alexander" <[email protected]>, "Kuehling, Felix" <[email protected]> 主题: Re: [PATCH 1/2] drm//amdgpu: Add job fence to resv conditionally
Yeah, but this is still the wrong resv object :)
See the object passed to amdgpu_vm_sdma_prepare() is the one of the BO which is mapped into the page tables and NOT the one of the page tables.
You need to use p->vm->root.base.bo->tbo.base.resv here.
Regards,
Christian.
Am 13.03.20 um 14:43 schrieb Pan, Xinhui:
[AMD Official Use Only - Internal Distribution Only]
page table BOs share same resv.It should be ok using any of them, root bo resv or bo resv. I forgot to unref bos which cause problems. not good at rebasing...
________________________________
From: Koenig, Christian <[email protected]><mailto:[email protected]>
Sent: Friday, March 13, 2020 9:34:42 PM
To: Pan, Xinhui <[email protected]><mailto:[email protected]>; [email protected]<mailto:[email protected]> <[email protected]><mailto:[email protected]>
Cc: Deucher, Alexander <[email protected]><mailto:[email protected]>; Kuehling, Felix <[email protected]><mailto:[email protected]>
Subject: Re: [PATCH 1/2] drm//amdgpu: Add job fence to resv conditionally
Am 13.03.20 um 12:53 schrieb xinhui pan:
Quoted 2 lines
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
That won't work because this is the wrong resv object :)
You added the fence to the mapped BO and not the page table.
No wonder that this doesn't work,
Christian.
Quoted 44 lines
Cc Christian König <[email protected]><mailto:[email protected]>Cc Alex Deucher <[email protected]><mailto:[email protected]>Cc Felix Kuehling <[email protected]><mailto:[email protected]>Signed-off-by xinhui pan <[email protected]><mailto:[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 5 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 9 +++++++++ 2 files changed, 14 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5
-0@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + +/** +* @resv: sync the resv and add job fence to it conditionally. +*/ +struct dma_resv *resv; };struct amdgpu_vm_update_funcs {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2
-0@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; + p->resv = resv; +if (!resv)return 0 ;@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,swap (p->vm->last_delayed, tmp);dma_fence_put (tmp);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, · line 125
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +7
-0+/* add job fence to resv. +* MM notifier path is an exception as we can not grab the +* resv lock. +*/ +if (!p->direct && p->resv) +dma_resv_add_shared_fence (p->resv, f); +if (fence && !p->direct)swap (*fence, f);dma_fence_put (f);
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Official Use Only - Internal Distribution Only]
page table BOs share same resv.It should be ok using any of them, root bo resv or bo resv. I forgot to unref bos which cause problems. not good at rebasing...
thanks
xinhui
________________________________
From: Koenig, Christian <[email protected]>
Sent: Friday, March 13, 2020 9:34:42 PM
To: Pan, Xinhui <[email protected]>; [email protected] <[email protected]>
Cc: Deucher, Alexander <[email protected]>; Kuehling, Felix <[email protected]>
Subject: Re: [PATCH 1/2] drm//amdgpu: Add job fence to resv conditionally
Am 13.03.20 um 12:53 schrieb xinhui pan:
Quoted 2 lines
If a job need sync the bo resv, it is likely that bo need the job fence to sync with others.
That won't work because this is the wrong resv object :)
You added the fence to the mapped BO and not the page table.
No wonder that this doesn't work,
Christian.
Quoted 44 lines
Cc Christian König <[email protected]>Cc Alex Deucher <[email protected]>Cc Felix Kuehling <[email protected]>Suggested-by Christian König <[email protected]>Signed-off-by xinhui pan <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 5 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 9 +++++++++ 2 files changed, 14 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5
-0@@ -226,6 +226,11 @@ struct amdgpu_vm_update_params {* @num_dw_left: number of dw left for the IB */ unsigned int num_dw_left; + +/** +* @resv: sync the resv and add job fence to it conditionally. +*/ +struct dma_resv *resv; };struct amdgpu_vm_update_funcs {drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +2
-0@@ -70,6 +70,8 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; + p->resv = resv; +if (!resv)return 0 ;@@ -111,6 +113,13 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,swap (p->vm->last_delayed, tmp);dma_fence_put (tmp);
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, · line 125
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +7
-0+/* add job fence to resv. +* MM notifier path is an exception as we can not grab the +* resv lock. +*/ +if (!p->direct && p->resv) +dma_resv_add_shared_fence (p->resv, f); +if (fence && !p->direct)swap (*fence, f);dma_fence_put (f);
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx