drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2
Subsystems (4)
PATCH 1/1 drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This partial reverts commit ff20caa0bcbfef9f7686f8d1868a3b990921afd6.
v2: remove unnecessary check and unused variable
--- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 + drivers/gpu/drm/ttm/ttm_tt.c | 19 ++++++++++++------- include/drm/ttm/ttm_bo_driver.h | 9 +++++++++ 3 files changed, 22 insertions(+), 7 deletions(-)
drivers/gpu/drm/ttm/ttm_bo_util.c +1 -0
@@ -53,6 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,int ret;if (old_mem->mem_type !=TTM_PL_SYSTEM ) { +ttm_tt_unbind (ttm);ttm_bo_free_old_node (bo);ttm_flag_masked (&old_mem->placement,TTM_PL_FLAG_SYSTEM ,TTM_PL_MASK_MEM );
drivers/gpu/drm/ttm/ttm_tt.c +12 -7
@@ -166,16 +166,10 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);void ttm_tt_destroy (struct ttm_tt *ttm) { -int ret; -if (ttm ==NULL )return ; -if (ttm->state == tt_bound) { - ret = ttm->func->unbind (ttm); -BUG_ON (ret); - ttm->state = tt_unbound; - } +ttm_tt_unbind (ttm);if (ttm->state == tt_unbound)ttm_tt_unpopulate (ttm);
@@ -255,6 +249,17 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) }EXPORT_SYMBOL (ttm_dma_tt_fini); +void ttm_tt_unbind (struct ttm_tt *ttm) +{ +int ret; + +if (ttm->state == tt_bound) { + ret = ttm->func->unbind (ttm); +BUG_ON (ret); + ttm->state = tt_unbound; + } +} +int ttm_tt_bind (struct ttm_tt *ttm,struct ttm_mem_reg *bo_mem) {int ret =0 ;
include/drm/ttm/ttm_bo_driver.h +9 -0
@@ -628,6 +628,15 @@ extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem);extern void ttm_tt_destroy (struct ttm_tt *ttm);/** +* ttm_ttm_unbind: +* +* @ttm: The struct ttm_tt. +* +* Unbind a struct ttm_tt. +*/ +extern void ttm_tt_unbind (struct ttm_tt *ttm); + +/** * ttm_tt_swapin: * * @ttm: The struct ttm_tt.
Signature · 6 lines
2.5.0 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 1469193013-11482-1-git-send-email-deathsimple-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Edward O'Callaghan
Reviewed-byQuoted 78 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This partial reverts commit
ff20caa0bcbfef9f7686f8d1868a3b990921afd6.v2: remove unnecessary check and unused variable
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 + drivers/gpu/drm/ttm/ttm_tt.c | 19 ++++++++++++------- include/drm/ttm/ttm_bo_driver.h | 9 +++++++++ 3 files changed, 22 insertions(+), 7 deletions(-)drivers/gpu/drm/ttm/ttm_bo_util.c +1
-0@@ -53,6 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,int ret;if (old_mem->mem_type !=TTM_PL_SYSTEM ) { +ttm_tt_unbind (ttm);ttm_bo_free_old_node (bo);ttm_flag_masked (&old_mem->placement,TTM_PL_FLAG_SYSTEM ,TTM_PL_MASK_MEM );drivers/gpu/drm/ttm/ttm_tt.c +12
-7@@ -166,16 +166,10 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);void ttm_tt_destroy (struct ttm_tt *ttm) { -int ret; -if (ttm ==NULL )return ; -if (ttm->state == tt_bound) { - ret = ttm->func->unbind (ttm); -BUG_ON (ret); - ttm->state = tt_unbound; - } +ttm_tt_unbind (ttm);if (ttm->state == tt_unbound)ttm_tt_unpopulate (ttm);@@ -255,6 +249,17 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) }EXPORT_SYMBOL (ttm_dma_tt_fini); +void ttm_tt_unbind (struct ttm_tt *ttm) +{ +int ret; + +if (ttm->state == tt_bound) { + ret = ttm->func->unbind (ttm); +BUG_ON (ret); + ttm->state = tt_unbound; + } +} +int ttm_tt_bind (struct ttm_tt *ttm,struct ttm_mem_reg *bo_mem) {int ret =0 ;include/drm/ttm/ttm_bo_driver.h +2
-0@@ -628,6 +628,15 @@ extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem);extern void ttm_tt_destroy (struct ttm_tt *ttm);/** +* ttm_ttm_unbind: +*
include/drm/ttm/ttm_bo_driver.h extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem); · line 642
Quoted 10 lines
include/drm/ttm/ttm_bo_driver.h +7
-0+* @ttm: The struct ttm_tt. +* +* Unbind a struct ttm_tt. +*/ +extern void ttm_tt_unbind (struct ttm_tt *ttm); + +/** * ttm_tt_swapin: * * @ttm: The struct ttm_tt.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 3 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
[TTM] Buffer eviction failed
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what the problem could be? If not, I'll try bisecting.
Signature · 6 lines
Earthling Michel Dänzer | http://www.amd.com Libre software enthusiast | Mesa and X developer _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 95d462b5-7c20-2f40-c406-48f5aece0817-otUistvHUpPR7s880joybQ@public.gmane.org
We're also looking into a hang with a KFD unit test that allocates lots of memory and fragments it deliberately, without mapping it all at once. It's a new problem for us as we're rebasing on amd-staging-4.6. Something weird seems to be happening with evictions, but I haven't been able to figure it out.
I was able to see that SDMA page table updates stop working at some point, though SDMA fences are still signaling. If I let the test run longer, SDMA and CP hang. I dumped the SDMA IBs and didn't see anything suspicious. My guess was that maybe the SDMA IBs or the ring are getting corrupted, or maybe the GART table entries for the IBs or ring are corrupted. But I haven't been able to prove that or track it down to a root cause. We're now trying to reimplement the test using libdrm-amdgpu APIs so we can bisect on the amd-staging-4.6 branch without KFD.
Regards,
Felix
Quoted 12 lines · Michel Dänzer
Quoted 3 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
Quoted 8 lines
[TTM] Buffer eviction failed
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what the problem could be? If not, I'll try bisecting.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 58e8df15-98dd-facc-acb6-68f94a937374-5C7GfCeVMHo@public.gmane.org
For the record, Michel's patch "drm/ttm: Wait for a BO to become idle before unbinding it from GTT" fixes our KFD problem as well.
Thanks,
Felix
Quoted 15 lines · Felix Kuehling
We're also looking into a hang with a KFD unit test that allocates lots of memory and fragments it deliberately, without mapping it all at once. It's a new problem for us as we're rebasing on amd-staging-4.6. Something weird seems to be happening with evictions, but I haven't been able to figure it out.
I was able to see that SDMA page table updates stop working at some point, though SDMA fences are still signaling. If I let the test run longer, SDMA and CP hang. I dumped the SDMA IBs and didn't see anything suspicious. My guess was that maybe the SDMA IBs or the ring are getting corrupted, or maybe the GART table entries for the IBs or ring are corrupted. But I haven't been able to prove that or track it down to a root cause. We're now trying to reimplement the test using libdrm-amdgpu APIs so we can bisect on the amd-staging-4.6 branch without KFD.
Quoted 24 lines
Regards,
FelixQuoted 20 lines · Michel Dänzer
Quoted 3 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
[TTM] Buffer eviction failed
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what the problem could be? If not, I'll try bisecting.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 67b3c224-29e0-0a98-6c3a-8439744149da-5C7GfCeVMHo@public.gmane.org
Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
We had to revert another change on the KFD branch to fix a buffer move
problem: 8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa (drm/amdgpu: group BOs by log2 of the size on the LRU v2
We haven't looked into this change in detail yet, to understand the cause. Kent found it by bisecting on amd-staging-4.6 and applying KFD changes on top.
Regards,
Felix
Quoted 46 lines · Felix Kuehling
For the record, Michel's patch "drm/ttm: Wait for a BO to become idle before unbinding it from GTT" fixes our KFD problem as well.
Thanks,
FelixQuoted 39 lines · Felix Kuehling
We're also looking into a hang with a KFD unit test that allocates lots of memory and fragments it deliberately, without mapping it all at once. It's a new problem for us as we're rebasing on amd-staging-4.6. Something weird seems to be happening with evictions, but I haven't been able to figure it out.
I was able to see that SDMA page table updates stop working at some point, though SDMA fences are still signaling. If I let the test run longer, SDMA and CP hang. I dumped the SDMA IBs and didn't see anything suspicious. My guess was that maybe the SDMA IBs or the ring are getting corrupted, or maybe the GART table entries for the IBs or ring are corrupted. But I haven't been able to prove that or track it down to a root cause. We're now trying to reimplement the test using libdrm-amdgpu APIs so we can bisect on the amd-staging-4.6 branch without KFD.
Regards,
FelixQuoted 20 lines · Michel Dänzer
Quoted 3 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
[TTM] Buffer eviction failed
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what the problem could be? If not, I'll try bisecting.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): c76c7703-6c0d-0d2a-ba9f-c012d74c01bf-5C7GfCeVMHo@public.gmane.org
RE: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
Quoted 9 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)
Quoted 4 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2
That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
Alex
Quoted 56 lines
We haven't looked into this change in detail yet, to understand the cause. Kent found it by bisecting on amd-staging-4.6 and applying KFD changes on top.
Regards,
FelixQuoted 11 lines · Felix Kuehling
For the record, Michel's patch "drm/ttm: Wait for a BO to become idle before unbinding it from GTT" fixes our KFD problem as well.
Thanks,
FelixQuoted 4 lines · Felix Kuehling
We're also looking into a hang with a KFD unit test that allocates lots of memory and fragments it deliberately, without mapping it all at once. It's a new problem for us as we're rebasing on amd-staging-4.6. Something weird seems to be happening with evictions, but I haven't
been
Quoted 5 lines
Quoted 5 lines
able to figure it out.
I was able to see that SDMA page table updates stop working at some point, though SDMA fences are still signaling. If I let the test run longer, SDMA and CP hang. I dumped the SDMA IBs and didn't see
anything
Quoted 4 lines
Quoted 4 lines
suspicious. My guess was that maybe the SDMA IBs or the ring are getting corrupted, or maybe the GART table entries for the IBs or ring are corrupted. But I haven't been able to prove that or track it down to a root cause. We're now trying to reimplement the test using libdrm-
amdgpu
Quoted 18 lines
Quoted 18 lines
APIs so we can bisect on the amd-staging-4.6 branch without KFD.
Regards,
FelixQuoted 12 lines · Michel Dänzer
Quoted 3 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
[TTM] Buffer eviction failed
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what
the
Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
problem could be? If not, I'll try bisecting.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): MWHPR12MB16948D7420076C10DCFAEEBBF71E0-Gy0DoCVfaSW4WA4dJ5YXGAdYzm3356FpvxpqHgZTriW3zl9H0oFU5g@public.gmane.org
Re: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
[CC Kent FYI]
Quoted 9 lines · Alexander
Quoted 9 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)
Quoted 5 lines
Quoted 4 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
[FK] But I think the patch shouldn't cause hangs, regardless.
I eventually found what the problem was. The "group BOs by log2 of the size on the LRU v2" patch exposed a latent bug related to the GART size. On our KFD branch, we calculate the GART size differently, and it can easily go above 4GB. I think on amd-staging-4.6 the GART size can also go above 4GB on cards with lots of VRAM.
However, the offset parameter in amdgpu_gart_bind and unbind is only 32-bit. With the patch our test ended up using GART offsets beyond 4GB for the first time. Changing the offset parameter to uint64_t fixes the problem.
Our test also demonstrates a potential flaw in the log2 grouping patch:
When a buffer of a previously unused size is added to the LRU, it gets added to the front of the list, rather than the tail. So an application that allocates a very large buffer after a bunch of smaller buffers, is very likely to have that buffer evicted over and over again before any smaller buffers are considered for eviction. I believe, this can result in thrashing of large buffers.
Some other observations: When the last BO of a given size is removed from the LRU list, the LRU tail for that size is left "floating" in the middle of the LRU list. So the next BO of that size that is added, will be added at an arbitrary position in the list. It may even end up in the middle of a block of pages of a different size. So a log2 grouping may end up being split.
Regards,
Felix
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 12c479b4-64d0-952c-f9d2-decd25533251-5C7GfCeVMHo@public.gmane.org
Re: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
Am 13.08.2016 um 01:22 schrieb Felix Kuehling:
Quoted 25 lines
[CC Kent FYI]
Quoted 9 lines · Alexander
Quoted 9 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)Quoted 5 lines
Quoted 4 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
[FK] But I think the patch shouldn't cause hangs, regardless.
I eventually found what the problem was. The "group BOs by log2 of the size on the LRU v2" patch exposed a latent bug related to the GART size. On our KFD branch, we calculate the GART size differently, and it can easily go above 4GB. I think on amd-staging-4.6 the GART size can also go above 4GB on cards with lots of VRAM.
Quoted 4 lines
However, the offset parameter in
amdgpu_gart_bindand unbind is only 32-bit. With the patch our test ended up using GART offsets beyond 4GB for the first time. Changing the offset parameter touint64_tfixes the problem.
Nice catch, please provide a patch to fix this.
Quoted 8 lines
Our test also demonstrates a potential flaw in the log2 grouping patch:
When a buffer of a previously unused size is added to the LRU, it gets added to the front of the list, rather than the tail. So an application that allocates a very large buffer after a bunch of smaller buffers, is very likely to have that buffer evicted over and over again before any smaller buffers are considered for eviction. I believe, this can result in thrashing of large buffers.
Quoted 6 lines
Some other observations: When the last BO of a given size is removed from the LRU list, the LRU tail for that size is left "floating" in the middle of the LRU list. So the next BO of that size that is added, will be added at an arbitrary position in the list. It may even end up in the middle of a block of pages of a different size. So a log2 grouping may end up being split.
Yeah, those are more or less known issues.
Keep in mind that we only added the grouping by log2 of the size to have a justification to push the TTM changes upstream for the coming KFD fences.
E.g. so that we are able to have this upstream before we try to push on the fence code.
I will take a look at fixing those issues when I have time, shouldn't be to complicated to set the entries to zero when they aren't used or adjust other entries as well when some are added.
Regards,
Christian.
Quoted 8 lines
Regards,
Felix_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): c016cb14-f262-7432-f4ef-cdbb638807f4-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Re: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
Patch against current amd-staging-4.6 is attached.
Regards,
Felix
Quoted 69 lines · Christian König
Am 13.08.2016 um 01:22 schrieb Felix Kuehling:
Quoted 26 lines
[CC Kent FYI]
Quoted 9 lines · Alexander
Quoted 9 lines
-----Original Message-----
From: amd-gfx [mailto:amd-gfx-bounces-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org] On Behalf Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)Quoted 6 lines
Quoted 4 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
[FK] But I think the patch shouldn't cause hangs, regardless.
I eventually found what the problem was. The "group BOs by log2 of the size on the LRU v2" patch exposed a latent bug related to the GART size. On our KFD branch, we calculate the GART size differently, and it can easily go above 4GB. I think on amd-staging-4.6 the GART size can also go above 4GB on cards with lots of VRAM.
Quoted 4 lines
However, the offset parameter in
amdgpu_gart_bindand unbind is only 32-bit. With the patch our test ended up using GART offsets beyond 4GB for the first time. Changing the offset parameter touint64_tfixes the problem.Nice catch, please provide a patch to fix this.
Quoted 8 lines
Our test also demonstrates a potential flaw in the log2 grouping patch:
When a buffer of a previously unused size is added to the LRU, it gets added to the front of the list, rather than the tail. So an application that allocates a very large buffer after a bunch of smaller buffers, is very likely to have that buffer evicted over and over again before any smaller buffers are considered for eviction. I believe, this can result in thrashing of large buffers.Quoted 6 lines
Some other observations: When the last BO of a given size is removed from the LRU list, the LRU tail for that size is left "floating" in the middle of the LRU list. So the next BO of that size that is added, will be added at an arbitrary position in the list. It may even end up in the middle of a block of pages of a different size. So a log2 grouping may end up being split.
Yeah, those are more or less known issues.
Keep in mind that we only added the grouping by log2 of the size to have a justification to push the TTM changes upstream for the coming KFD fences.
E.g. so that we are able to have this upstream before we try to push on the fence code.
I will take a look at fixing those issues when I have time, shouldn't be to complicated to set the entries to zero when they aren't used or adjust other entries as well when some are added.
Regards,
Christian.Quoted 4 lines
Regards,
Felix
Quoted 8 lines
Quoted 4 lines
_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfxFrom
34b9f9e33fc83e375c291e8d3480f8faff150183Mon Sep 17 00:00:00 2001
From: Felix Kuehling <[email protected]>
Date: Fri, 12 Aug 2016 19:25:21 -0400
Subject: [PATCH] drm/amdgpu: Change GART offset to 64-bit
The GART aperture size can be bigger than 4GB. Therefore the offset used in amdgpu_gart_bind and amdgpu_gart_unbind must be 64-bit.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 -2
@@ -662,9 +662,9 @@ int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);void amdgpu_gart_table_vram_unpin (struct amdgpu_device *adev);int amdgpu_gart_init (struct amdgpu_device *adev);void amdgpu_gart_fini (struct amdgpu_device *adev); -void amdgpu_gart_unbind (struct amdgpu_device *adev,unsigned offset, +void amdgpu_gart_unbind (struct amdgpu_device *adev,uint64_t offset,int pages); -int amdgpu_gart_bind (struct amdgpu_device *adev,unsigned offset, +int amdgpu_gart_bind (struct amdgpu_device *adev,uint64_t offset,int pages,struct page **pagelist,dma_addr_t *dma_addr,uint32_t flags);
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +2 -2
@@ -221,7 +221,7 @@ void amdgpu_gart_table_vram_free(struct amdgpu_device *adev)* Unbinds the requested pages from the gart page table and * replaces them with the dummy page (all asics). */ -void amdgpu_gart_unbind (struct amdgpu_device *adev,unsigned offset, +void amdgpu_gart_unbind (struct amdgpu_device *adev,uint64_t offset,int pages) {unsigned t;
@@ -268,7 +268,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, unsigned offset,* (all asics). * Returns 0 for success, -EINVAL for failure. */ -int amdgpu_gart_bind (struct amdgpu_device *adev,unsigned offset, +int amdgpu_gart_bind (struct amdgpu_device *adev,uint64_t offset,int pages,struct page **pagelist,dma_addr_t *dma_addr,uint32_t flags) {
Signature · 1 line
1.9.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): d120fde0-a175-c9b5-a3ab-b6840207190e-5C7GfCeVMHo@public.gmane.org
Alex Deucher
Reviewed-byRe: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
Quoted 1 line · Felix Kuehling
Patch against current amd-staging-4.6 is attached.
Quoted 83 lines
Regards,
FelixQuoted 58 lines · Christian König
Am 13.08.2016 um 01:22 schrieb Felix Kuehling:
Quoted 26 lines
[CC Kent FYI]
Quoted 9 lines · Alexander
Quoted 9 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)Quoted 6 lines
Quoted 4 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
[FK] But I think the patch shouldn't cause hangs, regardless.
I eventually found what the problem was. The "group BOs by log2 of the size on the LRU v2" patch exposed a latent bug related to the GART size. On our KFD branch, we calculate the GART size differently, and it can easily go above 4GB. I think on amd-staging-4.6 the GART size can also go above 4GB on cards with lots of VRAM.
Quoted 4 lines
However, the offset parameter in
amdgpu_gart_bindand unbind is only 32-bit. With the patch our test ended up using GART offsets beyond 4GB for the first time. Changing the offset parameter touint64_tfixes the problem.Nice catch, please provide a patch to fix this.
Quoted 8 lines
Our test also demonstrates a potential flaw in the log2 grouping patch:
When a buffer of a previously unused size is added to the LRU, it gets added to the front of the list, rather than the tail. So an application that allocates a very large buffer after a bunch of smaller buffers, is very likely to have that buffer evicted over and over again before any smaller buffers are considered for eviction. I believe, this can result in thrashing of large buffers.Quoted 6 lines
Some other observations: When the last BO of a given size is removed from the LRU list, the LRU tail for that size is left "floating" in the middle of the LRU list. So the next BO of that size that is added, will be added at an arbitrary position in the list. It may even end up in the middle of a block of pages of a different size. So a log2 grouping may end up being split.
Yeah, those are more or less known issues.
Keep in mind that we only added the grouping by log2 of the size to have a justification to push the TTM changes upstream for the coming KFD fences.
E.g. so that we are able to have this upstream before we try to push on the fence code.
Quoted 17 lines
I will take a look at fixing those issues when I have time, shouldn't be to complicated to set the entries to zero when they aren't used or adjust other entries as well when some are added.
Regards,
Christian.Quoted 8 lines
Regards,
Felix_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 5 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CADnq5_M2orK5FoDD5+xZjhStj6mRRkRNho+tLHRuvKq27nSHZQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Christian König
Reviewed-byRe: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
Am 15.08.2016 um 23:03 schrieb Alex Deucher:
Quoted 3 lines · Felix Kuehling
Quoted 1 line
Patch against current amd-staging-4.6 is attached.
Reviewed-by Alex Deucher <[email protected]>
Quoted 78 lines
Quoted 78 lines
Regards,
FelixQuoted 71 lines · Christian König
Am 13.08.2016 um 01:22 schrieb Felix Kuehling:
Quoted 26 lines
[CC Kent FYI]
Quoted 9 lines · Alexander
Quoted 9 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)Quoted 6 lines
Quoted 4 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
[FK] But I think the patch shouldn't cause hangs, regardless.
I eventually found what the problem was. The "group BOs by log2 of the size on the LRU v2" patch exposed a latent bug related to the GART size. On our KFD branch, we calculate the GART size differently, and it can easily go above 4GB. I think on amd-staging-4.6 the GART size can also go above 4GB on cards with lots of VRAM.
Quoted 4 lines
However, the offset parameter in
amdgpu_gart_bindand unbind is only 32-bit. With the patch our test ended up using GART offsets beyond 4GB for the first time. Changing the offset parameter touint64_tfixes the problem.Nice catch, please provide a patch to fix this.
Quoted 8 lines
Our test also demonstrates a potential flaw in the log2 grouping patch:
When a buffer of a previously unused size is added to the LRU, it gets added to the front of the list, rather than the tail. So an application that allocates a very large buffer after a bunch of smaller buffers, is very likely to have that buffer evicted over and over again before any smaller buffers are considered for eviction. I believe, this can result in thrashing of large buffers.Quoted 6 lines
Some other observations: When the last BO of a given size is removed from the LRU list, the LRU tail for that size is left "floating" in the middle of the LRU list. So the next BO of that size that is added, will be added at an arbitrary position in the list. It may even end up in the middle of a block of pages of a different size. So a log2 grouping may end up being split.
Yeah, those are more or less known issues.
Keep in mind that we only added the grouping by log2 of the size to have a justification to push the TTM changes upstream for the coming KFD fences.
E.g. so that we are able to have this upstream before we try to push on the fence code.
I will take a look at fixing those issues when I have time, shouldn't be to complicated to set the entries to zero when they aren't used or adjust other entries as well when some are added.
Regards,
Christian.Quoted 7 lines
Regards,
Felix_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 5 lines
Quoted 5 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): abfdc55c-f95c-327f-fa9e-114f0b3884e3-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Re: Reverted another change to fix buffer move hangs (was Re: [PATCH] drm/ttm: partial revert "cleanup ttm_tt_(unbind|destroy)" v2)
Thank you. Sorry, I already pushed it with Alex's R-B, without yours.
Quoted 110 lines · Christian König
Am 15.08.2016 um 23:03 schrieb Alex Deucher:
Quoted 4 lines
Quoted 1 line · Felix Kuehling
Patch against current amd-staging-4.6 is attached.
Reviewed-by Alex Deucher <[email protected]>Reviewed-by Christian König <[email protected]>.Quoted 101 lines
Quoted 101 lines
Regards,
FelixQuoted 74 lines · Christian König
Am 13.08.2016 um 01:22 schrieb Felix Kuehling:
Quoted 32 lines
[CC Kent FYI]
Quoted 11 lines · Alexander
Quoted 11 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf
Of Felix Kuehling
Sent: Thursday, August 11, 2016 3:52 PM
To: Michel Dänzer; Christian König
Cc: [email protected]
Subject: Reverted another change to fix buffer move hangs (was Re:
[PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2)Quoted 7 lines
Quoted 5 lines
We had to revert another change on the KFD branch to fix a buffer move
problem:8b6b79f43801f00ddcdc10a4d5719eba4b2e32aa(drm/amdgpu:
group BOs
by log2 of the size on the LRU v2That makes sense. I think you may want a different LRU scheme for KFD or at least special handling for KFD buffers.
[FK] But I think the patch shouldn't cause hangs, regardless.
I eventually found what the problem was. The "group BOs by log2 of the
size on the LRU v2" patch exposed a latent bug related to the GART size.
On our KFD branch, we calculate the GART size differently, and it can easily go above 4GB. I think on amd-staging-4.6 the GART size can also
go above 4GB on cards with lots of VRAM.Quoted 6 lines
However, the offset parameter in
amdgpu_gart_bindand unbind is only 32-bit. With the patch our test ended up using GART offsets beyond 4GB
for the first time. Changing the offset parameter touint64_tfixes the
problem.Nice catch, please provide a patch to fix this.
Quoted 14 lines
Our test also demonstrates a potential flaw in the log2 grouping patch:
When a buffer of a previously unused size is added to the LRU, it gets
added to the front of the list, rather than the tail. So an application
that allocates a very large buffer after a bunch of smaller buffers, is
very likely to have that buffer evicted over and over again before any
smaller buffers are considered for eviction. I believe, this can result
in thrashing of large buffers.Quoted 10 lines
Some other observations: When the last BO of a given size is removed from the LRU list, the LRU tail for that size is left "floating" in the
middle of the LRU list. So the next BO of that size that is added, will
be added at an arbitrary position in the list. It may even end up in the
middle of a block of pages of a different size. So a log2 grouping may
end up being split.Yeah, those are more or less known issues.
Keep in mind that we only added the grouping by log2 of the size to have a justification to push the TTM changes upstream for the coming KFD fences.
E.g. so that we are able to have this upstream before we try to push on the fence code.
Quoted 15 lines
I will take a look at fixing those issues when I have time, shouldn't be to complicated to set the entries to zero when they aren't used or adjust other entries as well when some are added.
Regards,
Christian.Quoted 7 lines
Regards,
Felix_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Am 27.07.2016 um 04:26 schrieb Michel Dänzer:
Quoted 12 lines · Christian König
Quoted 3 lines
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
Quoted 6 lines
[TTM] Buffer eviction failed
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what the problem could be? If not, I'll try bisecting.
Of hand I would certainly try to disable pipelined buffer moves.
You could also try to revert "6b5f77799274c151f6cc5467c5abe6ad366a965e drm/amdgpu: remove pre move wait".
Regards,
Christian.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 53b30c35-782a-2668-ea8b-199efae8b650-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org
Quoted 15 lines · Christian König
Am 27.07.2016 um 04:26 schrieb Michel Dänzer:
Quoted 14 lines · Christian König
Quoted 3 lines
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
This change fixed a hang for me when running the piglit test max-texture-size with the radeon driver on Kaveri.
However, there's still a similar hang left when letting the piglit test tex3d-maxsize run concurrently with other tests (running tex3d-maxsize alone doesn't hang, but fails due to running out of GPU memory; that's a recent radeonsi regression). There are
[TTM] Buffer eviction failed
Quoted 8 lines
Quoted 4 lines
messages in dmesg shortly before the hang.
I haven't seen such hangs with older kernels. Any ideas offhand what the problem could be? If not, I'll try bisecting.
Of hand I would certainly try to disable pipelined buffer moves.
You could also try to revert "
6b5f77799274c151f6cc5467c5abe6ad366a965edrm/amdgpu: remove pre move wait".
This is using the radeon driver.
Signature · 6 lines
Earthling Michel Dänzer | http://www.amd.com Libre software enthusiast | Mesa and X developer _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Deucher, Alexander
Reviewed-by Alex DeucherQuoted 11 lines
-----Original Message-----
From: amd-gfx [mailto:[email protected]] On Behalf Of Christian König
Sent: Friday, July 22, 2016 9:10 AM
To: [email protected]; [email protected]
Cc: Lin, Amber; Kuehling, Felix; [email protected]
Subject: [PATCH] drm/ttm: partial revert "cleanupttm_tt_(unbind|destroy)" v2From: Christian König <[email protected]>
Quoted 7 lines
We still need to unbind explicitely during a move.
This partial reverts commit
ff20caa0bcbfef9f7686f8d1868a3b990921afd6.v2: remove unnecessary check and unused variable
Signed-off-by Christian König <[email protected]>
Quoted 85 lines
--- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 + drivers/gpu/drm/ttm/ttm_tt.c | 19 ++++++++++++------- include/drm/ttm/ttm_bo_driver.h | 9 +++++++++ 3 files changed, 22 insertions(+), 7 deletions(-)drivers/gpu/drm/ttm/ttm_bo_util.c +1
-0@@ -53,6 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,int ret;if (old_mem->mem_type !=TTM_PL_SYSTEM ) { +ttm_tt_unbind (ttm);ttm_bo_free_old_node (bo); ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,drivers/gpu/drm/ttm/ttm_bo_util.c +0
-0TTM_PL_MASK_MEM);drivers/gpu/drm/ttm/ttm_tt.c +12
-7@@ -166,16 +166,10 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);void ttm_tt_destroy (struct ttm_tt *ttm) { -int ret; -if (ttm ==NULL )return ; -if (ttm->state == tt_bound) { - ret = ttm->func->unbind (ttm); -BUG_ON (ret); - ttm->state = tt_unbound; - } +ttm_tt_unbind (ttm);if (ttm->state == tt_unbound)ttm_tt_unpopulate (ttm);@@ -255,6 +249,17 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) }EXPORT_SYMBOL (ttm_dma_tt_fini); +void ttm_tt_unbind (struct ttm_tt *ttm) +{ +int ret; + +if (ttm->state == tt_bound) { + ret = ttm->func->unbind (ttm); +BUG_ON (ret); + ttm->state = tt_unbound; + } +} +int ttm_tt_bind (struct ttm_tt *ttm,struct ttm_mem_reg *bo_mem) {int ret =0 ;include/drm/ttm/ttm_bo_driver.h +9
-0@@ -628,6 +628,15 @@ extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem);extern void ttm_tt_destroy (struct ttm_tt *ttm);/** +* ttm_ttm_unbind: +* +* @ttm: The struct ttm_tt. +* +* Unbind a struct ttm_tt. +*/ +extern void ttm_tt_unbind (struct ttm_tt *ttm); + +/** * ttm_tt_swapin: * * @ttm: The struct
ttm_tt.
--
2.5.0
Quoted 4 lines
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Quoted 3 lines · Christian König
From: Christian König <[email protected]>
We still need to unbind explicitely during a move.
Sorry for the drive-by nit, but:
s/explicitely/explicitly/
Quoted 87 lines
This partial reverts commit
ff20caa0bcbfef9f7686f8d1868a3b990921afd6.v2: remove unnecessary check and unused variable
Signed-off-by Christian König <[email protected]>--- drivers/gpu/drm/ttm/ttm_bo_util.c | 1 + drivers/gpu/drm/ttm/ttm_tt.c | 19 ++++++++++++------- include/drm/ttm/ttm_bo_driver.h | 9 +++++++++ 3 files changed, 22 insertions(+), 7 deletions(-)drivers/gpu/drm/ttm/ttm_bo_util.c +1
-0@@ -53,6 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,int ret;if (old_mem->mem_type !=TTM_PL_SYSTEM ) { +ttm_tt_unbind (ttm);ttm_bo_free_old_node (bo);ttm_flag_masked (&old_mem->placement,TTM_PL_FLAG_SYSTEM ,TTM_PL_MASK_MEM );drivers/gpu/drm/ttm/ttm_tt.c +12
-7@@ -166,16 +166,10 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);void ttm_tt_destroy (struct ttm_tt *ttm) { -int ret; -if (ttm ==NULL )return ; -if (ttm->state == tt_bound) { - ret = ttm->func->unbind (ttm); -BUG_ON (ret); - ttm->state = tt_unbound; - } +ttm_tt_unbind (ttm);if (ttm->state == tt_unbound)ttm_tt_unpopulate (ttm);@@ -255,6 +249,17 @@ void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) }EXPORT_SYMBOL (ttm_dma_tt_fini); +void ttm_tt_unbind (struct ttm_tt *ttm) +{ +int ret; + +if (ttm->state == tt_bound) { + ret = ttm->func->unbind (ttm); +BUG_ON (ret); + ttm->state = tt_unbound; + } +} +int ttm_tt_bind (struct ttm_tt *ttm,struct ttm_mem_reg *bo_mem) {int ret =0 ;include/drm/ttm/ttm_bo_driver.h +9
-1@@ -628,6 +628,15 @@ extern int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem);extern void ttm_tt_destroy (struct ttm_tt *ttm);/** +* ttm_ttm_unbind: +* +* @ttm: The struct ttm_tt. +* +* Unbind a struct ttm_tt. +*/ +extern void ttm_tt_unbind (struct ttm_tt *ttm); + +/** * ttm_tt_swapin: * * @ttm: The struct ttm_tt. -- 2.5.0
Quoted 4 lines
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel