drm/amdgpu/gmc9: disallow gfxoff around TLB flushes
Subsystems (2)
Alex Deucher
maintainerPATCH 1/14 drm/amdgpu/gmc9: disallow gfxoff around TLB flushes
We need to disallow gfxoff if we touch GC MMIO registers. At the moment we use KIQ or MES for TLB flushes so no intended functional change.
--- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +7 -0
@@ -808,6 +808,10 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,return ; } +/* disabllow gfxoff when we invalidate */ +if (vmhub <AMDGPU_MMHUB0 (0 )) +amdgpu_gfx_off_ctrl (adev, false); +/* This path is needed before KIQ/MES/GFXOFF are set up */ spin_lock (&adev->gmc.invalidate_lock);
@@ -873,6 +877,9 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,spin_unlock (&adev->gmc.invalidate_lock); +if (vmhub <AMDGPU_MMHUB0 (0 )) +amdgpu_gfx_off_ctrl (adev, true); +if (j < adev->usec_timeout)return ;
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 2/14 drm/amdgpu/gmc10: disallow gfxoff around TLB flushes
We need to disallow gfxoff if we touch GC MMIO registers. At the moment we use KIQ or MES for TLB flushes so no intended functional change.
--- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +7 -0
@@ -265,6 +265,10 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,/* This path is needed before KIQ/MES/GFXOFF are set up */ hub_ip = (vmhub ==AMDGPU_GFXHUB (0 )) ?GC_HWIP :MMHUB_HWIP ; +/* disabllow gfxoff when we invalidate */ +if (hub_ip ==GC_HWIP ) +amdgpu_gfx_off_ctrl (adev, false); +spin_lock (&adev->gmc.invalidate_lock);/* * It may lose gpuvm invalidate acknowldege state across power-gating
@@ -313,6 +317,9 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,spin_unlock (&adev->gmc.invalidate_lock); +if (hub_ip ==GC_HWIP ) +amdgpu_gfx_off_ctrl (adev, true); +if (i >= adev->usec_timeout)dev_err (adev->dev,"Timeout waiting for VM flush hub: %d!\n" , vmhub);
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 3/14 drm/amdgpu/gmc11: disallow gfxoff around TLB flushes
In reply to Alex DeucherWe need to disallow gfxoff if we touch GC MMIO registers. At the moment we use KIQ or MES for TLB flushes so no intended functional change.
--- drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 7 +++++++ 1 file changed, 7 insertions(+)
drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +7 -0
@@ -253,6 +253,10 @@ static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,/* This path is needed before KIQ/MES/GFXOFF are set up */ hub_ip = (vmhub ==AMDGPU_GFXHUB (0 )) ?GC_HWIP :MMHUB_HWIP ; +/* disabllow gfxoff when we invalidate */ +if (hub_ip ==GC_HWIP ) +amdgpu_gfx_off_ctrl (adev, false); +spin_lock (&adev->gmc.invalidate_lock);/* * It may lose gpuvm invalidate acknowldege state across power-gating
@@ -306,6 +310,9 @@ static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,spin_unlock (&adev->gmc.invalidate_lock); +if (hub_ip ==GC_HWIP ) +amdgpu_gfx_off_ctrl (adev, true); +if (i >= adev->usec_timeout)dev_err (adev->dev,"Timeout waiting for VM flush ACK!\n" ); }
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 4/14 drm/amdgpu/gmc12: disallow gfxoff around TLB flushes
In reply to Alex DeucherWe need to disallow gfxoff if we touch GC MMIO registers. At the moment we use KIQ or MES for TLB flushes so no intended functional change.
--- drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +7 -1
@@ -327,8 +327,14 @@ static void gmc_v12_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,return ; } +/* disabllow gfxoff when we invalidate */ +if (vmhub ==AMDGPU_GFXHUB (0 )) +amdgpu_gfx_off_ctrl (adev, false); +gmc_v12_0_flush_vm_hub (adev, vmid, vmhub,0 ); -return ; + +if (vmhub ==AMDGPU_GFXHUB (0 )) +amdgpu_gfx_off_ctrl (adev, true); }/**
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 5/14 drm/amdgpu: add an buffer funcs callback for TLB invalidation
In reply to Alex DeucherUse this interface to issue TLB invalidations using SDMA.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18 -0
@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Signature · 1 line
2.55.0
Christian König
maintainerRe: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 2 lines · Alex Deucher
Use this interface to issue TLB invalidations using SDMA.
Hui? What should that be good for?
Regards,
Christian.
Quoted 42 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18
-0@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Re: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 6 lines · Christian König
Quoted 2 lines · Alex Deucher
Use this interface to issue TLB invalidations using SDMA.
Hui? What should that be good for?
To use as a replacement for doing the invalidation using MES.
Alex
Quoted 47 lines
Regards,
Christian.Quoted 42 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18
-0@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Christian König
maintainerRe: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 10 lines · Alex Deucher
Quoted 6 lines · Christian König
Quoted 2 lines · Alex Deucher
Use this interface to issue TLB invalidations using SDMA.
Hui? What should that be good for?
To use as a replacement for doing the invalidation using MES.
We should already have that as workaround for the Navi 1x SDMA bug.
I suggest to just move that code into a separate function instead.
But the SDMA can't do PASID based invalidation and that is what we need the MES for, so I'm not sure how useful that will be.
Regards,
Christian.
Quoted 50 lines
Alex
Quoted 47 lines
Regards,
Christian.Quoted 42 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18
-0@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Re: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 11 lines · Christian König
Quoted 9 lines · Alex Deucher
Quoted 6 lines · Christian König
Quoted 2 lines · Alex Deucher
Use this interface to issue TLB invalidations using SDMA.
Hui? What should that be good for?
Quoted 7 lines
Quoted 1 line
To use as a replacement for doing the invalidation using MES.
We should already have that as workaround for the Navi 1x SDMA bug.
I suggest to just move that code into a separate function instead.
But the SDMA can't do PASID based invalidation and that is what we need the MES for, so I'm not sure how useful that will be.
The current gmc code doesn't do pasid based invalidation either. Everything ends up in gmc_vXX_0_flush_gpu_tlb() for each gmc version.
Alex
Quoted 55 lines
Regards,
Christian.Quoted 50 lines
Alex
Quoted 47 lines
Regards,
Christian.Quoted 42 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18
-0@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Christian König
maintainerRe: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 15 lines · Alex Deucher
Quoted 13 lines · Christian König
Quoted 10 lines · Alex Deucher
Quoted 6 lines · Christian König
Quoted 2 lines · Alex Deucher
Use this interface to issue TLB invalidations using SDMA.
Hui? What should that be good for?
To use as a replacement for doing the invalidation using MES.
Quoted 8 lines
Quoted 5 lines
We should already have that as workaround for the Navi 1x SDMA bug.
I suggest to just move that code into a separate function instead.
But the SDMA can't do PASID based invalidation and that is what we need the MES for, so I'm not sure how useful that will be.
The current gmc code doesn't do pasid based invalidation either. Everything ends up in
gmc_vXX_0_flush_gpu_tlb()for each gmc version.
Yeah, but that one shouldn't use the MES in the first place. We could use direct register accesses here.
Using the MES makes only sense if we invalidate per PASID.
Why are we using the MES for per VMID invalidation?
Regards,
Christian.
Quoted 58 lines
Alex
Quoted 55 lines
Regards,
Christian.Quoted 50 lines
Alex
Quoted 47 lines
Regards,
Christian.Quoted 42 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18
-0@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Re: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 25 lines · Christian König
Quoted 21 lines · Alex Deucher
Quoted 18 lines · Christian König
Quoted 10 lines · Alex Deucher
Quoted 6 lines · Christian König
Quoted 2 lines · Alex Deucher
Use this interface to issue TLB invalidations using SDMA.
Hui? What should that be good for?
To use as a replacement for doing the invalidation using MES.
We should already have that as workaround for the Navi 1x SDMA bug.
I suggest to just move that code into a separate function instead.
But the SDMA can't do PASID based invalidation and that is what we need the MES for, so I'm not sure how useful that will be.
Quoted 8 lines
Quoted 2 lines
The current gmc code doesn't do pasid based invalidation either. Everything ends up in
gmc_vXX_0_flush_gpu_tlb()for each gmc version.Yeah, but that one shouldn't use the MES in the first place. We could use direct register accesses here.
Using the MES makes only sense if we invalidate per PASID.
Why are we using the MES for per VMID invalidation?
gfxoff. We can't access the registers directly unless we disallow gfxoff.
Alex
Quoted 63 lines
Regards,
Christian.Quoted 58 lines
Alex
Quoted 55 lines
Regards,
Christian.Quoted 50 lines
Alex
Quoted 47 lines
Regards,
Christian.Quoted 42 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +18
-0@@ -155,6 +155,23 @@ struct amdgpu_buffer_funcs {uint64_t dst_offset,/* number of byte to fill */ uint32_t byte_count); + +/* number of dw to reserve per operation */ +unsigned tlb_inv_num_dw; + +/* used for buffer clearing */ +void (*emit_tlb_inv)(struct amdgpu_device *adev, +struct amdgpu_ib *ib, +/* vmid to target */ +unsigned int vmid, +/* vmhub to target */ +u32 vmhub, +/* inv eng to target */ +u32 eng, +/* flush type */ +u32 flush_type, +/* XCC to target */ +u32 xcc_inst); }; int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,@@ -162,6 +179,7 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,#define amdgpu_emit_copy_buffer (adev, ib, s, d, b, t) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b), (t))#define amdgpu_emit_fill_buffer (adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_tlb_inv (adev, ib, v, h, e, f, x) (adev)->mman.buffer_funcs->emit_tlb_inv((adev), (ib), (v), (h), (e), (f), (x))struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring (struct amdgpu_ring *ring);
Christian König
maintainerRe: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 3 lines · Alex Deucher
Quoted 1 line
Why are we using the MES for per VMID invalidation?
gfxoff. We can't access the registers directly unless we disallow gfxoff.
Well there is a remark that VM_INVALIDATE_ENG*_ACK is always save to read, even when gfxoff is active.
And we explicitly acquire the semaphore to avoid that GFXOFF kicks in.
I also don't see how we use the MES here?
What we do is to use the KIQ on gfx9, but that is actually only as a workaround for some SRIOV problems.
Christian.
Quoted 2 lines
Alex
Re: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 9 lines · Christian König
Quoted 3 lines · Alex Deucher
Quoted 1 line
Why are we using the MES for per VMID invalidation?
gfxoff. We can't access the registers directly unless we disallow gfxoff.
Well there is a remark that VM_INVALIDATE_ENG*_ACK is always save to read, even when gfxoff is active.
And we explicitly acquire the semaphore to avoid that GFXOFF kicks in.
I'm pretty sure the semaphore just keeps it active if it's already active, but the registers are in the gfx tile so they are gfxoff controlled. I think the semaphore is just meant to be used as a lock to keep gfxoff from kicking in in the middle of an invalidation.
Quoted 5 lines
I also don't see how we use the MES here?
What we do is to use the KIQ on gfx9, but that is actually only as a workaround for some SRIOV problems.
We do it for everything gfx9 and newer. For example gmc_v11_0_flush_gpu_tlb():
/* This is necessary for SRIOV as well as for GFXOFF to
function
* properly under bare metal
*/
if ((adev->gfx.kiq[0].ring.sched.ready ||
adev->mes.ring[0].sched.ready) &&
(amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
1 << vmid, GET_INST(GC, 0));
return;
}The only time we use the MMIO path is if KIQ or MES is not ready yet. The goal of this patch set was to switch to using SDMA rather than MES and to clean up the gmc code since most of the logic is the same for gfx9 and above.
Alex
Quoted 4 lines
Christian.
Quoted 2 lines
Alex
Christian König
maintainerRe: [PATCH 05/14] drm/amdgpu: add an buffer funcs callback for TLB invalidation
Quoted 8 lines · Alex Deucher
Quoted 6 lines · Christian König
Quoted 3 lines · Alex Deucher
Quoted 1 line
Why are we using the MES for per VMID invalidation?
gfxoff. We can't access the registers directly unless we disallow gfxoff.
Quoted 8 lines
Quoted 3 lines
Well there is a remark that VM_INVALIDATE_ENG*_ACK is always save to read, even when gfxoff is active.
And we explicitly acquire the semaphore to avoid that GFXOFF kicks in.
I'm pretty sure the semaphore just keeps it active if it's already active, but the registers are in the gfx tile so they are gfxoff controlled. I think the semaphore is just meant to be used as a lock to keep gfxoff from kicking in in the middle of an invalidation.
What exactly is the failure you are seeing?
What could potentially happen is that GFXOFF suspended the block and we are not able to acquire the semaphore, but in that care we could also simply ignore the invalidation request.
Quoted 20 lines
Quoted 5 lines
I also don't see how we use the MES here?
What we do is to use the KIQ on gfx9, but that is actually only as a workaround for some SRIOV problems.
We do it for everything gfx9 and newer. For example
gmc_v11_0_flush_gpu_tlb():/* This is necessary for SRIOV as well as for GFXOFF to function * properly under bare metal */ if ((adev->gfx.kiq[0].ring.sched.ready || adev->mes.ring[0].sched.ready) && (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) { amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, 1 << vmid, GET_INST(GC, 0)); return; }
Quoted 4 lines
The only time we use the MMIO path is if KIQ or MES is not ready yet. The goal of this patch set was to switch to using SDMA rather than MES and to clean up the gmc code since most of the logic is the same for gfx9 and above.
We should probably stop doing this. As far as I can see that should be completely unnecessary.
If we really want to move this to the SDMA we should do it a level higher in amdgpu_gmc_flush_gpu_tlb().
Regards,
Christian.
Quoted 8 lines
Alex
Quoted 4 lines
Christian.
Quoted 2 lines
Alex
Alex Deucher
maintainerPATCH 6/14 drm/amdgpu/sdma5.0: add tlb invalidation buffer func callback
In reply to Alex DeucherWill be used for TLB invalidation.
--- drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +49 -0
@@ -2045,6 +2045,52 @@ static void sdma_v5_0_emit_fill_buffer(struct amdgpu_ib *ib, ib->ptr[ib->length_dw++] = byte_count -1 ; } +/** +* sdma_v5_0_emit_tlb_inv - Invalidate TLB using the sDMA engine +* +* @adev: amdgpu device structure +* @ib: indirect buffer to fill +* @vmid: vmid to target +* @vmhub: vmhub to target +* @eng: invalidation engine to use +* @flush_type: type of flush (lightweight, heavyweight) +* @xcc_inst: XCC to target +* +* Invalidate TLB using the DMA engine. +*/ +static void sdma_v5_0_emit_tlb_inv (struct amdgpu_device *adev, +struct amdgpu_ib *ib, +unsigned int vmid, +u32 vmhub, +u32 eng, +u32 flush_type, +u32 xcc_inst) +{ +struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; +u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); +u32 mmhub_eng, gfxhub_eng; + +if (AMDGPU_IS_GFXHUB (vmhub)) { + mmhub_eng =0x1f ; + gfxhub_eng = eng; + }else { + mmhub_eng = eng; + gfxhub_eng =0x1f ; + } + +/* Trigger invalidation. */ + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_HEADER_OP (SDMA_OP_POLL_REGMEM ) | +SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP (SDMA_SUBOP_VM_INVALIDATION ) | +SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID (gfxhub_eng) | +SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID (mmhub_eng)); + ib->ptr[ib->length_dw++] = inv_req; + ib->ptr[ib->length_dw++] =0xFFFFFFFF ; + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK (1 << vmid) | +SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI (0x1F )); +} +static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = { .copy_max_bytes =0x400000 , .copy_num_dw =7 ,
@@ -2053,6 +2099,9 @@ static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = { .fill_max_bytes = 0x400000, .fill_num_dw = 5, .emit_fill_buffer = sdma_v5_0_emit_fill_buffer, + + .tlb_inv_num_dw = 4, + .emit_tlb_inv = sdma_v5_0_emit_tlb_inv, };static void sdma_v5_0_set_buffer_funcs (struct amdgpu_device *adev)
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 7/14 drm/amdgpu/sdma5.2: add tlb invalidation buffer func callback
In reply to Alex DeucherWill be used for TLB invalidation.
--- drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +49 -0
@@ -2053,6 +2053,52 @@ static void sdma_v5_2_emit_fill_buffer(struct amdgpu_ib *ib, ib->ptr[ib->length_dw++] = byte_count -1 ; } +/** +* sdma_v5_2_emit_tlb_inv - Invalidate TLB using the sDMA engine +* +* @adev: amdgpu device structure +* @ib: indirect buffer to fill +* @vmid: vmid to target +* @vmhub: vmhub to target +* @eng: invalidation engine to use +* @flush_type: type of flush (lightweight, heavyweight) +* @xcc_inst: XCC to target +* +* Invalidate TLB using the DMA engine. +*/ +static void sdma_v5_2_emit_tlb_inv (struct amdgpu_device *adev, +struct amdgpu_ib *ib, +unsigned int vmid, +u32 vmhub, +u32 eng, +u32 flush_type, +u32 xcc_inst) +{ +struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; +u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); +u32 mmhub_eng, gfxhub_eng; + +if (AMDGPU_IS_GFXHUB (vmhub)) { + mmhub_eng =0x1f ; + gfxhub_eng = eng; + }else { + mmhub_eng = eng; + gfxhub_eng =0x1f ; + } + +/* Trigger invalidation. */ + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_HEADER_OP (SDMA_OP_POLL_REGMEM ) | +SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP (SDMA_SUBOP_VM_INVALIDATION ) | +SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID (gfxhub_eng) | +SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID (mmhub_eng)); + ib->ptr[ib->length_dw++] = inv_req; + ib->ptr[ib->length_dw++] =0xFFFFFFFF ; + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK (1 << vmid) | +SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI (0x1F )); +} +static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = { .copy_max_bytes =1 <<30 , .copy_num_dw =7 ,
@@ -2061,6 +2107,9 @@ static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = { .fill_max_bytes = 1 << 30, /* HW supports 1 << 30, but PAL uses 1 << 22 */ .fill_num_dw = 5, .emit_fill_buffer = sdma_v5_2_emit_fill_buffer, + + .tlb_inv_num_dw = 4, + .emit_tlb_inv = sdma_v5_2_emit_tlb_inv, };static void sdma_v5_2_set_buffer_funcs (struct amdgpu_device *adev)
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 8/14 drm/amdgpu/sdma6: add tlb invalidation buffer func callback
In reply to Alex DeucherWill be used for TLB invalidation.
--- drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +49 -0
@@ -1861,6 +1861,52 @@ static void sdma_v6_0_emit_fill_buffer(struct amdgpu_ib *ib, ib->ptr[ib->length_dw++] = byte_count -1 ; } +/** +* sdma_v6_0_emit_tlb_inv - Invalidate TLB using the sDMA engine +* +* @adev: amdgpu device structure +* @ib: indirect buffer to fill +* @vmid: vmid to target +* @vmhub: vmhub to target +* @eng: invalidation engine to use +* @flush_type: type of flush (lightweight, heavyweight) +* @xcc_inst: XCC to target +* +* Invalidate TLB using the DMA engine. +*/ +static void sdma_v6_0_emit_tlb_inv (struct amdgpu_device *adev, +struct amdgpu_ib *ib, +unsigned int vmid, +u32 vmhub, +u32 eng, +u32 flush_type, +u32 xcc_inst) +{ +struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; +u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); +u32 mmhub_eng, gfxhub_eng; + +if (AMDGPU_IS_GFXHUB (vmhub)) { + mmhub_eng =0x1f ; + gfxhub_eng = eng; + }else { + mmhub_eng = eng; + gfxhub_eng =0x1f ; + } + +/* Trigger invalidation. */ + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_HEADER_OP (SDMA_OP_POLL_REGMEM ) | +SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP (SDMA_SUBOP_VM_INVALIDATION ) | +SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID (gfxhub_eng) | +SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID (mmhub_eng)); + ib->ptr[ib->length_dw++] = inv_req; + ib->ptr[ib->length_dw++] =0xFFFFFFFF ; + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK (1 << vmid) | +SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI (0x1F )); +} +static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = { .copy_max_bytes =1 <<30 , .copy_num_dw =7 ,
@@ -1869,6 +1915,9 @@ static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = { .fill_max_bytes = 1 << 30, .fill_num_dw = 5, .emit_fill_buffer = sdma_v6_0_emit_fill_buffer, + + .tlb_inv_num_dw = 4, + .emit_tlb_inv = sdma_v6_0_emit_tlb_inv, };static void sdma_v6_0_set_buffer_funcs (struct amdgpu_device *adev)
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 9/14 drm/amdgpu/sdma7: add tlb invalidation buffer func callback
In reply to Alex DeucherWill be used for TLB invalidation.
--- drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +48 -0
@@ -1811,6 +1811,52 @@ static void sdma_v7_0_emit_fill_buffer(struct amdgpu_ib *ib, ib->ptr[ib->length_dw++] = byte_count -1 ; } +/** +* sdma_v7_0_emit_tlb_inv - Invalidate TLB using the sDMA engine +* +* @adev: amdgpu device structure +* @ib: indirect buffer to fill +* @vmid: vmid to target +* @vmhub: vmhub to target +* @eng: invalidation engine to use +* @flush_type: type of flush (lightweight, heavyweight) +* @xcc_inst: XCC to target +* +* Invalidate TLB using the DMA engine. +*/ +static void sdma_v7_0_emit_tlb_inv (struct amdgpu_device *adev, +struct amdgpu_ib *ib, +unsigned int vmid, +u32 vmhub, +u32 eng, +u32 flush_type, +u32 xcc_inst) +{ +struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; +u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); +u32 mmhub_eng, gfxhub_eng; + +if (AMDGPU_IS_GFXHUB (vmhub)) { + mmhub_eng =0x1f ; + gfxhub_eng = eng; + }else { + mmhub_eng = eng; + gfxhub_eng =0x1f ; + } + +/* Trigger invalidation. */ + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_HEADER_OP (SDMA_OP_POLL_REGMEM ) | +SDMA_PKT_VM_INVALIDATION_HEADER_SUB_OP (SDMA_SUBOP_VM_INVALIDATION ) | +SDMA_PKT_VM_INVALIDATION_HEADER_GFX_ENG_ID (gfxhub_eng) | +SDMA_PKT_VM_INVALIDATION_HEADER_MM_ENG_ID (mmhub_eng)); + ib->ptr[ib->length_dw++] = inv_req; + ib->ptr[ib->length_dw++] =0xFFFFFFFF ; + ib->ptr[ib->length_dw++] = + (SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_INVALIDATEACK (1 << vmid) | +SDMA_PKT_VM_INVALIDATION_ADDRESSRANGEHI_ADDRESSRANGEHI (0x1F )); +} +static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = { .copy_max_bytes =1 <<30 , .copy_num_dw =8 ,
@@ -1818,6 +1864,8 @@ static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = { .fill_max_bytes = 1 << 30, .fill_num_dw = 5, .emit_fill_buffer = sdma_v7_0_emit_fill_buffer, + .tlb_inv_num_dw = 4, + .emit_tlb_inv = sdma_v7_0_emit_tlb_inv, };static void sdma_v7_0_set_buffer_funcs (struct amdgpu_device *adev)
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 10/14 drm/amdgpu: add core helper to do TLB invalidation via SDMA
In reply to Alex DeucherAdds a new helper to do TLB invalidation using SDMA.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 42 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++++ 4 files changed, 86 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +36 -0
@@ -928,6 +928,42 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,dev_err (adev->dev,"failed to write reg %x wait reg %x\n" , reg0, reg1); } +int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type) +{ +struct dma_fence *fence; +/* Use register 17 for GART */ +u32 inst, eng =17 ; +int r; + +if (AMDGPU_IS_GFXHUB (vmhub) && !adev->gfx.is_poweron) +return 0 ; + +if (vmhub >=AMDGPU_MMHUB0 (0 )) + inst =0 ; +else + inst = vmhub; + +/* flush hdp cache */ +amdgpu_device_flush_hdp (adev,NULL ); + +mutex_lock (&adev->mman.default_entity.lock); + r =amdgpu_ttm_tlb_inv (adev, vmid, vmhub, eng, flush_type, inst, + &adev->mman.default_entity, +NULL , &fence); +if (r) + goto exit; + r =dma_fence_wait (fence, false); +dma_fence_put (fence); +if (r) + goto exit; + +exit: +mutex_unlock (&adev->mman.default_entity.lock); + +return r; +} +/** * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ * @adev: amdgpu_device pointer
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2 -0
@@ -450,6 +450,8 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask, uint32_t xcc_inst); +int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type);extern void amdgpu_gmc_tmz_set (struct amdgpu_device *adev);extern void amdgpu_gmc_noretry_set (struct amdgpu_device *adev);
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +42 -0
48 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +6 -0
@@ -195,6 +195,12 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,struct dma_resv *resv, struct dma_fence **fence, bool vm_needs_flush, uint32_t copy_flags); +int amdgpu_ttm_tlb_inv (struct amdgpu_device *adev, +unsigned int vmid,u32 vmhub,u32 eng, +u32 flush_type,u32 xcc_inst, +struct amdgpu_ttm_buffer_entity *entity, +struct dma_resv *resv, +struct dma_fence **fence); int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, struct amdgpu_bo *bo, struct dma_resv *resv,
Signature · 1 line
2.55.0
Christian König
maintainerRe: [PATCH 10/14] drm/amdgpu: add core helper to do TLB invalidation via SDMA
Quoted 1 line · Alex Deucher
Adds a new helper to do TLB invalidation using SDMA.
We already have that. See function amdgpu_gmc_flush_gpu_tlb().
Regards,
Christian.
Quoted 139 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 42 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++++ 4 files changed, 86 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +36
-0@@ -928,6 +928,42 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,dev_err (adev->dev,"failed to write reg %x wait reg %x\n" , reg0, reg1); } +int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type) +{ +struct dma_fence *fence; +/* Use register 17 for GART */ +u32 inst, eng =17 ; +int r; + +if (AMDGPU_IS_GFXHUB (vmhub) && !adev->gfx.is_poweron) +return 0 ; + +if (vmhub >=AMDGPU_MMHUB0 (0 )) + inst =0 ; +else + inst = vmhub; + +/* flush hdp cache */ +amdgpu_device_flush_hdp (adev,NULL ); + +mutex_lock (&adev->mman.default_entity.lock); + r =amdgpu_ttm_tlb_inv (adev, vmid, vmhub, eng, flush_type, inst, + &adev->mman.default_entity, +NULL , &fence); +if (r) + goto exit; + r =dma_fence_wait (fence, false); +dma_fence_put (fence); +if (r) + goto exit; + +exit: +mutex_unlock (&adev->mman.default_entity.lock); + +return r; +} +/** * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ * @adev: amdgpu_device pointer drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2
-0@@ -450,6 +450,8 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask, uint32_t xcc_inst); +int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type);extern void amdgpu_gmc_tmz_set (struct amdgpu_device *adev);extern void amdgpu_gmc_noretry_set (struct amdgpu_device *adev);drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +42
-0@@ -2513,6 +2513,48 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,return r; } +int amdgpu_ttm_tlb_inv (struct amdgpu_device *adev, +unsigned int vmid,u32 vmhub,u32 eng, +u32 flush_type,u32 xcc_inst, +struct amdgpu_ttm_buffer_entity *entity, +struct dma_resv *resv, +struct dma_fence **fence) +{ +struct amdgpu_ring *ring; +struct amdgpu_job *job; +int r, num_dw; + +if (!adev->mman.buffer_funcs_enabled) +return -EINVAL ; + + ring =to_amdgpu_ring (adev->mman.buffer_funcs_scheds[0 ]); + +if (!ring->sched.ready) { +dev_err (adev->dev, +"Trying to inv tlbs with ring turned off.\n" ); +return -EINVAL ; + } + + num_dw =ALIGN (adev->mman.buffer_funcs->tlb_inv_num_dw,8 ); + r =amdgpu_ttm_prepare_job (adev, entity, num_dw, + resv, false, &job, +AMDGPU_KERNEL_JOB_ID_VM_UPDATE ); +if (r) + goto error_free; + +amdgpu_emit_tlb_inv (adev, &job->ibs[0 ], vmid, vmhub, eng, + flush_type, xcc_inst); + + *fence =amdgpu_ttm_job_submit (adev, entity, job, num_dw); + +return 0 ; + +error_free: +amdgpu_job_free (job); +dev_err (adev->dev,"Error scheduling IBs (%d)\n" , r); +return r; +} + static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entity *entity, uint32_t src_data,drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +6
-0@@ -195,6 +195,12 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,struct dma_resv *resv, struct dma_fence **fence, bool vm_needs_flush, uint32_t copy_flags); +int amdgpu_ttm_tlb_inv (struct amdgpu_device *adev, +unsigned int vmid,u32 vmhub,u32 eng, +u32 flush_type,u32 xcc_inst, +struct amdgpu_ttm_buffer_entity *entity, +struct dma_resv *resv, +struct dma_fence **fence); int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, struct amdgpu_bo *bo, struct dma_resv *resv,
Re: [PATCH 10/14] drm/amdgpu: add core helper to do TLB invalidation via SDMA
Quoted 5 lines · Christian König
Quoted 1 line · Alex Deucher
Adds a new helper to do TLB invalidation using SDMA.
We already have that. See function
amdgpu_gmc_flush_gpu_tlb().
This is a helper for the gmc IP components to replace their gmc specific callbacks that are used internally by amdgpu_gmc_flush_gpu_tlb(). See the last 3 patches in the series. This cleans up a lot of duplicated code in gmc10/11/12 and also replaces the use of MES for tlb invalidation with SDMA.
Alex
Quoted 144 lines
Regards,
Christian.Quoted 139 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 36 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 42 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 6 ++++ 4 files changed, 86 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +36
-0@@ -928,6 +928,42 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,dev_err (adev->dev,"failed to write reg %x wait reg %x\n" , reg0, reg1); } +int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type) +{ +struct dma_fence *fence; +/* Use register 17 for GART */ +u32 inst, eng =17 ; +int r; + +if (AMDGPU_IS_GFXHUB (vmhub) && !adev->gfx.is_poweron) +return 0 ; + +if (vmhub >=AMDGPU_MMHUB0 (0 )) + inst =0 ; +else + inst = vmhub; + +/* flush hdp cache */ +amdgpu_device_flush_hdp (adev,NULL ); + +mutex_lock (&adev->mman.default_entity.lock); + r =amdgpu_ttm_tlb_inv (adev, vmid, vmhub, eng, flush_type, inst, + &adev->mman.default_entity, +NULL , &fence); +if (r) + goto exit; + r =dma_fence_wait (fence, false); +dma_fence_put (fence); +if (r) + goto exit; + +exit: +mutex_unlock (&adev->mman.default_entity.lock); + +return r; +} +/** * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ * @adev: amdgpu_device pointer drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2
-0@@ -450,6 +450,8 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask, uint32_t xcc_inst); +int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type);extern void amdgpu_gmc_tmz_set (struct amdgpu_device *adev);extern void amdgpu_gmc_noretry_set (struct amdgpu_device *adev);drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +42
-0@@ -2513,6 +2513,48 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,return r; } +int amdgpu_ttm_tlb_inv (struct amdgpu_device *adev, +unsigned int vmid,u32 vmhub,u32 eng, +u32 flush_type,u32 xcc_inst, +struct amdgpu_ttm_buffer_entity *entity, +struct dma_resv *resv, +struct dma_fence **fence) +{ +struct amdgpu_ring *ring; +struct amdgpu_job *job; +int r, num_dw; + +if (!adev->mman.buffer_funcs_enabled) +return -EINVAL ; + + ring =to_amdgpu_ring (adev->mman.buffer_funcs_scheds[0 ]); + +if (!ring->sched.ready) { +dev_err (adev->dev, +"Trying to inv tlbs with ring turned off.\n" ); +return -EINVAL ; + } + + num_dw =ALIGN (adev->mman.buffer_funcs->tlb_inv_num_dw,8 ); + r =amdgpu_ttm_prepare_job (adev, entity, num_dw, + resv, false, &job, +AMDGPU_KERNEL_JOB_ID_VM_UPDATE ); +if (r) + goto error_free; + +amdgpu_emit_tlb_inv (adev, &job->ibs[0 ], vmid, vmhub, eng, + flush_type, xcc_inst); + + *fence =amdgpu_ttm_job_submit (adev, entity, job, num_dw); + +return 0 ; + +error_free: +amdgpu_job_free (job); +dev_err (adev->dev,"Error scheduling IBs (%d)\n" , r); +return r; +} + static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entity *entity, uint32_t src_data,drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +6
-0@@ -195,6 +195,12 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,struct dma_resv *resv, struct dma_fence **fence, bool vm_needs_flush, uint32_t copy_flags); +int amdgpu_ttm_tlb_inv (struct amdgpu_device *adev, +unsigned int vmid,u32 vmhub,u32 eng, +u32 flush_type,u32 xcc_inst, +struct amdgpu_ttm_buffer_entity *entity, +struct dma_resv *resv, +struct dma_fence **fence); int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity, struct amdgpu_bo *bo, struct dma_resv *resv,
Alex Deucher
maintainerPATCH 11/14 drm/amdgpu/gmc: add more gmc tlb inv helpers
In reply to Alex DeucherMove more of the common code into the gmc.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 172 +++++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 9 +- 2 files changed, 174 insertions(+), 7 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +167 -5
@@ -928,16 +928,158 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev,dev_err (adev->dev,"failed to write reg %x wait reg %x\n" , reg0, reg1); } -int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, -uint32_t vmhub,uint32_t flush_type) +static bool amdgpu_gmc_use_invalidate_semaphore (struct amdgpu_device *adev, +uint32_t vmhub) +{ +return (!AMDGPU_IS_GFXHUB (vmhub) && + !amdgpu_sriov_vf (adev)); +} + +uint64_t amdgpu_gmc_emit_flush_gpu_tlb_helper (struct amdgpu_ring *ring, +unsigned vmid,uint64_t pd_addr) +{ +bool use_semaphore =amdgpu_gmc_use_invalidate_semaphore (ring->adev, ring->vm_hub); +struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; +uint32_t req = hub->vmhub_funcs->get_invalidate_req (vmid,0 ); +unsigned eng = ring->vm_inv_eng; + +/* +* It may lose gpuvm invalidate acknowldege state across power-gating +* off cycle, add semaphore acquire before invalidation and semaphore +* release after invalidation to avoid entering power gated state +* to WA the Issue +*/ + +if (use_semaphore) +/* a read return value of 1 means semaphore acuqire */ +amdgpu_ring_emit_reg_wait (ring, + hub->vm_inv_eng0_sem + + hub->eng_distance * eng,0x1 ,0x1 ); + +amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_lo32 + + (hub->ctx_addr_distance * vmid), +lower_32_bits (pd_addr)); + +amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_hi32 + + (hub->ctx_addr_distance * vmid), +upper_32_bits (pd_addr)); + +amdgpu_ring_emit_reg_write_reg_wait (ring, hub->vm_inv_eng0_req + + hub->eng_distance * eng, + hub->vm_inv_eng0_ack + + hub->eng_distance * eng, + req,1 << vmid); + +if (use_semaphore) +/* +* add semaphore release after invalidation, +* write with 0 means semaphore release +*/ +amdgpu_ring_emit_wreg (ring, hub->vm_inv_eng0_sem + + hub->eng_distance * eng,0 ); + +return pd_addr; +} + +static void amdgpu_gmc_flush_vm_hub_mmio (struct amdgpu_device *adev, +uint32_t vmid,unsigned int vmhub, +uint32_t flush_type) +{ +bool use_semaphore =amdgpu_gmc_use_invalidate_semaphore (adev, vmhub); +struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; +u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); +u32 tmp; +/* Use register 17 for GART */ +const unsigned eng =17 ; +unsigned int i; +unsigned char hub_ip =0 ; + + hub_ip =AMDGPU_IS_GFXHUB (vmhub) ? +GC_HWIP :MMHUB_HWIP ; + +/* disabllow gfxoff when we invalidate */ +if (hub_ip ==GC_HWIP ) +amdgpu_gfx_off_ctrl (adev, false); + +spin_lock (&adev->gmc.invalidate_lock); +/* +* It may lose gpuvm invalidate acknowldege state across power-gating +* off cycle, add semaphore acquire before invalidation and semaphore +* release after invalidation to avoid entering power gated state +* to WA the Issue +*/ +if (use_semaphore) { +for (i =0 ; i < adev->usec_timeout; i++) { +/* a read return value of 1 means semaphore acuqire */ + tmp =RREG32_RLC_NO_KIQ (hub->vm_inv_eng0_sem + + hub->eng_distance * eng, hub_ip); +if (tmp &0x1 ) +break ; +udelay (1 ); + } + +if (i >= adev->usec_timeout) +dev_err (adev->dev, +"Timeout waiting for sem acquire in VM flush!\n" ); + } + +WREG32_RLC_NO_KIQ (hub->vm_inv_eng0_req + hub->eng_distance * eng, + inv_req, hub_ip); + +/* Wait for ACK with a delay.*/ +for (i =0 ; i < adev->usec_timeout; i++) { + tmp =RREG32_RLC_NO_KIQ (hub->vm_inv_eng0_ack + + hub->eng_distance * eng, hub_ip); + tmp &=1 << vmid; +if (tmp) +break ; + +udelay (1 ); + } + +if (use_semaphore) +/* +* add semaphore release after invalidation, +* write with 0 means semaphore release +*/ +WREG32_RLC_NO_KIQ (hub->vm_inv_eng0_sem + + hub->eng_distance * eng,0 , hub_ip); + +/* Issue additional private vm invalidation to MMHUB */ +if (!AMDGPU_IS_GFXHUB (vmhub) && + hub->vm_l2_bank_select_reserved_cid2 && + !amdgpu_sriov_vf (adev)) { + inv_req =RREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2); +/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ + inv_req |= (1 <<25 ); +/* Issue private invalidation */ +WREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2, inv_req); +/* Read back to ensure invalidation is done*/ +RREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2); + } + +spin_unlock (&adev->gmc.invalidate_lock); + +if (hub_ip ==GC_HWIP ) +amdgpu_gfx_off_ctrl (adev, true); + +if (i < adev->usec_timeout) +return ; + +dev_err (adev->dev,"Timeout waiting for VM flush ACK!\n" ); +} + +void amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type) {struct dma_fence *fence;/* Use register 17 for GART */ -u32 inst, eng =17 ; +const unsigned eng =17 ; +u32 inst;int r;if (AMDGPU_IS_GFXHUB (vmhub) && !adev->gfx.is_poweron) -return 0 ; +return ;if (vmhub >=AMDGPU_MMHUB0 (0 )) inst =0 ;
@@ -947,6 +1089,7 @@ int amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device *adev, uint32_t vmid,/* flush hdp cache */ amdgpu_device_flush_hdp (adev,NULL ); +/* use SDMA */ mutex_lock (&adev->mman.default_entity.lock); r = amdgpu_ttm_tlb_inv(adev, vmid, vmhub, eng, flush_type, inst, &adev->mman.default_entity,
@@ -961,7 +1104,26 @@ int amdgpu_gmc_flush_gpu_tlb_helper(struct amdgpu_device *adev, uint32_t vmid, exit:mutex_unlock (&adev->mman.default_entity.lock); -return r; +/* if SDMA fails, use MMIO */ +if (r) +amdgpu_gmc_flush_vm_hub_mmio (adev, vmid, vmhub, flush_type); +} + +void amdgpu_gmc_flush_gpu_tlb_pasid_helper (struct amdgpu_device *adev, +unsigned int vmid,uint32_t flush_type, +bool all_hub,uint32_t inst) +{ +int i; + +if (all_hub) { +for_each_set_bit (i, adev->vmhubs_mask,AMDGPU_MAX_VMHUBS ) +amdgpu_gmc_flush_gpu_tlb_helper (adev, vmid, i, + flush_type); + }else { +amdgpu_gmc_flush_gpu_tlb_helper (adev, vmid, +AMDGPU_GFXHUB (inst), + flush_type); + } }/**
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +7 -2
@@ -450,8 +450,13 @@ void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, uint32_t reg0, uint32_t reg1, uint32_t ref, uint32_t mask, uint32_t xcc_inst); -int amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, -uint32_t vmhub,uint32_t flush_type); +uint64_t amdgpu_gmc_emit_flush_gpu_tlb_helper (struct amdgpu_ring *ring, +unsigned vmid,uint64_t pd_addr); +void amdgpu_gmc_flush_gpu_tlb_helper (struct amdgpu_device *adev,uint32_t vmid, +uint32_t vmhub,uint32_t flush_type); +void amdgpu_gmc_flush_gpu_tlb_pasid_helper (struct amdgpu_device *adev, +unsigned int vmid,uint32_t flush_type, +bool all_hub,uint32_t inst);extern void amdgpu_gmc_tmz_set (struct amdgpu_device *adev);extern void amdgpu_gmc_noretry_set (struct amdgpu_device *adev);
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 12/14 drm/amdgpu/gmc10: switch to new gmc tlb inv helpers
In reply to Alex DeucherUse the new gmc helpers for tlb invalidations.
--- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 186 +------------------------ 1 file changed, 7 insertions(+), 179 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +7 -179
@@ -188,20 +188,6 @@ static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev) } } -/** -* gmc_v10_0_use_invalidate_semaphore - judge whether to use semaphore -* -* @adev: amdgpu_device pointer -* @vmhub: vmhub type -* -*/ -static bool gmc_v10_0_use_invalidate_semaphore (struct amdgpu_device *adev, -uint32_t vmhub) -{ -return ((vmhub ==AMDGPU_MMHUB0 (0 )) && - (!amdgpu_sriov_vf (adev))); -} -static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info (struct amdgpu_device *adev,uint8_t vmid,uint16_t *p_pasid)
@@ -222,109 +208,6 @@ static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info(* by the amdgpu vm/hsa code. */ -/** -* gmc_v10_0_flush_gpu_tlb - gart tlb flush callback -* -* @adev: amdgpu_device pointer -* @vmid: vm instance to flush -* @vmhub: vmhub type -* @flush_type: the flush type -* -* Flush the TLB for the requested page table. -*/ -static void gmc_v10_0_flush_gpu_tlb (struct amdgpu_device *adev,uint32_t vmid, -uint32_t vmhub,uint32_t flush_type) -{ -bool use_semaphore =gmc_v10_0_use_invalidate_semaphore (adev, vmhub); -struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; -u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); -/* Use register 17 for GART */ -const unsigned int eng =17 ; -unsigned char hub_ip =0 ; -u32 sem, req, ack; -unsigned int i; -u32 tmp; - - sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng; - req = hub->vm_inv_eng0_req + hub->eng_distance * eng; - ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; - -/* flush hdp cache */ -amdgpu_device_flush_hdp (adev,NULL ); - -/* This is necessary for SRIOV as well as for GFXOFF to function -* properly under bare metal -*/ -if (adev->gfx.kiq[0 ].ring.sched.ready && !adev->enable_mes && - (amdgpu_sriov_runtime (adev) || !amdgpu_sriov_vf (adev))) { -amdgpu_gmc_fw_reg_write_reg_wait (adev, req, ack, inv_req, -1 << vmid,GET_INST (GC ,0 )); -return ; - } - -/* This path is needed before KIQ/MES/GFXOFF are set up */ - hub_ip = (vmhub ==AMDGPU_GFXHUB (0 )) ?GC_HWIP :MMHUB_HWIP ; - -/* disabllow gfxoff when we invalidate */ -if (hub_ip ==GC_HWIP ) -amdgpu_gfx_off_ctrl (adev, false); - -spin_lock (&adev->gmc.invalidate_lock); -/* -* It may lose gpuvm invalidate acknowldege state across power-gating -* off cycle, add semaphore acquire before invalidation and semaphore -* release after invalidation to avoid entering power gated state -* to WA the Issue -*/ - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) { -for (i =0 ; i < adev->usec_timeout; i++) { -/* a read return value of 1 means semaphore acuqire */ - tmp =RREG32_RLC_NO_KIQ (sem, hub_ip); -if (tmp &0x1 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("Timeout waiting for sem acquire in VM flush!\n" ); - } - -WREG32_RLC_NO_KIQ (req, inv_req, hub_ip); - -/* -* Issue a dummy read to wait for the ACK register to be cleared -* to avoid a false ACK due to the new fast GRBM interface. -*/ -if ((vmhub ==AMDGPU_GFXHUB (0 )) && - (amdgpu_ip_version (adev,GC_HWIP ,0 ) <IP_VERSION (10 ,3 ,0 ))) -RREG32_RLC_NO_KIQ (req, hub_ip); - -/* Wait for ACK with a delay.*/ -for (i =0 ; i < adev->usec_timeout; i++) { - tmp =RREG32_RLC_NO_KIQ (ack, hub_ip); - tmp &=1 << vmid; -if (tmp) -break ; - -udelay (1 ); - } - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -WREG32_RLC_NO_KIQ (sem,0 , hub_ip); - -spin_unlock (&adev->gmc.invalidate_lock); - -if (hub_ip ==GC_HWIP ) -amdgpu_gfx_off_ctrl (adev, true); - -if (i >= adev->usec_timeout) -dev_err (adev->dev,"Timeout waiting for VM flush hub: %d!\n" , - vmhub); -} -/** * gmc_v10_0_flush_gpu_tlb_pasid - tlb flush via pasid *
@@ -341,7 +224,7 @@ static void gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, bool all_hub, uint32_t inst) {uint16_t queried; -int vmid, i; +int vmid;for (vmid =1 ; vmid <AMDGPU_NUM_VMID ; vmid++) {bool valid;
@@ -351,66 +234,11 @@ static void gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,if (!valid || queried != pasid)continue ; -if (all_hub) { -for_each_set_bit (i, adev->vmhubs_mask, -AMDGPU_MAX_VMHUBS ) -gmc_v10_0_flush_gpu_tlb (adev, vmid, i, - flush_type); - }else { -gmc_v10_0_flush_gpu_tlb (adev, vmid,AMDGPU_GFXHUB (0 ), - flush_type); - } +amdgpu_gmc_flush_gpu_tlb_pasid_helper (adev, vmid, flush_type, + all_hub, inst); } } -static uint64_t gmc_v10_0_emit_flush_gpu_tlb (struct amdgpu_ring *ring, -unsigned int vmid,uint64_t pd_addr) -{ -bool use_semaphore =gmc_v10_0_use_invalidate_semaphore (ring->adev, ring->vm_hub); -struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; -uint32_t req = hub->vmhub_funcs->get_invalidate_req (vmid,0 ); -unsigned int eng = ring->vm_inv_eng; - -/* -* It may lose gpuvm invalidate acknowldege state across power-gating -* off cycle, add semaphore acquire before invalidation and semaphore -* release after invalidation to avoid entering power gated state -* to WA the Issue -*/ - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* a read return value of 1 means semaphore acuqire */ -amdgpu_ring_emit_reg_wait (ring, - hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0x1 ,0x1 ); - -amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_lo32 + - (hub->ctx_addr_distance * vmid), -lower_32_bits (pd_addr)); - -amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_hi32 + - (hub->ctx_addr_distance * vmid), -upper_32_bits (pd_addr)); - -amdgpu_ring_emit_reg_write_reg_wait (ring, hub->vm_inv_eng0_req + - hub->eng_distance * eng, - hub->vm_inv_eng0_ack + - hub->eng_distance * eng, - req,1 << vmid); - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* -* add semaphore release after invalidation, -* write with 0 means semaphore release -*/ -amdgpu_ring_emit_wreg (ring, hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0 ); - -return pd_addr; -} -static void gmc_v10_0_emit_pasid_mapping (struct amdgpu_ring *ring,unsigned int vmid,unsigned int pasid) {
@@ -551,9 +379,9 @@ static unsigned int gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev) }static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = { - .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb, + .flush_gpu_tlb = amdgpu_gmc_flush_gpu_tlb_helper, .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid, - .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb, + .emit_flush_gpu_tlb = amdgpu_gmc_emit_flush_gpu_tlb_helper, .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping, .get_vm_pde = gmc_v10_0_get_vm_pde, .get_vm_pte = gmc_v10_0_get_vm_pte,
@@ -955,9 +783,9 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)if (!adev->in_s0ix) adev->gfxhub.funcs->set_fault_enable_default (adev, value); adev->mmhub.funcs->set_fault_enable_default (adev, value); -gmc_v10_0_flush_gpu_tlb (adev,0 ,AMDGPU_MMHUB0 (0 ),0 ); + adev->gmc.gmc_funcs->flush_gpu_tlb (adev,0 ,AMDGPU_MMHUB0 (0 ),0 );if (!adev->in_s0ix) -gmc_v10_0_flush_gpu_tlb (adev,0 ,AMDGPU_GFXHUB (0 ),0 ); + adev->gmc.gmc_funcs->flush_gpu_tlb (adev,0 ,AMDGPU_MMHUB0 (0 ),0 ); drm_info(adev_to_drm(adev), "PCIE GART of %uM enabled (table at 0x%016llX).\n", (unsigned int)(adev->gmc.gart_size >> 20),
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 13/14 drm/amdgpu/gmc11: switch to new gmc tlb inv helpers
In reply to Alex DeucherUse the new gmc helpers for tlb invalidations.
--- drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 191 +------------------------ 1 file changed, 6 insertions(+), 185 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +6 -185
@@ -184,20 +184,6 @@ static void gmc_v11_0_set_irq_funcs(struct amdgpu_device *adev) } } -/** -* gmc_v11_0_use_invalidate_semaphore - judge whether to use semaphore -* -* @adev: amdgpu_device pointer -* @vmhub: vmhub type -* -*/ -static bool gmc_v11_0_use_invalidate_semaphore (struct amdgpu_device *adev, -uint32_t vmhub) -{ -return ((vmhub ==AMDGPU_MMHUB0 (0 )) && - (!amdgpu_sriov_vf (adev))); -} -static bool gmc_v11_0_get_vmid_pasid_mapping_info (struct amdgpu_device *adev,uint8_t vmid,uint16_t *p_pasid)
@@ -207,116 +193,6 @@ static bool gmc_v11_0_get_vmid_pasid_mapping_info(return !!(*p_pasid); } -/** -* gmc_v11_0_flush_gpu_tlb - gart tlb flush callback -* -* @adev: amdgpu_device pointer -* @vmid: vm instance to flush -* @vmhub: which hub to flush -* @flush_type: the flush type -* -* Flush the TLB for the requested page table. -*/ -static void gmc_v11_0_flush_gpu_tlb (struct amdgpu_device *adev,uint32_t vmid, -uint32_t vmhub,uint32_t flush_type) -{ -bool use_semaphore =gmc_v11_0_use_invalidate_semaphore (adev, vmhub); -struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; -u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); -/* Use register 17 for GART */ -const unsigned int eng =17 ; -unsigned char hub_ip; -u32 sem, req, ack; -unsigned int i; -u32 tmp; - -if ((vmhub ==AMDGPU_GFXHUB (0 )) && !adev->gfx.is_poweron) -return ; - - sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng; - req = hub->vm_inv_eng0_req + hub->eng_distance * eng; - ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; - -/* flush hdp cache */ -amdgpu_device_flush_hdp (adev,NULL ); - -/* This is necessary for SRIOV as well as for GFXOFF to function -* properly under bare metal -*/ -if ((adev->gfx.kiq[0 ].ring.sched.ready || adev->mes.ring[0 ].sched.ready) && - (amdgpu_sriov_runtime (adev) || !amdgpu_sriov_vf (adev))) { -amdgpu_gmc_fw_reg_write_reg_wait (adev, req, ack, inv_req, -1 << vmid,GET_INST (GC ,0 )); -return ; - } - -/* This path is needed before KIQ/MES/GFXOFF are set up */ - hub_ip = (vmhub ==AMDGPU_GFXHUB (0 )) ?GC_HWIP :MMHUB_HWIP ; - -/* disabllow gfxoff when we invalidate */ -if (hub_ip ==GC_HWIP ) -amdgpu_gfx_off_ctrl (adev, false); - -spin_lock (&adev->gmc.invalidate_lock); -/* -* It may lose gpuvm invalidate acknowldege state across power-gating -* off cycle, add semaphore acquire before invalidation and semaphore -* release after invalidation to avoid entering power gated state -* to WA the Issue -*/ - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) { -for (i =0 ; i < adev->usec_timeout; i++) { -/* a read return value of 1 means semaphore acuqire */ - tmp =RREG32_RLC_NO_KIQ (sem, hub_ip); -if (tmp &0x1 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -DRM_ERROR ("Timeout waiting for sem acquire in VM flush!\n" ); - } - -WREG32_RLC_NO_KIQ (req, inv_req, hub_ip); - -/* Wait for ACK with a delay.*/ -for (i =0 ; i < adev->usec_timeout; i++) { - tmp =RREG32_RLC_NO_KIQ (ack, hub_ip); - tmp &=1 << vmid; -if (tmp) -break ; - -udelay (1 ); - } - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -WREG32_RLC_NO_KIQ (sem,0 , hub_ip); - -/* Issue additional private vm invalidation to MMHUB */ -if ((vmhub !=AMDGPU_GFXHUB (0 )) && - (hub->vm_l2_bank_select_reserved_cid2) && - !amdgpu_sriov_vf (adev)) { - inv_req =RREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2); -/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ - inv_req |= (1 <<25 ); -/* Issue private invalidation */ -WREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2, inv_req); -/* Read back to ensure invalidation is done*/ -RREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2); - } - -spin_unlock (&adev->gmc.invalidate_lock); - -if (hub_ip ==GC_HWIP ) -amdgpu_gfx_off_ctrl (adev, true); - -if (i >= adev->usec_timeout) -dev_err (adev->dev,"Timeout waiting for VM flush ACK!\n" ); -} -/** * gmc_v11_0_flush_gpu_tlb_pasid - tlb flush via pasid *
@@ -333,7 +209,7 @@ static void gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, bool all_hub, uint32_t inst) {uint16_t queried; -int vmid, i; +int vmid;for (vmid =1 ; vmid <16 ; vmid++) {bool valid;
@@ -343,66 +219,11 @@ static void gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,if (!valid || queried != pasid)continue ; -if (all_hub) { -for_each_set_bit (i, adev->vmhubs_mask, -AMDGPU_MAX_VMHUBS ) -gmc_v11_0_flush_gpu_tlb (adev, vmid, i, - flush_type); - }else { -gmc_v11_0_flush_gpu_tlb (adev, vmid,AMDGPU_GFXHUB (0 ), - flush_type); - } +amdgpu_gmc_flush_gpu_tlb_pasid_helper (adev, vmid, flush_type, + all_hub, inst); } } -static uint64_t gmc_v11_0_emit_flush_gpu_tlb (struct amdgpu_ring *ring, -unsigned int vmid,uint64_t pd_addr) -{ -bool use_semaphore =gmc_v11_0_use_invalidate_semaphore (ring->adev, ring->vm_hub); -struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; -uint32_t req = hub->vmhub_funcs->get_invalidate_req (vmid,0 ); -unsigned int eng = ring->vm_inv_eng; - -/* -* It may lose gpuvm invalidate acknowldege state across power-gating -* off cycle, add semaphore acquire before invalidation and semaphore -* release after invalidation to avoid entering power gated state -* to WA the Issue -*/ - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* a read return value of 1 means semaphore acuqire */ -amdgpu_ring_emit_reg_wait (ring, - hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0x1 ,0x1 ); - -amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_lo32 + - (hub->ctx_addr_distance * vmid), -lower_32_bits (pd_addr)); - -amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_hi32 + - (hub->ctx_addr_distance * vmid), -upper_32_bits (pd_addr)); - -amdgpu_ring_emit_reg_write_reg_wait (ring, hub->vm_inv_eng0_req + - hub->eng_distance * eng, - hub->vm_inv_eng0_ack + - hub->eng_distance * eng, - req,1 << vmid); - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* -* add semaphore release after invalidation, -* write with 0 means semaphore release -*/ -amdgpu_ring_emit_wreg (ring, hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0 ); - -return pd_addr; -} -static void gmc_v11_0_emit_pasid_mapping (struct amdgpu_ring *ring,unsigned int vmid,unsigned int pasid) {
@@ -542,9 +363,9 @@ static unsigned int gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev) }static const struct amdgpu_gmc_funcs gmc_v11_0_gmc_funcs = { - .flush_gpu_tlb = gmc_v11_0_flush_gpu_tlb, + .flush_gpu_tlb = amdgpu_gmc_flush_gpu_tlb_helper, .flush_gpu_tlb_pasid = gmc_v11_0_flush_gpu_tlb_pasid, - .emit_flush_gpu_tlb = gmc_v11_0_emit_flush_gpu_tlb, + .emit_flush_gpu_tlb = amdgpu_gmc_emit_flush_gpu_tlb_helper, .emit_pasid_mapping = gmc_v11_0_emit_pasid_mapping, .get_vm_pde = gmc_v11_0_get_vm_pde, .get_vm_pte = gmc_v11_0_get_vm_pte,
@@ -933,7 +754,7 @@ static int gmc_v11_0_gart_enable(struct amdgpu_device *adev) value = amdgpu_vm_fault_stop !=AMDGPU_VM_FAULT_STOP_ALWAYS ; adev->mmhub.funcs->set_fault_enable_default (adev, value); -gmc_v11_0_flush_gpu_tlb (adev,0 ,AMDGPU_MMHUB0 (0 ),0 ); + adev->gmc.gmc_funcs->flush_gpu_tlb (adev,0 ,AMDGPU_MMHUB0 (0 ),0 ); drm_info(adev_to_drm(adev), "PCIE GART of %uM enabled (table at 0x%016llX).\n", (unsigned int)(adev->gmc.gart_size >> 20),
Signature · 1 line
2.55.0
Alex Deucher
maintainerPATCH 14/14 drm/amdgpu/gmc12: switch to new gmc tlb inv helpers
In reply to Alex DeucherUse the new gmc helpers for tlb invalidations.
--- drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 213 +------------------------ 1 file changed, 5 insertions(+), 208 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +5 -208
@@ -180,20 +180,6 @@ static void gmc_v12_0_set_irq_funcs(struct amdgpu_device *adev) } } -/** -* gmc_v12_0_use_invalidate_semaphore - judge whether to use semaphore -* -* @adev: amdgpu_device pointer -* @vmhub: vmhub type -* -*/ -static bool gmc_v12_0_use_invalidate_semaphore (struct amdgpu_device *adev, -uint32_t vmhub) -{ -return ((vmhub ==AMDGPU_MMHUB0 (0 )) && - (!amdgpu_sriov_vf (adev))); -} -static bool gmc_v12_0_get_vmid_pasid_mapping_info (struct amdgpu_device *adev,uint8_t vmid,uint16_t *p_pasid)
@@ -203,140 +189,6 @@ static bool gmc_v12_0_get_vmid_pasid_mapping_info(return !!(*p_pasid); } -/* -* GART -* VMID 0 is the physical GPU addresses as used by the kernel. -* VMIDs 1-15 are used for userspace clients and are handled -* by the amdgpu vm/hsa code. -*/ - -static void gmc_v12_0_flush_vm_hub (struct amdgpu_device *adev,uint32_t vmid, -unsigned int vmhub,uint32_t flush_type) -{ -bool use_semaphore =gmc_v12_0_use_invalidate_semaphore (adev, vmhub); -struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; -u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); -u32 tmp; -/* Use register 17 for GART */ -const unsigned eng =17 ; -unsigned int i; -unsigned char hub_ip =0 ; - - hub_ip = (vmhub ==AMDGPU_GFXHUB (0 )) ? -GC_HWIP :MMHUB_HWIP ; - -spin_lock (&adev->gmc.invalidate_lock); -/* -* It may lose gpuvm invalidate acknowldege state across power-gating -* off cycle, add semaphore acquire before invalidation and semaphore -* release after invalidation to avoid entering power gated state -* to WA the Issue -*/ - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) { -for (i =0 ; i < adev->usec_timeout; i++) { -/* a read return value of 1 means semaphore acuqire */ - tmp =RREG32_RLC_NO_KIQ (hub->vm_inv_eng0_sem + - hub->eng_distance * eng, hub_ip); -if (tmp &0x1 ) -break ; -udelay (1 ); - } - -if (i >= adev->usec_timeout) -dev_err (adev->dev, -"Timeout waiting for sem acquire in VM flush!\n" ); - } - -WREG32_RLC_NO_KIQ (hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip); - -/* Wait for ACK with a delay.*/ -for (i =0 ; i < adev->usec_timeout; i++) { - tmp =RREG32_RLC_NO_KIQ (hub->vm_inv_eng0_ack + - hub->eng_distance * eng, hub_ip); - tmp &=1 << vmid; -if (tmp) -break ; - -udelay (1 ); - } - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* -* add semaphore release after invalidation, -* write with 0 means semaphore release -*/ -WREG32_RLC_NO_KIQ (hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0 , hub_ip); - -/* Issue additional private vm invalidation to MMHUB */ -if ((vmhub !=AMDGPU_GFXHUB (0 )) && - (hub->vm_l2_bank_select_reserved_cid2) && - !amdgpu_sriov_vf (adev)) { - inv_req =RREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2); -/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ - inv_req |= (1 <<25 ); -/* Issue private invalidation */ -WREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2, inv_req); -/* Read back to ensure invalidation is done*/ -RREG32_NO_KIQ (hub->vm_l2_bank_select_reserved_cid2); - } - -spin_unlock (&adev->gmc.invalidate_lock); - -if (i < adev->usec_timeout) -return ; - -dev_err (adev->dev,"Timeout waiting for VM flush ACK!\n" ); -} - -/** -* gmc_v12_0_flush_gpu_tlb - gart tlb flush callback -* -* @adev: amdgpu_device pointer -* @vmid: vm instance to flush -* @vmhub: which hub to flush -* @flush_type: the flush type -* -* Flush the TLB for the requested page table. -*/ -static void gmc_v12_0_flush_gpu_tlb (struct amdgpu_device *adev,uint32_t vmid, -uint32_t vmhub,uint32_t flush_type) -{ -if ((vmhub ==AMDGPU_GFXHUB (0 )) && !adev->gfx.is_poweron) -return ; - -/* flush hdp cache */ -amdgpu_device_flush_hdp (adev,NULL ); - -/* This is necessary for SRIOV as well as for GFXOFF to function -* properly under bare metal -*/ -if ((adev->gfx.kiq[0 ].ring.sched.ready || adev->mes.ring[0 ].sched.ready) && - (amdgpu_sriov_runtime (adev) || !amdgpu_sriov_vf (adev))) { -struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; -const unsigned eng =17 ; -u32 inv_req = hub->vmhub_funcs->get_invalidate_req (vmid, flush_type); -u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; -u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; - -amdgpu_gmc_fw_reg_write_reg_wait (adev, req, ack, inv_req, -1 << vmid,GET_INST (GC ,0 )); -return ; - } - -/* disabllow gfxoff when we invalidate */ -if (vmhub ==AMDGPU_GFXHUB (0 )) -amdgpu_gfx_off_ctrl (adev, false); - -gmc_v12_0_flush_vm_hub (adev, vmid, vmhub,0 ); - -if (vmhub ==AMDGPU_GFXHUB (0 )) -amdgpu_gfx_off_ctrl (adev, true); -} -/** * gmc_v12_0_flush_gpu_tlb_pasid - tlb flush via pasid *
@@ -353,7 +205,7 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, bool all_hub, uint32_t inst) { uint16_t queried; -int vmid, i; +int vmid;if (adev->enable_uni_mes && adev->mes.ring[AMDGPU_MES_SCHED_PIPE ].sched.ready && (adev->mes.sched_version &AMDGPU_MES_VERSION_MASK ) >=0x84 ) {
@@ -379,66 +231,11 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,if (!valid || queried != pasid)continue ; -if (all_hub) { -for_each_set_bit (i, adev->vmhubs_mask, -AMDGPU_MAX_VMHUBS ) -gmc_v12_0_flush_gpu_tlb (adev, vmid, i, - flush_type); - }else { -gmc_v12_0_flush_gpu_tlb (adev, vmid,AMDGPU_GFXHUB (0 ), - flush_type); - } +amdgpu_gmc_flush_gpu_tlb_pasid_helper (adev, vmid, flush_type, + all_hub, inst); } } -static uint64_t gmc_v12_0_emit_flush_gpu_tlb (struct amdgpu_ring *ring, -unsigned vmid,uint64_t pd_addr) -{ -bool use_semaphore =gmc_v12_0_use_invalidate_semaphore (ring->adev, ring->vm_hub); -struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; -uint32_t req = hub->vmhub_funcs->get_invalidate_req (vmid,0 ); -unsigned eng = ring->vm_inv_eng; - -/* -* It may lose gpuvm invalidate acknowldege state across power-gating -* off cycle, add semaphore acquire before invalidation and semaphore -* release after invalidation to avoid entering power gated state -* to WA the Issue -*/ - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* a read return value of 1 means semaphore acuqire */ -amdgpu_ring_emit_reg_wait (ring, - hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0x1 ,0x1 ); - -amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_lo32 + - (hub->ctx_addr_distance * vmid), -lower_32_bits (pd_addr)); - -amdgpu_ring_emit_wreg (ring, hub->ctx0_ptb_addr_hi32 + - (hub->ctx_addr_distance * vmid), -upper_32_bits (pd_addr)); - -amdgpu_ring_emit_reg_write_reg_wait (ring, hub->vm_inv_eng0_req + - hub->eng_distance * eng, - hub->vm_inv_eng0_ack + - hub->eng_distance * eng, - req,1 << vmid); - -/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ -if (use_semaphore) -/* -* add semaphore release after invalidation, -* write with 0 means semaphore release -*/ -amdgpu_ring_emit_wreg (ring, hub->vm_inv_eng0_sem + - hub->eng_distance * eng,0 ); - -return pd_addr; -} -static void gmc_v12_0_emit_pasid_mapping (struct amdgpu_ring *ring,unsigned vmid,unsigned pasid) {
@@ -577,9 +374,9 @@ static unsigned int gmc_v12_0_get_dcc_alignment(struct amdgpu_device *adev) }static const struct amdgpu_gmc_funcs gmc_v12_0_gmc_funcs = { - .flush_gpu_tlb = gmc_v12_0_flush_gpu_tlb, + .flush_gpu_tlb = amdgpu_gmc_flush_gpu_tlb_helper, .flush_gpu_tlb_pasid = gmc_v12_0_flush_gpu_tlb_pasid, - .emit_flush_gpu_tlb = gmc_v12_0_emit_flush_gpu_tlb, + .emit_flush_gpu_tlb = amdgpu_gmc_emit_flush_gpu_tlb_helper, .emit_pasid_mapping = gmc_v12_0_emit_pasid_mapping, .get_vm_pde = gmc_v12_0_get_vm_pde, .get_vm_pte = gmc_v12_0_get_vm_pte,
Signature · 1 line
2.55.0
Christian König
maintainerQuoted 3 lines · Alex Deucher
We need to disallow gfxoff if we touch GC MMIO registers. At the moment we use KIQ or MES for TLB flushes so no intended functional change.
IIRC we actually tried this before and reverted it because it caused quite a bit of trouble.
The final solution was to use the semaphore to prevent GFXOFF from happening.
Regards,
Christian.
Quoted 31 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +7
-0@@ -808,6 +808,10 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,return ; } +/* disabllow gfxoff when we invalidate */ +if (vmhub <AMDGPU_MMHUB0 (0 )) +amdgpu_gfx_off_ctrl (adev, false); +/* This path is needed before KIQ/MES/GFXOFF are set up */ spin_lock (&adev->gmc.invalidate_lock);@@ -873,6 +877,9 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,spin_unlock (&adev->gmc.invalidate_lock); +if (vmhub <AMDGPU_MMHUB0 (0 )) +amdgpu_gfx_off_ctrl (adev, true); +if (j < adev->usec_timeout)return ;
Quoted 9 lines · Christian König
Quoted 3 lines · Alex Deucher
We need to disallow gfxoff if we touch GC MMIO registers. At the moment we use KIQ or MES for TLB flushes so no intended functional change.
IIRC we actually tried this before and reverted it because it caused quite a bit of trouble.
The final solution was to use the semaphore to prevent GFXOFF from happening.
I think you need to disallow gfxoff to access the semaphore registers via MMIO. The semaphore registers just prevent the smu from entering gfxoff while they are held. I think the problem was that the smu would enter gfxoff while the CP/SDMA was waiting for the ACK which caused the ACK to never signal.
Alex
Quoted 36 lines
Regards,
Christian.Quoted 31 lines
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 7 +++++++ 1 file changed, 7 insertions(+)drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +7
-0@@ -808,6 +808,10 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,return ; } +/* disabllow gfxoff when we invalidate */ +if (vmhub <AMDGPU_MMHUB0 (0 )) +amdgpu_gfx_off_ctrl (adev, false); +/* This path is needed before KIQ/MES/GFXOFF are set up */ spin_lock (&adev->gmc.invalidate_lock);@@ -873,6 +877,9 @@ static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,spin_unlock (&adev->gmc.invalidate_lock); +if (vmhub <AMDGPU_MMHUB0 (0 )) +amdgpu_gfx_off_ctrl (adev, true); +if (j < adev->usec_timeout)return ;