drm/amdgpu: reserve GDS resources on the gfx ring for gfx10
Subsystems (2)
PATCH 1/1 drm/amdgpu: reserve GDS resources on the gfx ring for gfx10
From: Marek Olšák <[email protected]>
Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.
This is a workaround to ensure stability of transform feedback. Shaders hang waiting for a GDS instruction (ds_sub, not ds_ordered_count).
The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 10 ++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {struct drm_file *filp;struct amdgpu_ctx *ctx;/* chunks */ unsigned nchunks;struct amdgpu_cs_chunk *chunks;/* scheduler job object */ struct amdgpu_job *job;struct drm_sched_entity *entity; +unsigned hw_ip;/* buffer objects */ struct ww_acquire_ctx ticket;struct amdgpu_bo_list *bo_list;struct amdgpu_mn *mn;struct amdgpu_bo_list_entry vm_pd;struct list_head validated;struct dma_fence *fence;uint64_t bytes_moved_threshold;uint64_t bytes_moved_vis_threshold;
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +10 -0
@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,if (r) goto error_validate;amdgpu_cs_report_moved_bytes (p->adev, p->bytes_moved, p->bytes_moved_vis); gds = p->bo_list->gds_obj; gws = p->bo_list->gws_obj; oa = p->bo_list->oa_obj; +if (p->hw_ip ==AMDGPU_HW_IP_GFX ) { +/* Only gfx10 allocates these. */ +if (!gds) + gds = p->adev->gds.gds_gfx_bo; +if (!oa) + oa = p->adev->gds.oa_gfx_bo; + } +amdgpu_bo_list_for_each_entry (e, p->bo_list) {struct amdgpu_bo *bo =ttm_to_amdgpu_bo (e->tv.bo);/* Make sure we use the exclusive slot for shared BOs */ if (bo->prime_shared_count) e->tv.num_shared =0 ; e->bo_va =amdgpu_vm_bo_find (vm, bo); }if (gds) {
@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,struct drm_amdgpu_cs_chunk_ib *chunk_ib;struct drm_sched_entity *entity; chunk = &parser->chunks[i]; ib = &parser->job->ibs[j]; chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;if (chunk->chunk_id !=AMDGPU_CHUNK_ID_IB )continue ; + parser->hw_ip = chunk_ib->ip_type; +if (chunk_ib->ip_type ==AMDGPU_HW_IP_GFX && (amdgpu_mcbp ||amdgpu_sriov_vf (adev))) {if (chunk_ib->flags &AMDGPU_IB_FLAG_PREEMPT ) {if (chunk_ib->flags &AMDGPU_IB_FLAG_CE ) ce_preempt++;else de_preempt++; }/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6 -0
@@ -26,20 +26,26 @@struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size;uint32_t gds_compute_max_wave_id;uint32_t vgt_gs_max_wave_id; + +/* Reserved OA for the gfx ring. (gfx10) */ +uint32_t gds_gfx_reservation_size; +uint32_t oa_gfx_reservation_size; +struct amdgpu_bo *gds_gfx_bo; +struct amdgpu_bo *oa_gfx_bo; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +20 -0
@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle) kiq = &adev->gfx.kiq; r =amdgpu_gfx_kiq_init_ring (adev, &kiq->ring, &kiq->irq);if (r)return r; r =amdgpu_gfx_mqd_sw_init (adev,sizeof (struct v10_compute_mqd ));if (r)return r; +/* allocate reserved GDS resources for transform feedback */ + r =amdgpu_bo_create_kernel (adev, adev->gds.gds_gfx_reservation_size, +4 ,AMDGPU_GEM_DOMAIN_GDS , + &adev->gds.gds_gfx_bo,NULL ,NULL ); +if (r) +return r; + + r =amdgpu_bo_create_kernel (adev, adev->gds.oa_gfx_reservation_size, +1 ,AMDGPU_GEM_DOMAIN_OA , + &adev->gds.oa_gfx_bo,NULL ,NULL ); +if (r) +return r; +/* allocate visible FB for rlc auto-loading fw */ if (adev->firmware.load_type ==AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO ) { r =gfx_v10_0_rlc_backdoor_autoload_buffer_init (adev);if (r)return r; } adev->gfx.ce_ram_size =F32_CE_PROGRAM_RAM_SIZE ;gfx_v10_0_gpu_early_init (adev);
@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev)amdgpu_bo_free_kernel (&adev->gfx.me.me_fw_obj, &adev->gfx.me.me_fw_gpu_addr, (void **)&adev->gfx.me.me_fw_ptr); }static int gfx_v10_0_sw_fini (void *handle) {int i;struct amdgpu_device *adev = (struct amdgpu_device *)handle; +amdgpu_bo_free_kernel (&adev->gds.gds_gfx_bo,NULL ,NULL ); +amdgpu_bo_free_kernel (&adev->gds.oa_gfx_bo,NULL ,NULL ); +for (i =0 ; i < adev->gfx.num_gfx_rings; i++)amdgpu_ring_fini (&adev->gfx.gfx_ring[i]);for (i =0 ; i < adev->gfx.num_compute_rings; i++)amdgpu_ring_fini (&adev->gfx.compute_ring[i]);amdgpu_gfx_mqd_sw_fini (adev);amdgpu_gfx_kiq_free_ring (&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);amdgpu_gfx_kiq_fini (adev);gfx_v10_0_pfp_fini (adev);
@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; + +/* Reserved for transform feedback. */ + adev->gds.gds_gfx_reservation_size =256 ; + adev->gds.oa_gfx_reservation_size =4 ; }static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap) {u32 data;if (!bitmap)return ;
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 20190717000606.7181-1-maraeo-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Am 17.07.19 um 02:06 schrieb Marek Olšák:
Quoted 10 lines
From: Marek Olšák <[email protected]>
Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.
This is a workaround to ensure stability of transform feedback. Shaders hang waiting for a GDS instruction (
ds_sub, notds_ordered_count).The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.
So far compute is only using GWS, but I don't think that those reservations are a good idea in general.
How severe is the ENOMEM problem you see with using an explicit GWS allocation?
Regards,
Christian.
Quoted 199 lines
Signed-off-by Marek Olšák <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 10 ++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {struct drm_file *filp;struct amdgpu_ctx *ctx;/* chunks */ unsigned nchunks;struct amdgpu_cs_chunk *chunks;/* scheduler job object */ struct amdgpu_job *job;struct drm_sched_entity *entity; +unsigned hw_ip;/* buffer objects */ struct ww_acquire_ctx ticket;struct amdgpu_bo_list *bo_list;struct amdgpu_mn *mn;struct amdgpu_bo_list_entry vm_pd;struct list_head validated;struct dma_fence *fence;uint64_t bytes_moved_threshold;uint64_t bytes_moved_vis_threshold;drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +10
-0@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,if (r) goto error_validate;amdgpu_cs_report_moved_bytes (p->adev, p->bytes_moved, p->bytes_moved_vis); gds = p->bo_list->gds_obj; gws = p->bo_list->gws_obj; oa = p->bo_list->oa_obj; +if (p->hw_ip ==AMDGPU_HW_IP_GFX ) { +/* Only gfx10 allocates these. */ +if (!gds) + gds = p->adev->gds.gds_gfx_bo; +if (!oa) + oa = p->adev->gds.oa_gfx_bo; + } +amdgpu_bo_list_for_each_entry (e, p->bo_list) {struct amdgpu_bo *bo =ttm_to_amdgpu_bo (e->tv.bo);/* Make sure we use the exclusive slot for shared BOs */ if (bo->prime_shared_count) e->tv.num_shared =0 ; e->bo_va =amdgpu_vm_bo_find (vm, bo); }if (gds) {@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,struct drm_amdgpu_cs_chunk_ib *chunk_ib;struct drm_sched_entity *entity; chunk = &parser->chunks[i]; ib = &parser->job->ibs[j]; chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;if (chunk->chunk_id !=AMDGPU_CHUNK_ID_IB )continue ; + parser->hw_ip = chunk_ib->ip_type; +if (chunk_ib->ip_type ==AMDGPU_HW_IP_GFX && (amdgpu_mcbp ||amdgpu_sriov_vf (adev))) {if (chunk_ib->flags &AMDGPU_IB_FLAG_PREEMPT ) {if (chunk_ib->flags &AMDGPU_IB_FLAG_CE ) ce_preempt++;else de_preempt++; }/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6
-0@@ -26,20 +26,26 @@struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size;uint32_t gds_compute_max_wave_id;uint32_t vgt_gs_max_wave_id; + +/* Reserved OA for the gfx ring. (gfx10) */ +uint32_t gds_gfx_reservation_size; +uint32_t oa_gfx_reservation_size; +struct amdgpu_bo *gds_gfx_bo; +struct amdgpu_bo *oa_gfx_bo; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +20
-0@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle) kiq = &adev->gfx.kiq; r =amdgpu_gfx_kiq_init_ring (adev, &kiq->ring, &kiq->irq);if (r)return r; r =amdgpu_gfx_mqd_sw_init (adev,sizeof (struct v10_compute_mqd ));if (r)return r; +/* allocate reserved GDS resources for transform feedback */ + r =amdgpu_bo_create_kernel (adev, adev->gds.gds_gfx_reservation_size, +4 ,AMDGPU_GEM_DOMAIN_GDS , + &adev->gds.gds_gfx_bo,NULL ,NULL ); +if (r) +return r; + + r =amdgpu_bo_create_kernel (adev, adev->gds.oa_gfx_reservation_size, +1 ,AMDGPU_GEM_DOMAIN_OA , + &adev->gds.oa_gfx_bo,NULL ,NULL ); +if (r) +return r; +/* allocate visible FB for rlc auto-loading fw */ if (adev->firmware.load_type ==AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO ) { r =gfx_v10_0_rlc_backdoor_autoload_buffer_init (adev);if (r)return r; } adev->gfx.ce_ram_size =F32_CE_PROGRAM_RAM_SIZE ;gfx_v10_0_gpu_early_init (adev);@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev)amdgpu_bo_free_kernel (&adev->gfx.me.me_fw_obj, &adev->gfx.me.me_fw_gpu_addr, (void **)&adev->gfx.me.me_fw_ptr); }static int gfx_v10_0_sw_fini (void *handle) {int i;struct amdgpu_device *adev = (struct amdgpu_device *)handle; +amdgpu_bo_free_kernel (&adev->gds.gds_gfx_bo,NULL ,NULL ); +amdgpu_bo_free_kernel (&adev->gds.oa_gfx_bo,NULL ,NULL ); +for (i =0 ; i < adev->gfx.num_gfx_rings; i++)amdgpu_ring_fini (&adev->gfx.gfx_ring[i]);for (i =0 ; i < adev->gfx.num_compute_rings; i++)amdgpu_ring_fini (&adev->gfx.compute_ring[i]);amdgpu_gfx_mqd_sw_fini (adev);amdgpu_gfx_kiq_free_ring (&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);amdgpu_gfx_kiq_fini (adev);gfx_v10_0_pfp_fini (adev);@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; + +/* Reserved for transform feedback. */ + adev->gds.gds_gfx_reservation_size =256 ; + adev->gds.oa_gfx_reservation_size =4 ;
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev) · line 5220
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0}static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap) {u32 data;if (!bitmap)return ;
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 26330d7e-3cc0-f460-b7e0-b234597f382d-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Quoted 10 lines · On Wed., Jul. 17, 2019, 03:15 Christian König, <…
Am 17.07.19 um 02:06 schrieb Marek Olšák:
Quoted 6 lines
From: Marek Olšák <[email protected]>
Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.
This is a workaround to ensure stability of transform feedback. Shaders
hang
Quoted 2 lines
waiting for a GDS instruction (
ds_sub, notds_ordered_count).
Quoted 9 lines
Quoted 2 lines
The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.
So far compute is only using GWS, but I don't think that those reservations are a good idea in general.
How severe is the ENOMEM problem you see with using an explicit GWS allocation?
I guess you mean GDS or OA.
There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for ds_sub and can't continue, but GDS is idle.
Marek
Quoted 215 lines
Regards,
Christian.Review
Quoted 10 lines
Signed-off-by Marek Olšák <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 10 ++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu.hReview
Quoted 26 lines
index
4b514a44184c..cbd55d061b72100644drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {struct drm_file *filp;struct amdgpu_ctx *ctx;/* chunks */ unsigned nchunks;struct amdgpu_cs_chunk *chunks;/* scheduler job object */ struct amdgpu_job *job;struct drm_sched_entity *entity; +unsigned hw_ip;/* buffer objects */ struct ww_acquire_ctx ticket;struct amdgpu_bo_list *bo_list;struct amdgpu_mn *mn;struct amdgpu_bo_list_entry vm_pd;struct list_head validated;struct dma_fence *fence;uint64_t bytes_moved_threshold;uint64_t bytes_moved_vis_threshold;(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.cdrivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_parser_bos(struct
Quoted 4 lines
index
c691df6f7a57..9125cd69a124100644drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0
-0@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct
amdgpu_cs_parser*p,drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device
Quoted 29 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +8
-0if (r) goto error_validate;amdgpu_cs_report_moved_bytes (p->adev, p->bytes_moved, p->bytes_moved_vis); gds = p->bo_list->gds_obj; gws = p->bo_list->gws_obj; oa = p->bo_list->oa_obj; +if (p->hw_ip ==AMDGPU_HW_IP_GFX ) { +/* Only gfx10 allocates these. */ +if (!gds) + gds = p->adev->gds.gds_gfx_bo; +if (!oa) + oa = p->adev->gds.oa_gfx_bo; + } +amdgpu_bo_list_for_each_entry (e, p->bo_list) {struct amdgpu_bo *bo =ttm_to_amdgpu_bo (e->tv.bo);/* Make sure we use the exclusive slot for shared BOs */ if (bo->prime_shared_count) e->tv.num_shared =0 ; e->bo_va =amdgpu_vm_bo_find (vm, bo); }if (gds) {@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device*adev,
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +2
-0struct drm_amdgpu_cs_chunk_ib *chunk_ib;struct drm_sched_entity *entity; chunk = &parser->chunks[i]; ib = &parser->job->ibs[j]; chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;if (chunk->chunk_id !=AMDGPU_CHUNK_ID_IB )continue ; + parser->hw_ip = chunk_ib->ip_type; +drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0
-0if (chunk_ib->ip_type ==AMDGPU_HW_IP_GFX && (amdgpu_mcbp ||amdgpu_sriov_vf (adev))) {if (chunk_ib->flags &AMDGPU_IB_FLAG_PREEMPT ) {if (chunk_ib->flags &AMDGPU_IB_FLAG_CE ) ce_preempt++;else de_preempt++; }/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
Review
Quoted 1 line
(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.hReview
Quoted 31 lines
index
df8a23554831..0943b8e1d97e100644drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6
-0@@ -26,20 +26,26 @@struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size;uint32_t gds_compute_max_wave_id;uint32_t vgt_gs_max_wave_id; + +/* Reserved OA for the gfx ring. (gfx10) */ +uint32_t gds_gfx_reservation_size; +uint32_t oa_gfx_reservation_size; +struct amdgpu_bo *gds_gfx_bo; +struct amdgpu_bo *oa_gfx_bo; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */ (quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.cQuoted 6 lines
index
618291df659b..3952754c04ff100644drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle) kiq = &adev->gfx.kiq;drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1325
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2
-0r =amdgpu_gfx_kiq_init_ring (adev, &kiq->ring, &kiq->irq);if (r)return r; r =amdgpu_gfx_mqd_sw_init (adev,sizeof (struct v10_compute_mqd ));if (r)return r; +/* allocate reserved GDS resources for transform feedback */ + r = amdgpu_bo_create_kernel(adev,
adev->gds.gds_gfx_reservation_size,drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle)
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +6
-0+4 ,AMDGPU_GEM_DOMAIN_GDS , + &adev->gds.gds_gfx_bo, NULL, NULL); +if (r) +return r; + + r = amdgpu_bo_create_kernel(adev,
adev->gds.oa_gfx_reservation_size,drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_me_fini(struct
Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-0+1 ,AMDGPU_GEM_DOMAIN_OA , + &adev->gds.oa_gfx_bo, NULL, NULL); +if (r) +return r; +/* allocate visible FB for rlc auto-loading fw */ if (adev->firmware.load_type ==AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO ) { r =gfx_v10_0_rlc_backdoor_autoload_buffer_init (adev);if (r)return r; } adev->gfx.ce_ram_size =F32_CE_PROGRAM_RAM_SIZE ;gfx_v10_0_gpu_early_init (adev);@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct
amdgpu_device*adev)drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct
Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +3
-0amdgpu_bo_free_kernel (&adev->gfx.me.me_fw_obj, &adev->gfx.me.me_fw_gpu_addr, (void **)&adev->gfx.me.me_fw_ptr); }static int gfx_v10_0_sw_fini (void *handle) {int i;struct amdgpu_device *adev = (struct amdgpu_device *)handle; +amdgpu_bo_free_kernel (&adev->gds.gds_gfx_bo,NULL ,NULL ); +amdgpu_bo_free_kernel (&adev->gds.oa_gfx_bo,NULL ,NULL ); +for (i =0 ; i < adev->gfx.num_gfx_rings; i++)amdgpu_ring_fini (&adev->gfx.gfx_ring[i]);for (i =0 ; i < adev->gfx.num_compute_rings; i++)amdgpu_ring_fini (&adev->gfx.compute_ring[i]);amdgpu_gfx_mqd_sw_fini (adev);amdgpu_gfx_kiq_free_ring (&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);amdgpu_gfx_kiq_fini (adev);gfx_v10_0_pfp_fini (adev);@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct
amdgpu_device*adev)Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-0case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; + +/* Reserved for transform feedback. */ + adev->gds.gds_gfx_reservation_size =256 ; + adev->gds.oa_gfx_reservation_size =4 ; }
Quoted 11 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct · line 5213
Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct
amdgpu_device*adev,Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0u32 bitmap) {u32 data;if (!bitmap)return ;
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CAAxE2A6s2SOhqhkZydtN7kzzFRzHKwRnSG-JgYGy_GceofHAsQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Am 17.07.19 um 16:27 schrieb Marek Olšák:
Quoted 18 lines · On Wed., Jul. 17, 2019, 03:15 Christian König…
Am 17.07.19 um 02:06 schrieb Marek Olšák:
> From: Marek Olšák <[email protected] <mailto:[email protected]>> >
> Hopefully we'll only use 1 gfx ring, because otherwise we'd have
to have
> separate GDS buffers for each gfx ring.
>
> This is a workaround to ensure stability of transform feedback.
Shaders hang
> waiting for a GDS instruction (ds_sub, notds_ordered_count).
>
> The disadvantage is that compute IBs might get a different VMID,
> because now gfx always has GDS and compute doesn't.
Quoted 8 lines
So far compute is only using GWS, but I don't think that those reservations are a good idea in general.
How severe is the ENOMEM problem you see with using an explicit GWS allocation?
I guess you mean GDS or OA.
Yeah, just a typo. Compute is using GWS and we want to use GDS and OA here.
Quoted 1 line
There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for
ds_suband can't continue, but GDS is idle.
Well could it be because we don't correctly handle non zero offsets or stuff like that?
Does it work with this hack when you don't allocate GDS/OA from the start? (Just allocate it twice or something like this).
Christian.
Quoted 239 lines
Marek
Regards, Christian. > > Signed-off-by: Marek Olšák <[email protected] <mailto:[email protected]>> > --- > drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + > drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 10 ++++++++++ > drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 6 ++++++ > drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 20 ++++++++++++++++++++ > 4 files changed, 37 insertions(+) > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h > index 4b514a44184c..cbd55d061b72 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h > @@ -456,20 +456,21 @@ struct amdgpu_cs_parser { > struct drm_file *filp; > struct amdgpu_ctx *ctx; > > /* chunks */ > unsigned nchunks; > struct amdgpu_cs_chunk *chunks; > > /* scheduler job object */ > struct amdgpu_job *job; > struct drm_sched_entity *entity; > + unsigned hw_ip; > > /* buffer objects */ > struct ww_acquire_ctx ticket; > struct amdgpu_bo_list *bo_list; > struct amdgpu_mn *mn; > struct amdgpu_bo_list_entry vm_pd; > struct list_head validated; > struct dma_fence *fence; > uint64_t bytes_moved_threshold; > uint64_t bytes_moved_vis_threshold; > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c > index c691df6f7a57..9125cd69a124 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c > @@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, > if (r) > goto error_validate; > > amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, > p->bytes_moved_vis); > > gds = p->bo_list->gds_obj; > gws = p->bo_list->gws_obj; > oa = p->bo_list->oa_obj; > > + if (p->hw_ip == AMDGPU_HW_IP_GFX) { > + /* Only gfx10 allocates these. */ > + if (!gds) > + gds = p->adev->gds.gds_gfx_bo; > + if (!oa) > + oa = p->adev->gds.oa_gfx_bo; > + } > + > amdgpu_bo_list_for_each_entry(e, p->bo_list) { > struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo <http://tv.bo>); > > /* Make sure we use the exclusive slot for shared BOs */ > if (bo->prime_shared_count) > e->tv.num_shared = 0; > e->bo_va = amdgpu_vm_bo_find(vm, bo); > } > > if (gds) { > @@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, > struct drm_amdgpu_cs_chunk_ib *chunk_ib; > struct drm_sched_entity *entity; > > chunk = &parser->chunks[i]; > ib = &parser->job->ibs[j]; > chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; > > if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) > continue; > > + parser->hw_ip = chunk_ib->ip_type; > + > if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && > (amdgpu_mcbp || amdgpu_sriov_vf(adev))) { > if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { > if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) > ce_preempt++; > else > de_preempt++; > } > > /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h > index df8a23554831..0943b8e1d97e 100644 > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h > @@ -26,20 +26,26 @@ > > struct amdgpu_ring; > struct amdgpu_bo; > > struct amdgpu_gds { > uint32_t gds_size; > uint32_t gws_size; > uint32_t oa_size; > uint32_t gds_compute_max_wave_id; > uint32_t vgt_gs_max_wave_id; > + > + /* Reserved OA for the gfx ring. (gfx10) */ > + uint32_t gds_gfx_reservation_size; > + uint32_t oa_gfx_reservation_size; > + struct amdgpu_bo *gds_gfx_bo; > + struct amdgpu_bo *oa_gfx_bo; > }; > > struct amdgpu_gds_reg_offset { > uint32_t mem_base; > uint32_t mem_size; > uint32_t gws; > uint32_t oa; > }; > > #endif /* __AMDGPU_GDS_H__ */ > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c > index 618291df659b..3952754c04ff 100644 > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c > @@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle) > > kiq = &adev->gfx.kiq; > r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); > if (r) > return r; > > r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd)); > if (r) > return r; > > + /* allocate reserved GDS resources for transform feedback */ > + r = amdgpu_bo_create_kernel(adev, adev->gds.gds_gfx_reservation_size, > + 4, AMDGPU_GEM_DOMAIN_GDS, > + &adev->gds.gds_gfx_bo, NULL, NULL); > + if (r) > + return r; > + > + r = amdgpu_bo_create_kernel(adev, adev->gds.oa_gfx_reservation_size, > + 1, AMDGPU_GEM_DOMAIN_OA, > + &adev->gds.oa_gfx_bo, NULL, NULL); > + if (r) > + return r; > + > /* allocate visible FB for rlc auto-loading fw */ > if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { > r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev); > if (r) > return r; > } > > adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE; > > gfx_v10_0_gpu_early_init(adev); > @@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev) > amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, > &adev->gfx.me.me_fw_gpu_addr, > (void **)&adev->gfx.me.me_fw_ptr); > } > > static int gfx_v10_0_sw_fini(void *handle) > { > int i; > struct amdgpu_device *adev = (struct amdgpu_device *)handle; > > + amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL); > + amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL); > + > for (i = 0; i < adev->gfx.num_gfx_rings; i++) > amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); > for (i = 0; i < adev->gfx.num_compute_rings; i++) > amdgpu_ring_fini(&adev->gfx.compute_ring[i]); > > amdgpu_gfx_mqd_sw_fini(adev); > amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq); > amdgpu_gfx_kiq_fini(adev); > > gfx_v10_0_pfp_fini(adev); > @@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev) > case CHIP_NAVI10: > default: > adev->gds.gds_size = 0x10000; > adev->gds.gds_compute_max_wave_id = 0x4ff; > adev->gds.vgt_gs_max_wave_id = 0x3ff; > break; > } > > adev->gds.gws_size = 64; > adev->gds.oa_size = 16; > + > + /* Reserved for transform feedback. */ > + adev->gds.gds_gfx_reservation_size = 256; > + adev->gds.oa_gfx_reservation_size = 4; > } > > static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, > u32 bitmap) > { > u32 data; > > if (!bitmap) > return; >
Quoted 4 lines
_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): 57de127c-a6ac-c459-fc77-4da8578a41ae-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org
Quoted 30 lines · On Wed., Jul. 17, 2019, 10:43 Christian König, <…
Am 17.07.19 um 16:27 schrieb Marek Olšák:
Quoted 10 lines · On Wed., Jul. 17, 2019, 03:15 Christian König, <…
Am 17.07.19 um 02:06 schrieb Marek Olšák:
Quoted 6 lines
From: Marek Olšák <[email protected]>
Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.
This is a workaround to ensure stability of transform feedback. Shaders
hang
Quoted 2 lines
waiting for a GDS instruction (
ds_sub, notds_ordered_count).Quoted 9 lines
Quoted 2 lines
The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.
So far compute is only using GWS, but I don't think that those reservations are a good idea in general.
How severe is the ENOMEM problem you see with using an explicit GWS allocation?
I guess you mean GDS or OA.
Quoted 9 lines
Yeah, just a typo. Compute is using GWS and we want to use GDS and OA here.
There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for
ds_suband can't continue, but GDS is idle.Well could it be because we don't correctly handle non zero offsets or stuff like that?
I don't know what you mean.
Quoted 3 lines
Does it work with this hack when you don't allocate GDS/OA from the start? (Just allocate it twice or something like this).
It's only allocated once by the kernel with this hack.
Marek
Quoted 228 lines
Christian.
Marek
Quoted 222 lines
Regards,
Christian.Review
Quoted 10 lines
Signed-off-by Marek Olšák <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 10 ++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+)(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu.hReview
Quoted 26 lines
index
4b514a44184c..cbd55d061b72100644drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {struct drm_file *filp;struct amdgpu_ctx *ctx;/* chunks */ unsigned nchunks;struct amdgpu_cs_chunk *chunks;/* scheduler job object */ struct amdgpu_job *job;struct drm_sched_entity *entity; +unsigned hw_ip;/* buffer objects */ struct ww_acquire_ctx ticket;struct amdgpu_bo_list *bo_list;struct amdgpu_mn *mn;struct amdgpu_bo_list_entry vm_pd;struct list_head validated;struct dma_fence *fence;uint64_t bytes_moved_threshold;uint64_t bytes_moved_vis_threshold;(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.cdrivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_parser_bos(struct
Quoted 4 lines
index
c691df6f7a57..9125cd69a124100644drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0
-0@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct
amdgpu_cs_parser*p,drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device
Quoted 29 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +8
-0if (r) goto error_validate;amdgpu_cs_report_moved_bytes (p->adev, p->bytes_moved, p->bytes_moved_vis); gds = p->bo_list->gds_obj; gws = p->bo_list->gws_obj; oa = p->bo_list->oa_obj; +if (p->hw_ip ==AMDGPU_HW_IP_GFX ) { +/* Only gfx10 allocates these. */ +if (!gds) + gds = p->adev->gds.gds_gfx_bo; +if (!oa) + oa = p->adev->gds.oa_gfx_bo; + } +amdgpu_bo_list_for_each_entry (e, p->bo_list) {struct amdgpu_bo *bo =ttm_to_amdgpu_bo (e->tv.bo);/* Make sure we use the exclusive slot for shared BOs */ if (bo->prime_shared_count) e->tv.num_shared =0 ; e->bo_va =amdgpu_vm_bo_find (vm, bo); }if (gds) {@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device*adev,
Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +2
-0struct drm_amdgpu_cs_chunk_ib *chunk_ib;struct drm_sched_entity *entity; chunk = &parser->chunks[i]; ib = &parser->job->ibs[j]; chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;if (chunk->chunk_id !=AMDGPU_CHUNK_ID_IB )continue ; + parser->hw_ip = chunk_ib->ip_type; +drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device · line 983
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0
-0if (chunk_ib->ip_type ==AMDGPU_HW_IP_GFX && (amdgpu_mcbp ||amdgpu_sriov_vf (adev))) {if (chunk_ib->flags &AMDGPU_IB_FLAG_PREEMPT ) {if (chunk_ib->flags &AMDGPU_IB_FLAG_CE ) ce_preempt++;else de_preempt++; }/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
Review
Quoted 1 line
(quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.hReview
Quoted 31 lines
index
df8a23554831..0943b8e1d97e100644drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6
-0@@ -26,20 +26,26 @@struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size;uint32_t gds_compute_max_wave_id;uint32_t vgt_gs_max_wave_id; + +/* Reserved OA for the gfx ring. (gfx10) */ +uint32_t gds_gfx_reservation_size; +uint32_t oa_gfx_reservation_size; +struct amdgpu_bo *gds_gfx_bo; +struct amdgpu_bo *oa_gfx_bo; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */ (quoted diff) +0
-0
b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.cQuoted 6 lines
index
618291df659b..3952754c04ff100644drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle) kiq = &adev->gfx.kiq;drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1325
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2
-0r =amdgpu_gfx_kiq_init_ring (adev, &kiq->ring, &kiq->irq);if (r)return r; r =amdgpu_gfx_mqd_sw_init (adev,sizeof (struct v10_compute_mqd ));if (r)return r; +/* allocate reserved GDS resources for transform feedback */ + r = amdgpu_bo_create_kernel(adev,
adev->gds.gds_gfx_reservation_size,drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1331
Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +6
-0+4 ,AMDGPU_GEM_DOMAIN_GDS , + &adev->gds.gds_gfx_bo, NULL, NULL); +if (r) +return r; + + r = amdgpu_bo_create_kernel(adev,
adev->gds.oa_gfx_reservation_size,drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1338
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5
-0+1 ,AMDGPU_GEM_DOMAIN_OA , + &adev->gds.oa_gfx_bo, NULL, NULL); +if (r) +return r; +/* allocate visible FB for rlc auto-loading fw */ if (adev->firmware.load_type ==AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO ){
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_me_fini(struct
Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0r =gfx_v10_0_rlc_backdoor_autoload_buffer_init (adev);if (r)return r; } adev->gfx.ce_ram_size =F32_CE_PROGRAM_RAM_SIZE ;gfx_v10_0_gpu_early_init (adev);@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct
amdgpu_device*adev)drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct
Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +3
-0amdgpu_bo_free_kernel (&adev->gfx.me.me_fw_obj, &adev->gfx.me.me_fw_gpu_addr, (void **)&adev->gfx.me.me_fw_ptr); }static int gfx_v10_0_sw_fini (void *handle) {int i;struct amdgpu_device *adev = (struct amdgpu_device *)handle; +amdgpu_bo_free_kernel (&adev->gds.gds_gfx_bo,NULL ,NULL ); +amdgpu_bo_free_kernel (&adev->gds.oa_gfx_bo,NULL ,NULL ); +for (i =0 ; i < adev->gfx.num_gfx_rings; i++)amdgpu_ring_fini (&adev->gfx.gfx_ring[i]);for (i =0 ; i < adev->gfx.num_compute_rings; i++)amdgpu_ring_fini (&adev->gfx.compute_ring[i]);amdgpu_gfx_mqd_sw_fini (adev);amdgpu_gfx_kiq_free_ring (&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);amdgpu_gfx_kiq_fini (adev);gfx_v10_0_pfp_fini (adev);@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct
amdgpu_device*adev)Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; }drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct · line 5213
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4
-0adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; + +/* Reserved for transform feedback. */ + adev->gds.gds_gfx_reservation_size =256 ; + adev->gds.oa_gfx_reservation_size =4 ; } static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct
amdgpu_device*adev,Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0u32 bitmap) {u32 data;
Quoted 10 lines
Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct · line 5220
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0if (!bitmap)return ;_______________________________________________
amd-gfx mailing listamd-gfx-PD4FTy7X32lNgt0PjOBp934avgP/[email protected]://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
A message that is not in the archive (sent off-list or not yet received): CAAxE2A67J0gN=ZV2t66zPc+E9kJDyFghd6oGobu4DLT5hvK1gA-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org
Am 17.07.19 um 16:54 schrieb Marek Olšák:
On Wed., Jul. 17, 2019, 10:43 Christian König, <[email protected]<mailto:[email protected]>> wrote:
Am 17.07.19 um 16:27 schrieb Marek Olšák:
On Wed., Jul. 17, 2019, 03:15 Christian König, <[email protected]<mailto:[email protected]>> wrote:
Am 17.07.19 um 02:06 schrieb Marek Olšák:
Quoted 10 lines
From: Marek Olšák <[email protected]<mailto:[email protected]>>
Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.
This is a workaround to ensure stability of transform feedback. Shaders hang waiting for a GDS instruction (
ds_sub, notds_ordered_count).The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.
So far compute is only using GWS, but I don't think that those reservations are a good idea in general.
How severe is the ENOMEM problem you see with using an explicit GWS allocation?
I guess you mean GDS or OA.
Yeah, just a typo. Compute is using GWS and we want to use GDS and OA here.
There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for ds_sub and can't continue, but GDS is idle.
Well could it be because we don't correctly handle non zero offsets or stuff like that?
I don't know what you mean.
I wanted to try to leak some GDS/OA during allocation to narrow down what is going wrong here.
Does it work with this hack when you don't allocate GDS/OA from the start? (Just allocate it twice or something like this).
It's only allocated once by the kernel with this hack.
Well my question is why does that actually help?
I mean is it a bug in the memory management or some hardware issue that we can't switch GDS/OS?
I think we need to find the root cause of the problem and if it's related to switching GDS/OS what the actual restrictions are.
Christian.
Marek
Christian.
Marek
Regards,
Christian.
Quoted 199 lines
--- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 10 ++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h | 6 ++++++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu.h +1
-0@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {struct drm_file *filp;struct amdgpu_ctx *ctx;/* chunks */ unsigned nchunks;struct amdgpu_cs_chunk *chunks;/* scheduler job object */ struct amdgpu_job *job;struct drm_sched_entity *entity; +unsigned hw_ip;/* buffer objects */ struct ww_acquire_ctx ticket;struct amdgpu_bo_list *bo_list;struct amdgpu_mn *mn;struct amdgpu_bo_list_entry vm_pd;struct list_head validated;struct dma_fence *fence;uint64_t bytes_moved_threshold;uint64_t bytes_moved_vis_threshold;drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +10
-0@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,if (r) goto error_validate;amdgpu_cs_report_moved_bytes (p->adev, p->bytes_moved, p->bytes_moved_vis); gds = p->bo_list->gds_obj; gws = p->bo_list->gws_obj; oa = p->bo_list->oa_obj; +if (p->hw_ip ==AMDGPU_HW_IP_GFX ) { +/* Only gfx10 allocates these. */ +if (!gds) + gds = p->adev->gds.gds_gfx_bo; +if (!oa) + oa = p->adev->gds.oa_gfx_bo; + } +amdgpu_bo_list_for_each_entry (e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo<http://tv.bo>); /* Make sure we use the exclusive slot for shared BOs */ if (bo->prime_shared_count) e->tv.num_shared = 0; e->bo_va =amdgpu_vm_bo_find (vm, bo); }if (gds) {@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,struct drm_amdgpu_cs_chunk_ib *chunk_ib;struct drm_sched_entity *entity; chunk = &parser->chunks[i]; ib = &parser->job->ibs[j]; chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;if (chunk->chunk_id !=AMDGPU_CHUNK_ID_IB )continue ; + parser->hw_ip = chunk_ib->ip_type; +if (chunk_ib->ip_type ==AMDGPU_HW_IP_GFX && (amdgpu_mcbp ||amdgpu_sriov_vf (adev))) {if (chunk_ib->flags &AMDGPU_IB_FLAG_PREEMPT ) {if (chunk_ib->flags &AMDGPU_IB_FLAG_CE ) ce_preempt++;else de_preempt++; }/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6
-0@@ -26,20 +26,26 @@struct amdgpu_ring ;struct amdgpu_bo ;struct amdgpu_gds {uint32_t gds_size;uint32_t gws_size;uint32_t oa_size;uint32_t gds_compute_max_wave_id;uint32_t vgt_gs_max_wave_id; + +/* Reserved OA for the gfx ring. (gfx10) */ +uint32_t gds_gfx_reservation_size; +uint32_t oa_gfx_reservation_size; +struct amdgpu_bo *gds_gfx_bo; +struct amdgpu_bo *oa_gfx_bo; };struct amdgpu_gds_reg_offset {uint32_t mem_base;uint32_t mem_size;uint32_t gws;uint32_t oa; };#endif /* __AMDGPU_GDS_H__ */ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +20
-0@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle) kiq = &adev->gfx.kiq; r =amdgpu_gfx_kiq_init_ring (adev, &kiq->ring, &kiq->irq);if (r)return r; r =amdgpu_gfx_mqd_sw_init (adev,sizeof (struct v10_compute_mqd ));if (r)return r; +/* allocate reserved GDS resources for transform feedback */ + r =amdgpu_bo_create_kernel (adev, adev->gds.gds_gfx_reservation_size, +4 ,AMDGPU_GEM_DOMAIN_GDS , + &adev->gds.gds_gfx_bo,NULL ,NULL ); +if (r) +return r; + + r =amdgpu_bo_create_kernel (adev, adev->gds.oa_gfx_reservation_size, +1 ,AMDGPU_GEM_DOMAIN_OA , + &adev->gds.oa_gfx_bo,NULL ,NULL ); +if (r) +return r; +/* allocate visible FB for rlc auto-loading fw */ if (adev->firmware.load_type ==AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO ) { r =gfx_v10_0_rlc_backdoor_autoload_buffer_init (adev);if (r)return r; } adev->gfx.ce_ram_size =F32_CE_PROGRAM_RAM_SIZE ;gfx_v10_0_gpu_early_init (adev);@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev)amdgpu_bo_free_kernel (&adev->gfx.me.me_fw_obj, &adev->gfx.me.me_fw_gpu_addr, (void **)&adev->gfx.me.me_fw_ptr); }static int gfx_v10_0_sw_fini (void *handle) {int i;struct amdgpu_device *adev = (struct amdgpu_device *)handle; +amdgpu_bo_free_kernel (&adev->gds.gds_gfx_bo,NULL ,NULL ); +amdgpu_bo_free_kernel (&adev->gds.oa_gfx_bo,NULL ,NULL ); +for (i =0 ; i < adev->gfx.num_gfx_rings; i++)amdgpu_ring_fini (&adev->gfx.gfx_ring[i]);for (i =0 ; i < adev->gfx.num_compute_rings; i++)amdgpu_ring_fini (&adev->gfx.compute_ring[i]);amdgpu_gfx_mqd_sw_fini (adev);amdgpu_gfx_kiq_free_ring (&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);amdgpu_gfx_kiq_fini (adev);gfx_v10_0_pfp_fini (adev);@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)case CHIP_NAVI10 :default : adev->gds.gds_size =0x10000 ; adev->gds.gds_compute_max_wave_id =0x4ff ; adev->gds.vgt_gs_max_wave_id =0x3ff ;break ; } adev->gds.gws_size =64 ; adev->gds.oa_size =16 ; + +/* Reserved for transform feedback. */ + adev->gds.gds_gfx_reservation_size =256 ; + adev->gds.oa_gfx_reservation_size =4 ;
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev) · line 5220
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0
-0}static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh (struct amdgpu_device *adev,u32 bitmap) {u32 data;if (!bitmap)return ;
_______________________________________________
amd-gfx mailing list [email protected]<mailto:[email protected]>
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx