Skip to contentSkip to conversation

drm/amdgpu: reserve GDS resources on the gfx ring for gfx10

PATCH In review Marek Olšák 3 participants 6 messages Jul 17, 2019 5 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amdgpu: reserve GDS resources on the gfx ring for gfx10

From: Marek Olšák <[email protected]>

Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.

This is a workaround to ensure stability of transform feedback. Shaders hang waiting for a GDS instruction (ds_sub, not ds_ordered_count).

The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 10 ++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h |  6 ++++++
 drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c  | 20 ++++++++++++++++++++
 4 files changed, 37 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 4b514a44184c..cbd55d061b72 100644
@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {
 	struct drm_file		*filp;
 	struct amdgpu_ctx	*ctx;
 
 	/* chunks */
 	unsigned		nchunks;
 	struct amdgpu_cs_chunk	*chunks;
 
 	/* scheduler job object */
 	struct amdgpu_job	*job;
 	struct drm_sched_entity	*entity;
+	unsigned		hw_ip;
 
 	/* buffer objects */
 	struct ww_acquire_ctx		ticket;
 	struct amdgpu_bo_list		*bo_list;
 	struct amdgpu_mn		*mn;
 	struct amdgpu_bo_list_entry	vm_pd;
 	struct list_head		validated;
 	struct dma_fence		*fence;
 	uint64_t			bytes_moved_threshold;
 	uint64_t			bytes_moved_vis_threshold;
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +10 -0
index c691df6f7a57..9125cd69a124 100644
@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	if (r)
 		goto error_validate;
 
 	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
 				     p->bytes_moved_vis);
 
 	gds = p->bo_list->gds_obj;
 	gws = p->bo_list->gws_obj;
 	oa = p->bo_list->oa_obj;
 
+	if (p->hw_ip == AMDGPU_HW_IP_GFX) {
+		/* Only gfx10 allocates these. */
+		if (!gds)
+			gds = p->adev->gds.gds_gfx_bo;
+		if (!oa)
+			oa = p->adev->gds.oa_gfx_bo;
+	}
+
 	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
 
 		/* Make sure we use the exclusive slot for shared BOs */
 		if (bo->prime_shared_count)
 			e->tv.num_shared = 0;
 		e->bo_va = amdgpu_vm_bo_find(vm, bo);
 	}
 
 	if (gds) {
@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
 		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
 		struct drm_sched_entity *entity;
 
 		chunk = &parser->chunks[i];
 		ib = &parser->job->ibs[j];
 		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
 
 		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
 			continue;
 
+		parser->hw_ip = chunk_ib->ip_type;
+
 		if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
 		    (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
 			if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
 				if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
 					ce_preempt++;
 				else
 					de_preempt++;
 			}
 
 			/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6 -0
index df8a23554831..0943b8e1d97e 100644
@@ -26,20 +26,26 @@
 
 struct amdgpu_ring;
 struct amdgpu_bo;
 
 struct amdgpu_gds {
 	uint32_t gds_size;
 	uint32_t gws_size;
 	uint32_t oa_size;
 	uint32_t gds_compute_max_wave_id;
 	uint32_t vgt_gs_max_wave_id;
+
+	/* Reserved OA for the gfx ring. (gfx10) */
+	uint32_t gds_gfx_reservation_size;
+	uint32_t oa_gfx_reservation_size;
+	struct amdgpu_bo *gds_gfx_bo;
+	struct amdgpu_bo *oa_gfx_bo;
 };
 
 struct amdgpu_gds_reg_offset {
 	uint32_t	mem_base;
 	uint32_t	mem_size;
 	uint32_t	gws;
 	uint32_t	oa;
 };
 
 #endif /* __AMDGPU_GDS_H__ */
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +20 -0
index 618291df659b..3952754c04ff 100644
@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle)
 
 	kiq = &adev->gfx.kiq;
 	r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
 	if (r)
 		return r;
 
 	r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
 	if (r)
 		return r;
 
+	/* allocate reserved GDS resources for transform feedback */
+	r = amdgpu_bo_create_kernel(adev, adev->gds.gds_gfx_reservation_size,
+				    4, AMDGPU_GEM_DOMAIN_GDS,
+				    &adev->gds.gds_gfx_bo, NULL, NULL);
+	if (r)
+		return r;
+
+	r = amdgpu_bo_create_kernel(adev, adev->gds.oa_gfx_reservation_size,
+				    1, AMDGPU_GEM_DOMAIN_OA,
+				    &adev->gds.oa_gfx_bo, NULL, NULL);
+	if (r)
+		return r;
+
 	/* allocate visible FB for rlc auto-loading fw */
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
 		r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
 		if (r)
 			return r;
 	}
 
 	adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
 
 	gfx_v10_0_gpu_early_init(adev);
@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev)
 	amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
 			      &adev->gfx.me.me_fw_gpu_addr,
 			      (void **)&adev->gfx.me.me_fw_ptr);
 }
 
 static int gfx_v10_0_sw_fini(void *handle)
 {
 	int i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
 	amdgpu_gfx_mqd_sw_fini(adev);
 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
 	amdgpu_gfx_kiq_fini(adev);
 
 	gfx_v10_0_pfp_fini(adev);
@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)
 	case CHIP_NAVI10:
 	default:
 		adev->gds.gds_size = 0x10000;
 		adev->gds.gds_compute_max_wave_id = 0x4ff;
 		adev->gds.vgt_gs_max_wave_id = 0x3ff;
 		break;
 	}
 
 	adev->gds.gws_size = 64;
 	adev->gds.oa_size = 16;
+
+	/* Reserved for transform feedback. */
+	adev->gds.gds_gfx_reservation_size = 256;
+	adev->gds.oa_gfx_reservation_size = 4;
 }
 
 static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
 							  u32 bitmap)
 {
 	u32 data;
 
 	if (!bitmap)
 		return;
 
Signature · 6 lines
2.17.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 20190717000606.7181-1-maraeo-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#2 lore raw

Am 17.07.19 um 02:06 schrieb Marek Olšák:

Quoted 10 lines

From: Marek Olšák <[email protected]>

Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.

This is a workaround to ensure stability of transform feedback. Shaders hang waiting for a GDS instruction (ds_sub, not ds_ordered_count).

The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.

So far compute is only using GWS, but I don't think that those reservations are a good idea in general.

How severe is the ENOMEM problem you see with using an explicit GWS allocation?

Regards,
Christian.

Quoted 199 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 10 ++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h |  6 ++++++
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c  | 20 ++++++++++++++++++++
  4 files changed, 37 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 4b514a44184c..cbd55d061b72 100644
@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {
  	struct drm_file		*filp;
  	struct amdgpu_ctx	*ctx;
 
  	/* chunks */
  	unsigned		nchunks;
  	struct amdgpu_cs_chunk	*chunks;
 
  	/* scheduler job object */
  	struct amdgpu_job	*job;
  	struct drm_sched_entity	*entity;
+	unsigned		hw_ip;
 
  	/* buffer objects */
  	struct ww_acquire_ctx		ticket;
  	struct amdgpu_bo_list		*bo_list;
  	struct amdgpu_mn		*mn;
  	struct amdgpu_bo_list_entry	vm_pd;
  	struct list_head		validated;
  	struct dma_fence		*fence;
  	uint64_t			bytes_moved_threshold;
  	uint64_t			bytes_moved_vis_threshold;
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +10 -0
index c691df6f7a57..9125cd69a124 100644
@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
  	if (r)
  		goto error_validate;
 
  	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
  				     p->bytes_moved_vis);
 
  	gds = p->bo_list->gds_obj;
  	gws = p->bo_list->gws_obj;
  	oa = p->bo_list->oa_obj;
 
+	if (p->hw_ip == AMDGPU_HW_IP_GFX) {
+		/* Only gfx10 allocates these. */
+		if (!gds)
+			gds = p->adev->gds.gds_gfx_bo;
+		if (!oa)
+			oa = p->adev->gds.oa_gfx_bo;
+	}
+
  	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
  		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
 
  		/* Make sure we use the exclusive slot for shared BOs */
  		if (bo->prime_shared_count)
  			e->tv.num_shared = 0;
  		e->bo_va = amdgpu_vm_bo_find(vm, bo);
  	}
 
  	if (gds) {
@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
  		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
  		struct drm_sched_entity *entity;
 
  		chunk = &parser->chunks[i];
  		ib = &parser->job->ibs[j];
  		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
 
  		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
  			continue;
 
+		parser->hw_ip = chunk_ib->ip_type;
+
  		if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
  		    (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
  			if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
  				if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
  					ce_preempt++;
  				else
  					de_preempt++;
  			}
 
  			/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6 -0
index df8a23554831..0943b8e1d97e 100644
@@ -26,20 +26,26 @@
 
  struct amdgpu_ring;
  struct amdgpu_bo;
 
  struct amdgpu_gds {
  	uint32_t gds_size;
  	uint32_t gws_size;
  	uint32_t oa_size;
  	uint32_t gds_compute_max_wave_id;
  	uint32_t vgt_gs_max_wave_id;
+
+	/* Reserved OA for the gfx ring. (gfx10) */
+	uint32_t gds_gfx_reservation_size;
+	uint32_t oa_gfx_reservation_size;
+	struct amdgpu_bo *gds_gfx_bo;
+	struct amdgpu_bo *oa_gfx_bo;
  };
 
  struct amdgpu_gds_reg_offset {
  	uint32_t	mem_base;
  	uint32_t	mem_size;
  	uint32_t	gws;
  	uint32_t	oa;
  };
 
  #endif /* __AMDGPU_GDS_H__ */
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +20 -0
index 618291df659b..3952754c04ff 100644
@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle)
 
  	kiq = &adev->gfx.kiq;
  	r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
  	if (r)
  		return r;
 
  	r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
  	if (r)
  		return r;
 
+	/* allocate reserved GDS resources for transform feedback */
+	r = amdgpu_bo_create_kernel(adev, adev->gds.gds_gfx_reservation_size,
+				    4, AMDGPU_GEM_DOMAIN_GDS,
+				    &adev->gds.gds_gfx_bo, NULL, NULL);
+	if (r)
+		return r;
+
+	r = amdgpu_bo_create_kernel(adev, adev->gds.oa_gfx_reservation_size,
+				    1, AMDGPU_GEM_DOMAIN_OA,
+				    &adev->gds.oa_gfx_bo, NULL, NULL);
+	if (r)
+		return r;
+
  	/* allocate visible FB for rlc auto-loading fw */
  	if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
  		r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
  		if (r)
  			return r;
  	}
 
  	adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
 
  	gfx_v10_0_gpu_early_init(adev);
@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev)
  	amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
  			      &adev->gfx.me.me_fw_gpu_addr,
  			      (void **)&adev->gfx.me.me_fw_ptr);
  }
 
  static int gfx_v10_0_sw_fini(void *handle)
  {
  	int i;
  	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+
  	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
  		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
  	for (i = 0; i < adev->gfx.num_compute_rings; i++)
  		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
  	amdgpu_gfx_mqd_sw_fini(adev);
  	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
  	amdgpu_gfx_kiq_fini(adev);
 
  	gfx_v10_0_pfp_fini(adev);
@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)
  	case CHIP_NAVI10:
  	default:
  		adev->gds.gds_size = 0x10000;
  		adev->gds.gds_compute_max_wave_id = 0x4ff;
  		adev->gds.vgt_gs_max_wave_id = 0x3ff;
  		break;
  	}
 
  	adev->gds.gws_size = 64;
  	adev->gds.oa_size = 16;
+
+	/* Reserved for transform feedback. */
+	adev->gds.gds_gfx_reservation_size = 256;
+	adev->gds.oa_gfx_reservation_size = 4;

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev) · line 5220

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
  }
 
  static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
  							  u32 bitmap)
  {
  	u32 data;
 
  	if (!bitmap)
  		return;
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 26330d7e-3cc0-f460-b7e0-b234597f382d-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#4 lore raw
Quoted 10 lines · On Wed., Jul. 17, 2019, 03:15 Christian König, <…

Am 17.07.19 um 02:06 schrieb Marek Olšák:

Quoted 6 lines

From: Marek Olšák <[email protected]>

Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.

This is a workaround to ensure stability of transform feedback. Shaders

hang

Quoted 2 lines

waiting for a GDS instruction (ds_sub, not ds_ordered_count).

Quoted 9 lines
Quoted 2 lines

The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.

So far compute is only using GWS, but I don't think that those reservations are a good idea in general.

How severe is the ENOMEM problem you see with using an explicit GWS allocation?

I guess you mean GDS or OA.

There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for ds_sub and can't continue, but GDS is idle.

Marek

Quoted 215 lines

Regards,
Christian.

Review

Quoted 10 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 10 ++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h |  6 ++++++
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c  | 20 ++++++++++++++++++++
  4 files changed, 37 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu.h

Review

Quoted 26 lines

index 4b514a44184c..cbd55d061b72 100644

drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {
      struct drm_file         *filp;
      struct amdgpu_ctx       *ctx;
 
      /* chunks */
      unsigned                nchunks;
      struct amdgpu_cs_chunk  *chunks;
 
      /* scheduler job object */
      struct amdgpu_job       *job;
      struct drm_sched_entity *entity;
+     unsigned                hw_ip;
 
      /* buffer objects */
      struct ww_acquire_ctx           ticket;
      struct amdgpu_bo_list           *bo_list;
      struct amdgpu_mn                *mn;
      struct amdgpu_bo_list_entry     vm_pd;
      struct list_head                validated;
      struct dma_fence                *fence;
      uint64_t                        bytes_moved_threshold;
      uint64_t                        bytes_moved_vis_threshold;
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_parser_bos(struct

Quoted 4 lines

index c691df6f7a57..9125cd69a124 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0
@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct

amdgpu_cs_parser *p,

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device

Quoted 29 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +8 -0
      if (r)
              goto error_validate;
 
      amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
                                   p->bytes_moved_vis);
 
      gds = p->bo_list->gds_obj;
      gws = p->bo_list->gws_obj;
      oa = p->bo_list->oa_obj;
 
+     if (p->hw_ip == AMDGPU_HW_IP_GFX) {
+             /* Only gfx10 allocates these. */
+             if (!gds)
+                     gds = p->adev->gds.gds_gfx_bo;
+             if (!oa)
+                     oa = p->adev->gds.oa_gfx_bo;
+     }
+
      amdgpu_bo_list_for_each_entry(e, p->bo_list) {
              struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
 
              /* Make sure we use the exclusive slot for shared BOs */
              if (bo->prime_shared_count)
                      e->tv.num_shared = 0;
              e->bo_va = amdgpu_vm_bo_find(vm, bo);
      }
 
      if (gds) {
@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device

*adev,

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +2 -0
              struct drm_amdgpu_cs_chunk_ib *chunk_ib;
              struct drm_sched_entity *entity;
 
              chunk = &parser->chunks[i];
              ib = &parser->job->ibs[j];
              chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
 
              if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
                      continue;
 
+             parser->hw_ip = chunk_ib->ip_type;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0
              if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
                  (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
                      if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
                              if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
                                      ce_preempt++;
                              else
                                      de_preempt++;
                      }
 
                      /* each GFX command submit allows 0 or 1 IB

preemptible for CE & DE */

Review

Quoted 1 line
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h

Review

Quoted 31 lines

index df8a23554831..0943b8e1d97e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6 -0
@@ -26,20 +26,26 @@
 
  struct amdgpu_ring;
  struct amdgpu_bo;
 
  struct amdgpu_gds {
      uint32_t gds_size;
      uint32_t gws_size;
      uint32_t oa_size;
      uint32_t gds_compute_max_wave_id;
      uint32_t vgt_gs_max_wave_id;
+
+     /* Reserved OA for the gfx ring. (gfx10) */
+     uint32_t gds_gfx_reservation_size;
+     uint32_t oa_gfx_reservation_size;
+     struct amdgpu_bo *gds_gfx_bo;
+     struct amdgpu_bo *oa_gfx_bo;
  };
 
  struct amdgpu_gds_reg_offset {
      uint32_t        mem_base;
      uint32_t        mem_size;
      uint32_t        gws;
      uint32_t        oa;
  };
 
  #endif /* __AMDGPU_GDS_H__ */
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c

Quoted 6 lines

index 618291df659b..3952754c04ff 100644

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle)
 
      kiq = &adev->gfx.kiq;

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1325

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2 -0
      r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
      if (r)
              return r;
 
      r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
      if (r)
              return r;
 
+     /* allocate reserved GDS resources for transform feedback */
+     r = amdgpu_bo_create_kernel(adev,

adev->gds.gds_gfx_reservation_size,

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle)

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +6 -0
+                                 4, AMDGPU_GEM_DOMAIN_GDS,
+                                 &adev->gds.gds_gfx_bo, NULL, NULL);
+     if (r)
+             return r;
+
+     r = amdgpu_bo_create_kernel(adev,

adev->gds.oa_gfx_reservation_size,

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_me_fini(struct

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5 -0
+                                 1, AMDGPU_GEM_DOMAIN_OA,
+                                 &adev->gds.oa_gfx_bo, NULL, NULL);
+     if (r)
+             return r;
+
      /* allocate visible FB for rlc auto-loading fw */
      if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
              r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
              if (r)
                      return r;
      }
 
      adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
 
      gfx_v10_0_gpu_early_init(adev);
@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct

amdgpu_device *adev)

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct

Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +3 -0
      amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
                            &adev->gfx.me.me_fw_gpu_addr,
                            (void **)&adev->gfx.me.me_fw_ptr);
  }
 
  static int gfx_v10_0_sw_fini(void *handle)
  {
      int i;
      struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+     amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+     amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+
      for (i = 0; i < adev->gfx.num_gfx_rings; i++)
              amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
      for (i = 0; i < adev->gfx.num_compute_rings; i++)
              amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
      amdgpu_gfx_mqd_sw_fini(adev);
      amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
      amdgpu_gfx_kiq_fini(adev);
 
      gfx_v10_0_pfp_fini(adev);
@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct

amdgpu_device *adev)

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4 -0
      case CHIP_NAVI10:
      default:
              adev->gds.gds_size = 0x10000;
              adev->gds.gds_compute_max_wave_id = 0x4ff;
              adev->gds.vgt_gs_max_wave_id = 0x3ff;
              break;
      }
 
      adev->gds.gws_size = 64;
      adev->gds.oa_size = 16;
+
+     /* Reserved for transform feedback. */
+     adev->gds.gds_gfx_reservation_size = 256;
+     adev->gds.oa_gfx_reservation_size = 4;
  }
 
Quoted 11 lines

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct · line 5213

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
  static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct

amdgpu_device *adev,

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
                                                        u32 bitmap)
  {
      u32 data;
 
      if (!bitmap)
              return;
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CAAxE2A6s2SOhqhkZydtN7kzzFRzHKwRnSG-JgYGy_GceofHAsQ-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#6 lore raw

Am 17.07.19 um 16:27 schrieb Marek Olšák:

Quoted 18 lines · On Wed., Jul. 17, 2019, 03:15 Christian König…

Am 17.07.19 um 02:06 schrieb Marek Olšák:
> From: Marek Olšák <[email protected] <mailto:[email protected]>> >
> Hopefully we'll only use 1 gfx ring, because otherwise we'd have
to have
> separate GDS buffers for each gfx ring.
>
> This is a workaround to ensure stability of transform feedback.
Shaders hang
> waiting for a GDS instruction (ds_sub, not ds_ordered_count).
>
> The disadvantage is that compute IBs might get a different VMID,
> because now gfx always has GDS and compute doesn't.

Quoted 8 lines

So far compute is only using GWS, but I don't think that those reservations are a good idea in general.

How severe is the ENOMEM problem you see with using an explicit GWS allocation?

I guess you mean GDS or OA.

Yeah, just a typo. Compute is using GWS and we want to use GDS and OA here.

Quoted 1 line

There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for ds_sub and can't continue, but GDS is idle.

Well could it be because we don't correctly handle non zero offsets or stuff like that?

Does it work with this hack when you don't allocate GDS/OA from the start? (Just allocate it twice or something like this).

Christian.

Quoted 239 lines

Marek

    Regards,
    Christian.

    >
    > Signed-off-by: Marek Olšák <[email protected]
    <mailto:[email protected]>>
    > ---
    >   drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  1 +
    >   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 10 ++++++++++
    >   drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h |  6 ++++++
    >   drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c  | 20 ++++++++++++++++++++
    >   4 files changed, 37 insertions(+)
    >
    > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
    b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
    > index 4b514a44184c..cbd55d061b72 100644
    > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
    > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
    > @@ -456,20 +456,21 @@ struct amdgpu_cs_parser {
    >       struct drm_file         *filp;
    >       struct amdgpu_ctx       *ctx;
    >
    >       /* chunks */
    >       unsigned                nchunks;
    >       struct amdgpu_cs_chunk  *chunks;
    >
    >       /* scheduler job object */
    >       struct amdgpu_job       *job;
    >       struct drm_sched_entity *entity;
    > +     unsigned                hw_ip;
    >
    >       /* buffer objects */
    >       struct ww_acquire_ctx           ticket;
    >       struct amdgpu_bo_list           *bo_list;
    >       struct amdgpu_mn                *mn;
    >       struct amdgpu_bo_list_entry     vm_pd;
    >       struct list_head                validated;
    >       struct dma_fence                *fence;
    >       uint64_t bytes_moved_threshold;
    >       uint64_t bytes_moved_vis_threshold;
    > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
    b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
    > index c691df6f7a57..9125cd69a124 100644
    > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
    > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
    > @@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct
    amdgpu_cs_parser *p,
    >       if (r)
    >               goto error_validate;
    >
    >       amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
    > p->bytes_moved_vis);
    >
    >       gds = p->bo_list->gds_obj;
    >       gws = p->bo_list->gws_obj;
    >       oa = p->bo_list->oa_obj;
    >
    > +     if (p->hw_ip == AMDGPU_HW_IP_GFX) {
    > +             /* Only gfx10 allocates these. */
    > +             if (!gds)
    > +                     gds = p->adev->gds.gds_gfx_bo;
    > +             if (!oa)
    > +                     oa = p->adev->gds.oa_gfx_bo;
    > +     }
    > +
    >       amdgpu_bo_list_for_each_entry(e, p->bo_list) {
    >               struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo
    <http://tv.bo>);
    >
    >               /* Make sure we use the exclusive slot for shared
    BOs */
    >               if (bo->prime_shared_count)
    >                       e->tv.num_shared = 0;
    >               e->bo_va = amdgpu_vm_bo_find(vm, bo);
    >       }
    >
    >       if (gds) {
    > @@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct
    amdgpu_device *adev,
    >               struct drm_amdgpu_cs_chunk_ib *chunk_ib;
    >               struct drm_sched_entity *entity;
    >
    >               chunk = &parser->chunks[i];
    >               ib = &parser->job->ibs[j];
    >               chunk_ib = (struct drm_amdgpu_cs_chunk_ib
    *)chunk->kdata;
    >
    >               if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
    >                       continue;
    >
    > +             parser->hw_ip = chunk_ib->ip_type;
    > +
    >               if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
    >                   (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
    >                       if (chunk_ib->flags &
    AMDGPU_IB_FLAG_PREEMPT) {
    >                               if (chunk_ib->flags &
    AMDGPU_IB_FLAG_CE)
    >                                       ce_preempt++;
    >                               else
    >                                       de_preempt++;
    >                       }
    >
    >                       /* each GFX command submit allows 0 or 1
    IB preemptible for CE & DE */
    > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
    b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
    > index df8a23554831..0943b8e1d97e 100644
    > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
    > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h
    > @@ -26,20 +26,26 @@
    >
    >   struct amdgpu_ring;
    >   struct amdgpu_bo;
    >
    >   struct amdgpu_gds {
    >       uint32_t gds_size;
    >       uint32_t gws_size;
    >       uint32_t oa_size;
    >       uint32_t gds_compute_max_wave_id;
    >       uint32_t vgt_gs_max_wave_id;
    > +
    > +     /* Reserved OA for the gfx ring. (gfx10) */
    > +     uint32_t gds_gfx_reservation_size;
    > +     uint32_t oa_gfx_reservation_size;
    > +     struct amdgpu_bo *gds_gfx_bo;
    > +     struct amdgpu_bo *oa_gfx_bo;
    >   };
    >
    >   struct amdgpu_gds_reg_offset {
    >       uint32_t        mem_base;
    >       uint32_t        mem_size;
    >       uint32_t        gws;
    >       uint32_t        oa;
    >   };
    >
    >   #endif /* __AMDGPU_GDS_H__ */
    > diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
    b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
    > index 618291df659b..3952754c04ff 100644
    > --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
    > +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
    > @@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle)
    >
    >       kiq = &adev->gfx.kiq;
    >       r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
    >       if (r)
    >               return r;
    >
    >       r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct
    v10_compute_mqd));
    >       if (r)
    >               return r;
    >
    > +     /* allocate reserved GDS resources for transform feedback */
    > +     r = amdgpu_bo_create_kernel(adev,
    adev->gds.gds_gfx_reservation_size,
    > +                                 4, AMDGPU_GEM_DOMAIN_GDS,
    > +  &adev->gds.gds_gfx_bo, NULL, NULL);
    > +     if (r)
    > +             return r;
    > +
    > +     r = amdgpu_bo_create_kernel(adev,
    adev->gds.oa_gfx_reservation_size,
    > +                                 1, AMDGPU_GEM_DOMAIN_OA,
    > +  &adev->gds.oa_gfx_bo, NULL, NULL);
    > +     if (r)
    > +             return r;
    > +
    >       /* allocate visible FB for rlc auto-loading fw */
    >       if (adev->firmware.load_type ==
    AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
    >               r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
    >               if (r)
    >                       return r;
    >       }
    >
    >       adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
    >
    >       gfx_v10_0_gpu_early_init(adev);
    > @@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct
    amdgpu_device *adev)
    >  amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
    >  &adev->gfx.me.me_fw_gpu_addr,
    >                             (void **)&adev->gfx.me.me_fw_ptr);
    >   }
    >
    >   static int gfx_v10_0_sw_fini(void *handle)
    >   {
    >       int i;
    >       struct amdgpu_device *adev = (struct amdgpu_device *)handle;
    >
    > +  amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
    > +  amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
    > +
    >       for (i = 0; i < adev->gfx.num_gfx_rings; i++)
    >  amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
    >       for (i = 0; i < adev->gfx.num_compute_rings; i++)
    >  amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
    >
    >       amdgpu_gfx_mqd_sw_fini(adev);
    >  amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
    >       amdgpu_gfx_kiq_fini(adev);
    >
    >       gfx_v10_0_pfp_fini(adev);
    > @@ -5181,20 +5197,24 @@ static void
    gfx_v10_0_set_gds_init(struct amdgpu_device *adev)
    >       case CHIP_NAVI10:
    >       default:
    >               adev->gds.gds_size = 0x10000;
    >               adev->gds.gds_compute_max_wave_id = 0x4ff;
    >               adev->gds.vgt_gs_max_wave_id = 0x3ff;
    >               break;
    >       }
    >
    >       adev->gds.gws_size = 64;
    >       adev->gds.oa_size = 16;
    > +
    > +     /* Reserved for transform feedback. */
    > +     adev->gds.gds_gfx_reservation_size = 256;
    > +     adev->gds.oa_gfx_reservation_size = 4;
    >   }
    >
    >   static void
    gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device
    *adev,
    >    u32 bitmap)
    >   {
    >       u32 data;
    >
    >       if (!bitmap)
    >               return;
    >
Quoted 4 lines

_______________________________________________
amd-gfx mailing list [email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 57de127c-a6ac-c459-fc77-4da8578a41ae-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org

#8 lore raw
Quoted 30 lines · On Wed., Jul. 17, 2019, 10:43 Christian König, <…

Am 17.07.19 um 16:27 schrieb Marek Olšák:

Quoted 10 lines · On Wed., Jul. 17, 2019, 03:15 Christian König, <…

Am 17.07.19 um 02:06 schrieb Marek Olšák:

Quoted 6 lines

From: Marek Olšák <[email protected]>

Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.

This is a workaround to ensure stability of transform feedback. Shaders

hang

Quoted 2 lines

waiting for a GDS instruction (ds_sub, not ds_ordered_count).

Quoted 9 lines
Quoted 2 lines

The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.

So far compute is only using GWS, but I don't think that those reservations are a good idea in general.

How severe is the ENOMEM problem you see with using an explicit GWS allocation?

I guess you mean GDS or OA.

Quoted 9 lines

Yeah, just a typo. Compute is using GWS and we want to use GDS and OA here.

There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for ds_sub and can't continue, but GDS is idle.

Well could it be because we don't correctly handle non zero offsets or stuff like that?

I don't know what you mean.

Quoted 3 lines

Does it work with this hack when you don't allocate GDS/OA from the start? (Just allocate it twice or something like this).

It's only allocated once by the kernel with this hack.

Marek

Quoted 228 lines

Christian.

Marek

Quoted 222 lines

Regards,
Christian.

Review

Quoted 10 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 10 ++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h |  6 ++++++
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c  | 20 ++++++++++++++++++++
  4 files changed, 37 insertions(+)
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu.h

Review

Quoted 26 lines

index 4b514a44184c..cbd55d061b72 100644

drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {
      struct drm_file         *filp;
      struct amdgpu_ctx       *ctx;
 
      /* chunks */
      unsigned                nchunks;
      struct amdgpu_cs_chunk  *chunks;
 
      /* scheduler job object */
      struct amdgpu_job       *job;
      struct drm_sched_entity *entity;
+     unsigned                hw_ip;
 
      /* buffer objects */
      struct ww_acquire_ctx           ticket;
      struct amdgpu_bo_list           *bo_list;
      struct amdgpu_mn                *mn;
      struct amdgpu_bo_list_entry     vm_pd;
      struct list_head                validated;
      struct dma_fence                *fence;
      uint64_t                        bytes_moved_threshold;
      uint64_t                        bytes_moved_vis_threshold;
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_parser_bos(struct

Quoted 4 lines

index c691df6f7a57..9125cd69a124 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0
@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct

amdgpu_cs_parser *p,

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device

Quoted 29 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +8 -0
      if (r)
              goto error_validate;
 
      amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
                                   p->bytes_moved_vis);
 
      gds = p->bo_list->gds_obj;
      gws = p->bo_list->gws_obj;
      oa = p->bo_list->oa_obj;
 
+     if (p->hw_ip == AMDGPU_HW_IP_GFX) {
+             /* Only gfx10 allocates these. */
+             if (!gds)
+                     gds = p->adev->gds.gds_gfx_bo;
+             if (!oa)
+                     oa = p->adev->gds.oa_gfx_bo;
+     }
+
      amdgpu_bo_list_for_each_entry(e, p->bo_list) {
              struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
 
              /* Make sure we use the exclusive slot for shared BOs */
              if (bo->prime_shared_count)
                      e->tv.num_shared = 0;
              e->bo_va = amdgpu_vm_bo_find(vm, bo);
      }
 
      if (gds) {
@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device

*adev,

Quoted 12 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +2 -0
              struct drm_amdgpu_cs_chunk_ib *chunk_ib;
              struct drm_sched_entity *entity;
 
              chunk = &parser->chunks[i];
              ib = &parser->job->ibs[j];
              chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
 
              if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
                      continue;
 
+             parser->hw_ip = chunk_ib->ip_type;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c static int amdgpu_cs_ib_fill(struct amdgpu_device · line 983

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +0 -0
              if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
                  (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
                      if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
                              if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
                                      ce_preempt++;
                              else
                                      de_preempt++;
                      }
 
                      /* each GFX command submit allows 0 or 1 IB

preemptible for CE & DE */

Review

Quoted 1 line
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h

Review

Quoted 31 lines

index df8a23554831..0943b8e1d97e 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6 -0
@@ -26,20 +26,26 @@
 
  struct amdgpu_ring;
  struct amdgpu_bo;
 
  struct amdgpu_gds {
      uint32_t gds_size;
      uint32_t gws_size;
      uint32_t oa_size;
      uint32_t gds_compute_max_wave_id;
      uint32_t vgt_gs_max_wave_id;
+
+     /* Reserved OA for the gfx ring. (gfx10) */
+     uint32_t gds_gfx_reservation_size;
+     uint32_t oa_gfx_reservation_size;
+     struct amdgpu_bo *gds_gfx_bo;
+     struct amdgpu_bo *oa_gfx_bo;
  };
 
  struct amdgpu_gds_reg_offset {
      uint32_t        mem_base;
      uint32_t        mem_size;
      uint32_t        gws;
      uint32_t        oa;
  };
 
  #endif /* __AMDGPU_GDS_H__ */
(quoted diff) +0 -0

b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c

Quoted 6 lines

index 618291df659b..3952754c04ff 100644

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle)
 
      kiq = &adev->gfx.kiq;

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1325

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +2 -0
      r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
      if (r)
              return r;
 
      r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
      if (r)
              return r;
 
+     /* allocate reserved GDS resources for transform feedback */
+     r = amdgpu_bo_create_kernel(adev,

adev->gds.gds_gfx_reservation_size,

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1331

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +6 -0
+                                 4, AMDGPU_GEM_DOMAIN_GDS,
+                                 &adev->gds.gds_gfx_bo, NULL, NULL);
+     if (r)
+             return r;
+
+     r = amdgpu_bo_create_kernel(adev,

adev->gds.oa_gfx_reservation_size,

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static int gfx_v10_0_sw_init(void *handle) · line 1338

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +5 -0
+                                 1, AMDGPU_GEM_DOMAIN_OA,
+                                 &adev->gds.oa_gfx_bo, NULL, NULL);
+     if (r)
+             return r;
+
      /* allocate visible FB for rlc auto-loading fw */
      if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)

{

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_me_fini(struct

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
              r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
              if (r)
                      return r;
      }
 
      adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
 
      gfx_v10_0_gpu_early_init(adev);
@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct

amdgpu_device *adev)

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct

Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +3 -0
      amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
                            &adev->gfx.me.me_fw_gpu_addr,
                            (void **)&adev->gfx.me.me_fw_ptr);
  }
 
  static int gfx_v10_0_sw_fini(void *handle)
  {
      int i;
      struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+     amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+     amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+
      for (i = 0; i < adev->gfx.num_gfx_rings; i++)
              amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
      for (i = 0; i < adev->gfx.num_compute_rings; i++)
              amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
      amdgpu_gfx_mqd_sw_fini(adev);
      amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
      amdgpu_gfx_kiq_fini(adev);
 
      gfx_v10_0_pfp_fini(adev);
@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct

amdgpu_device *adev)

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
      case CHIP_NAVI10:
      default:
              adev->gds.gds_size = 0x10000;
              adev->gds.gds_compute_max_wave_id = 0x4ff;
              adev->gds.vgt_gs_max_wave_id = 0x3ff;
              break;
      }

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct · line 5213

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +4 -0
 
      adev->gds.gws_size = 64;
      adev->gds.oa_size = 16;
+
+     /* Reserved for transform feedback. */
+     adev->gds.gds_gfx_reservation_size = 256;
+     adev->gds.oa_gfx_reservation_size = 4;
  }
 
  static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct

amdgpu_device *adev,

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
                                                        u32 bitmap)
  {
      u32 data;
 
Quoted 10 lines
Quoted 5 lines

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct · line 5220

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
      if (!bitmap)
              return;
 

_______________________________________________
amd-gfx mailing listamd-gfx-PD4FTy7X32lNgt0PjOBp934avgP/[email protected]://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): CAAxE2A67J0gN=ZV2t66zPc+E9kJDyFghd6oGobu4DLT5hvK1gA-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

#10 lore raw

Am 17.07.19 um 16:54 schrieb Marek Olšák:

On Wed., Jul. 17, 2019, 10:43 Christian König, <[email protected]<mailto:[email protected]>> wrote:
Am 17.07.19 um 16:27 schrieb Marek Olšák:

On Wed., Jul. 17, 2019, 03:15 Christian König, <[email protected]<mailto:[email protected]>> wrote:
Am 17.07.19 um 02:06 schrieb Marek Olšák:

Quoted 10 lines

From: Marek Olšák <[email protected]<mailto:[email protected]>>

Hopefully we'll only use 1 gfx ring, because otherwise we'd have to have separate GDS buffers for each gfx ring.

This is a workaround to ensure stability of transform feedback. Shaders hang waiting for a GDS instruction (ds_sub, not ds_ordered_count).

The disadvantage is that compute IBs might get a different VMID, because now gfx always has GDS and compute doesn't.

So far compute is only using GWS, but I don't think that those reservations are a good idea in general.

How severe is the ENOMEM problem you see with using an explicit GWS allocation?

I guess you mean GDS or OA.

Yeah, just a typo. Compute is using GWS and we want to use GDS and OA here.

There is no ENOMEM, it just hangs. I don't know why. The shader is waiting for ds_sub and can't continue, but GDS is idle.

Well could it be because we don't correctly handle non zero offsets or stuff like that?

I don't know what you mean.

I wanted to try to leak some GDS/OA during allocation to narrow down what is going wrong here.

Does it work with this hack when you don't allocate GDS/OA from the start? (Just allocate it twice or something like this).

It's only allocated once by the kernel with this hack.

Well my question is why does that actually help?

I mean is it a bug in the memory management or some hardware issue that we can't switch GDS/OS?

I think we need to find the root cause of the problem and if it's related to switching GDS/OS what the actual restrictions are.

Christian.

Marek

Christian.

Marek

Regards,
Christian.

Quoted 199 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h     |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 10 ++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h |  6 ++++++
  drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c  | 20 ++++++++++++++++++++
  4 files changed, 37 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index 4b514a44184c..cbd55d061b72 100644
@@ -456,20 +456,21 @@ struct amdgpu_cs_parser {
      struct drm_file         *filp;
      struct amdgpu_ctx       *ctx;
 
      /* chunks */
      unsigned                nchunks;
      struct amdgpu_cs_chunk  *chunks;
 
      /* scheduler job object */
      struct amdgpu_job       *job;
      struct drm_sched_entity *entity;
+     unsigned                hw_ip;
 
      /* buffer objects */
      struct ww_acquire_ctx           ticket;
      struct amdgpu_bo_list           *bo_list;
      struct amdgpu_mn                *mn;
      struct amdgpu_bo_list_entry     vm_pd;
      struct list_head                validated;
      struct dma_fence                *fence;
      uint64_t                        bytes_moved_threshold;
      uint64_t                        bytes_moved_vis_threshold;
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +10 -0
index c691df6f7a57..9125cd69a124 100644
@@ -678,20 +678,28 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
      if (r)
              goto error_validate;
 
      amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
                                   p->bytes_moved_vis);
 
      gds = p->bo_list->gds_obj;
      gws = p->bo_list->gws_obj;
      oa = p->bo_list->oa_obj;
 
+     if (p->hw_ip == AMDGPU_HW_IP_GFX) {
+             /* Only gfx10 allocates these. */
+             if (!gds)
+                     gds = p->adev->gds.gds_gfx_bo;
+             if (!oa)
+                     oa = p->adev->gds.oa_gfx_bo;
+     }
+
      amdgpu_bo_list_for_each_entry(e, p->bo_list) {
              struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo<http://tv.bo>);
 
              /* Make sure we use the exclusive slot for shared BOs */
              if (bo->prime_shared_count)
                      e->tv.num_shared = 0;
              e->bo_va = amdgpu_vm_bo_find(vm, bo);
      }
 
      if (gds) {
@@ -954,20 +962,22 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
              struct drm_amdgpu_cs_chunk_ib *chunk_ib;
              struct drm_sched_entity *entity;
 
              chunk = &parser->chunks[i];
              ib = &parser->job->ibs[j];
              chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
 
              if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
                      continue;
 
+             parser->hw_ip = chunk_ib->ip_type;
+
              if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
                  (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
                      if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
                              if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
                                      ce_preempt++;
                              else
                                      de_preempt++;
                      }
 
                      /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
drivers/gpu/drm/amd/amdgpu/amdgpu_gds.h +6 -0
index df8a23554831..0943b8e1d97e 100644
@@ -26,20 +26,26 @@
 
  struct amdgpu_ring;
  struct amdgpu_bo;
 
  struct amdgpu_gds {
      uint32_t gds_size;
      uint32_t gws_size;
      uint32_t oa_size;
      uint32_t gds_compute_max_wave_id;
      uint32_t vgt_gs_max_wave_id;
+
+     /* Reserved OA for the gfx ring. (gfx10) */
+     uint32_t gds_gfx_reservation_size;
+     uint32_t oa_gfx_reservation_size;
+     struct amdgpu_bo *gds_gfx_bo;
+     struct amdgpu_bo *oa_gfx_bo;
  };
 
  struct amdgpu_gds_reg_offset {
      uint32_t        mem_base;
      uint32_t        mem_size;
      uint32_t        gws;
      uint32_t        oa;
  };
 
  #endif /* __AMDGPU_GDS_H__ */
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +20 -0
index 618291df659b..3952754c04ff 100644
@@ -1314,20 +1314,33 @@ static int gfx_v10_0_sw_init(void *handle)
 
      kiq = &adev->gfx.kiq;
      r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
      if (r)
              return r;
 
      r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
      if (r)
              return r;
 
+     /* allocate reserved GDS resources for transform feedback */
+     r = amdgpu_bo_create_kernel(adev, adev->gds.gds_gfx_reservation_size,
+                                 4, AMDGPU_GEM_DOMAIN_GDS,
+                                 &adev->gds.gds_gfx_bo, NULL, NULL);
+     if (r)
+             return r;
+
+     r = amdgpu_bo_create_kernel(adev, adev->gds.oa_gfx_reservation_size,
+                                 1, AMDGPU_GEM_DOMAIN_OA,
+                                 &adev->gds.oa_gfx_bo, NULL, NULL);
+     if (r)
+             return r;
+
      /* allocate visible FB for rlc auto-loading fw */
      if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
              r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
              if (r)
                      return r;
      }
 
      adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
 
      gfx_v10_0_gpu_early_init(adev);
@@ -1354,20 +1367,23 @@ static void gfx_v10_0_me_fini(struct amdgpu_device *adev)
      amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
                            &adev->gfx.me.me_fw_gpu_addr,
                            (void **)&adev->gfx.me.me_fw_ptr);
  }
 
  static int gfx_v10_0_sw_fini(void *handle)
  {
      int i;
      struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+     amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
+     amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
+
      for (i = 0; i < adev->gfx.num_gfx_rings; i++)
              amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
      for (i = 0; i < adev->gfx.num_compute_rings; i++)
              amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
 
      amdgpu_gfx_mqd_sw_fini(adev);
      amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
      amdgpu_gfx_kiq_fini(adev);
 
      gfx_v10_0_pfp_fini(adev);
@@ -5181,20 +5197,24 @@ static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)
      case CHIP_NAVI10:
      default:
              adev->gds.gds_size = 0x10000;
              adev->gds.gds_compute_max_wave_id = 0x4ff;
              adev->gds.vgt_gs_max_wave_id = 0x3ff;
              break;
      }
 
      adev->gds.gws_size = 64;
      adev->gds.oa_size = 16;
+
+     /* Reserved for transform feedback. */
+     adev->gds.gds_gfx_reservation_size = 256;
+     adev->gds.oa_gfx_reservation_size = 4;

drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev) · line 5220

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +0 -0
  }
 
  static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
                                                        u32 bitmap)
  {
      u32 data;
 
      if (!bitmap)
              return;
 

_______________________________________________
amd-gfx mailing list [email protected]<mailto:[email protected]>
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx