Skip to contentSkip to conversation

drm/amdgpu:fix world switch hang

PATCH In review Monk Liu 3 participants 3 messages Jun 6, 2017 – Jun 7, 2017 2 messages not in the archive

Subsystems (2)
lore
Share & tools
lore

Monk Liu

author
#0 lore raw

PATCH 1/1 drm/amdgpu:fix world switch hang

for SR-IOV, we must keep the pipeline-sync in the protection of COND_EXEC, otherwise the command consumed by CPG is not consistent when world switch triggerd, e.g.:

world switch hit and the IB frame is skipped so the fence won't signal, thus CP will jump to the next DMAframe's pipeline-sync command, and it will make CP hang foever.

after pipelin-sync moved into COND_EXEC the consistency can be guaranteed

Change-Id I3059535745e2e1215343299b17f27fe081fc01bc
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 5 +++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 7 +++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 +-
 3 files changed, 9 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +3 -2
index 025e03b..947bd34 100644
@@ -130,6 +130,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 
 	unsigned i;
 	int r = 0;
+	bool need_pipe_sync = false;
 
 	if (num_ibs == 0)
 		return -EINVAL;
@@ -165,7 +166,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	if (ring->funcs->emit_pipeline_sync && job &&
 	    ((tmp = amdgpu_sync_get_fence(&job->sched_sync)) ||
 	     amdgpu_vm_need_pipeline_sync(ring, job))) {
-		amdgpu_ring_emit_pipeline_sync(ring);
+		need_pipe_sync = true;
 		fence_put(tmp);
 	}
 
@@ -173,7 +174,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		ring->funcs->insert_start(ring);
 
 	if (vm) {
-		r = amdgpu_vm_flush(ring, job);
+		r = amdgpu_vm_flush(ring, job, need_pipe_sync);
 		if (r) {
 			amdgpu_ring_undo(ring);
 			return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +5 -2
index b065f8b..6d62e34 100644
@@ -747,7 +747,7 @@ static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
  *
  * Emit a VM flush when it is necessary.
  */
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
 {
 	struct amdgpu_device *adev = ring->adev;
 	unsigned vmhub = ring->funcs->vmhub;
@@ -770,12 +770,15 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 		vm_flush_needed = true;
 	}
 
-	if (!vm_flush_needed && !gds_switch_needed)
+	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
 		return 0;
 
 	if (ring->funcs->init_cond_exec)
 		patch_offset = amdgpu_ring_init_cond_exec(ring);
 
+	if (need_pipe_sync)
+		amdgpu_ring_emit_pipeline_sync(ring);
+
 	if (ring->funcs->emit_vm_flush && vm_flush_needed) {
 		struct fence *fence;
 
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1 -1
index 9aa00d9..cc0009f 100644
@@ -222,7 +222,7 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 		      struct amdgpu_sync *sync, struct fence *fence,
 		      struct amdgpu_job *job);
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
 			unsigned vmid);
 void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev);
Signature · 6 lines
2.7.4

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1496741253-24964-1-git-send-email-Monk.Liu-5C7GfCeVMHo@public.gmane.org

Reviewed-by
#2 lore raw

Am 06.06.2017 um 11:27 schrieb Monk Liu:

Quoted 8 lines

for SR-IOV, we must keep the pipeline-sync in the protection of COND_EXEC, otherwise the command consumed by CPG is not consistent when world switch triggerd, e.g.:

world switch hit and the IB frame is skipped so the fence won't signal, thus CP will jump to the next DMAframe's pipeline-sync command, and it will make CP hang foever.

Quoted 5 lines

after pipelin-sync moved into COND_EXEC the consistency can be guaranteed

Change-Id I3059535745e2e1215343299b17f27fe081fc01bc
Quoted 71 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 5 +++--
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 7 +++++--
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 +-
  3 files changed, 9 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +3 -2
index 025e03b..947bd34 100644
@@ -130,6 +130,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 
  	unsigned i;
  	int r = 0;
+	bool need_pipe_sync = false;
 
  	if (num_ibs == 0)
  		return -EINVAL;
@@ -165,7 +166,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
  	if (ring->funcs->emit_pipeline_sync && job &&
  	    ((tmp = amdgpu_sync_get_fence(&job->sched_sync)) ||
  	     amdgpu_vm_need_pipeline_sync(ring, job))) {
-		amdgpu_ring_emit_pipeline_sync(ring);
+		need_pipe_sync = true;
  		fence_put(tmp);
  	}
 
@@ -173,7 +174,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
  		ring->funcs->insert_start(ring);
 
  	if (vm) {
-		r = amdgpu_vm_flush(ring, job);
+		r = amdgpu_vm_flush(ring, job, need_pipe_sync);
  		if (r) {
  			amdgpu_ring_undo(ring);
  			return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +5 -2
index b065f8b..6d62e34 100644
@@ -747,7 +747,7 @@ static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
   *
   * Emit a VM flush when it is necessary.
   */
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
  {
  	struct amdgpu_device *adev = ring->adev;
  	unsigned vmhub = ring->funcs->vmhub;
@@ -770,12 +770,15 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
  		vm_flush_needed = true;
  	}
 
-	if (!vm_flush_needed && !gds_switch_needed)
+	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
  		return 0;
 
  	if (ring->funcs->init_cond_exec)
  		patch_offset = amdgpu_ring_init_cond_exec(ring);
 
+	if (need_pipe_sync)
+		amdgpu_ring_emit_pipeline_sync(ring);
+
  	if (ring->funcs->emit_vm_flush && vm_flush_needed) {
  		struct fence *fence;
 
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +0 -0
index 9aa00d9..cc0009f 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, · line 228

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1 -1
@@ -222,7 +222,7 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
  int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
  		      struct amdgpu_sync *sync, struct fence *fence,
  		      struct amdgpu_job *job);
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
  void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
  			unsigned vmid);
  void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): f4d58615-318c-a093-e31a-2fa8995a6a85-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#4 lore raw
Quoted 41 lines · Christian König

Am 06.06.2017 um 11:27 schrieb Monk Liu:

Quoted 8 lines

for SR-IOV, we must keep the pipeline-sync in the protection of COND_EXEC, otherwise the command consumed by CPG is not consistent when world switch triggerd, e.g.:

world switch hit and the IB frame is skipped so the fence won't signal, thus CP will jump to the next DMAframe's pipeline-sync command, and it will make CP hang foever.

Quoted 5 lines

after pipelin-sync moved into COND_EXEC the consistency can be guaranteed

Change-Id I3059535745e2e1215343299b17f27fe081fc01bc
Quoted 24 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 5 +++--
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 7 +++++--
  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 2 +-
  3 files changed, 9 insertions(+), 5 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +2 -1
index 025e03b..947bd34 100644
@@ -130,6 +130,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, 
 unsigned num_ibs,
        unsigned i;
      int r = 0;
+    bool need_pipe_sync = false;
        if (num_ibs == 0)
          return -EINVAL;
@@ -165,7 +166,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, 
 unsigned num_ibs,
      if (ring->funcs->emit_pipeline_sync && job &&
          ((tmp = amdgpu_sync_get_fence(&job->sched_sync)) ||
           amdgpu_vm_need_pipeline_sync(ring, job))) {
-        amdgpu_ring_emit_pipeline_sync(ring);
+        need_pipe_sync = true;
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, · line 177

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +0 -0
          fence_put(tmp);
      }

@@ -173,7 +174,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring
*ring, unsigned num_ibs,

          ring->funcs->insert_start(ring);
        if (vm) {
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 -1
-        r = amdgpu_vm_flush(ring, job);
+        r = amdgpu_vm_flush(ring, job, need_pipe_sync);

There is a potential case lost, if no-vm job needs pipeline_sync, then which will be lost.

Regards,
David Zhou

Quoted 45 lines
Quoted 45 lines
          if (r) {
              amdgpu_ring_undo(ring);
              return r;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
index b065f8b..6d62e34 100644
@@ -747,7 +747,7 @@ static bool amdgpu_vm_is_large_bar(struct 
 amdgpu_device *adev)
   *
   * Emit a VM flush when it is necessary.
   */
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job 
 *job, bool need_pipe_sync)
  {
      struct amdgpu_device *adev = ring->adev;
      unsigned vmhub = ring->funcs->vmhub;
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +3 -0
@@ -770,12 +770,15 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, 
 struct amdgpu_job *job)
          vm_flush_needed = true;
      }
  -    if (!vm_flush_needed && !gds_switch_needed)
+    if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
          return 0;
        if (ring->funcs->init_cond_exec)
          patch_offset = amdgpu_ring_init_cond_exec(ring);
  +    if (need_pipe_sync)
+        amdgpu_ring_emit_pipeline_sync(ring);
+
      if (ring->funcs->emit_vm_flush && vm_flush_needed) {
          struct fence *fence;
  diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h 

b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 9aa00d9..cc0009f 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +1 -1
@@ -222,7 +222,7 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
  int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
                struct amdgpu_sync *sync, struct fence *fence,
                struct amdgpu_job *job);
-int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job 
 *job, bool need_pipe_sync);
  void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
Quoted 8 lines
Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +0 -0
              unsigned vmid);
  void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx