Skip to contentSkip to conversation

drm/amdgpu: flush gpu tlb using sdma ring

RFC Superseded Shashank Sharma 4 participants 7 messages Jun 9, 2023 – Jun 12, 2023

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

RFC 1/1 drm/amdgpu: flush gpu tlb using sdma ring

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41 ++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------
 3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +41 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
 /* SDMA CSA reside in the 3rd page of CSA */
 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 
+/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *
+ * @adev: amdgpu device pointer
+ *
+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA
+ * itself.
+ */
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev)
+{
+	int r;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+
+	mutex_lock(&adev->mman.gtt_window_lock);
+	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
+				     AMDGPU_FENCE_OWNER_UNDEFINED,
+				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     &job);
+	if (r)
+		goto error_alloc;
+
+	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+	job->vm_needs_flush = true;
+	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	fence = amdgpu_job_submit(job);
+
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return fence;
+
+error_alloc:
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return NULL;
+}
+
 /*
  * GPU SDMA IP block helpers function.
  */
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
         bool duplicate);
 void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev);
 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
 
 #endif
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +8 -28
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
 #include "athub_v2_1.h"
 
 #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
 
 #if 0
 static const struct soc15_reg_golden golden_settings_navi10_hdp[] =
@@ -333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 {
 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 	struct dma_fence *fence;
-	struct amdgpu_job *job;
-
-	int r;
 
 	/* flush hdp cache */
 	adev->hdp.funcs->flush_hdp(adev, NULL);
@@ -378,34 +376,16 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 		return;
 	}
 
-	/* The SDMA on Navi has a bug which can theoretically result in memory
-	 * corruption if an invalidation happens at the same time as an VA
-	 * translation. Avoid this by doing the invalidation from the SDMA
-	 * itself.
-	 */
-	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
-				     AMDGPU_FENCE_OWNER_UNDEFINED,
-				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-				     &job);
-	if (r)
-		goto error_alloc;
-
-	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-	job->vm_needs_flush = true;
-	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-	fence = amdgpu_job_submit(job);
-
 	mutex_unlock(&adev->mman.gtt_window_lock);
 
-	dma_fence_wait(fence, false);
-	dma_fence_put(fence);
-
-	return;
+	fence = amdgpu_sdma_flush_tlb(adev);
+	if (fence) {
+		dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		return;
+	}
 
-error_alloc:
-	mutex_unlock(&adev->mman.gtt_window_lock);
-	DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
+	DRM_ERROR("Error flushing GPU TLB using the SDMA !\n");
 }
 
 /**
Signature · 1 line
2.40.1
maintainer
#1 lore raw

Am 09.06.23 um 12:02 schrieb Shashank Sharma:

Quoted 24 lines

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41 ++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
  drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------
  3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +3 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
  /* SDMA CSA reside in the 3rd page of CSA */
  #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 
+/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c line 45

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
+ * @adev: amdgpu device pointer
+ *
+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA
+ * itself.
+ */
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev)

That function is actually not really SDMA related. It just happens that the SDMA is used for buffer moves and we needed that as workaround for HW problems.

Maybe add a function to amdgpu_ttm.c with the implementation to use any ring for TLB flush and a wrapper here to use the SDMA for TLB flush.

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c line 50

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +5 -0
+{
+	int r;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;

Variables like "r" last and longer lines first please.

Apart from those nit picks looks good to me,
Christian.

Quoted 101 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +23 -0
+
+	mutex_lock(&adev->mman.gtt_window_lock);
+	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
+				     AMDGPU_FENCE_OWNER_UNDEFINED,
+				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     &job);
+	if (r)
+		goto error_alloc;
+
+	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+	job->vm_needs_flush = true;
+	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	fence = amdgpu_job_submit(job);
+
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return fence;
+
+error_alloc:
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return NULL;
+}
+
  /*
   * GPU SDMA IP block helpers function.
   */
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
          bool duplicate);
  void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev);
  int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
 
  #endif
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +8 -28
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
  #include "athub_v2_1.h"
 
  #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
 
  #if 0
  static const struct soc15_reg_golden golden_settings_navi10_hdp[] =
@@ -333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
  {
  	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
  	struct dma_fence *fence;
-	struct amdgpu_job *job;
-
-	int r;
 
  	/* flush hdp cache */
  	adev->hdp.funcs->flush_hdp(adev, NULL);
@@ -378,34 +376,16 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
  		return;
  	}
 
-	/* The SDMA on Navi has a bug which can theoretically result in memory
-	 * corruption if an invalidation happens at the same time as an VA
-	 * translation. Avoid this by doing the invalidation from the SDMA
-	 * itself.
-	 */
-	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
-				     AMDGPU_FENCE_OWNER_UNDEFINED,
-				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-				     &job);
-	if (r)
-		goto error_alloc;
-
-	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-	job->vm_needs_flush = true;
-	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-	fence = amdgpu_job_submit(job);
-
  	mutex_unlock(&adev->mman.gtt_window_lock);
 
-	dma_fence_wait(fence, false);
-	dma_fence_put(fence);
-
-	return;
+	fence = amdgpu_sdma_flush_tlb(adev);
+	if (fence) {
+		dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		return;
+	}
 
-error_alloc:
-	mutex_unlock(&adev->mman.gtt_window_lock);
-	DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
+	DRM_ERROR("Error flushing GPU TLB using the SDMA !\n");
  }
 
  /**
#2 lore raw
Quoted 37 lines · Christian König

Am 09.06.23 um 12:02 schrieb Shashank Sharma:

Quoted 25 lines

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 + drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------

  3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +0 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
   /* SDMA CSA reside in the 3rd page of CSA */
   #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)

  +/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *
+ * @adev: amdgpu device pointer
+ *

Quoted 8 lines

+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA
+ * itself.
+ */
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev)

That function is actually not really SDMA related. It just happens that the SDMA is used for buffer moves and we needed that as workaround for HW problems.

Quoted 1 line

Maybe add a function to amdgpu_ttm.c with the implementation to use any ring for TLB flush and a wrapper here to use the SDMA for TLB flush.

Noted,

Quoted 9 lines
Quoted 6 lines

+{
+    int r;
+    struct dma_fence *fence;
+    struct amdgpu_job *job;
+    struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;

Variables like "r" last and longer lines first please.

Noted, I will do these changes and resend the patch.

- Shashank

Quoted 100 lines

Apart from those nit picks looks good to me,
Christian.

Quoted 96 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +23 -0
+
+    mutex_lock(&adev->mman.gtt_window_lock);
+    r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
+                     AMDGPU_FENCE_OWNER_UNDEFINED,
+                     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
+                     &job);
+    if (r)
+        goto error_alloc;
+
+    job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+    job->vm_needs_flush = true;
+    job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+    amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+    fence = amdgpu_job_submit(job);
+
+    mutex_unlock(&adev->mman.gtt_window_lock);
+    return fence;
+
+error_alloc:
+    mutex_unlock(&adev->mman.gtt_window_lock);
+    return NULL;
+}
+

  /*
   * GPU SDMA IP block helpers function.
   */

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct 
 amdgpu_device *adev,
           bool duplicate);
   void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev);
   int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
     #endif
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1 -0
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
   #include "athub_v2_1.h"
     #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
     #if 0

  static const struct soc15_reg_golden golden_settings_navi10_hdp[] =

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0 -3
@@ -333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct 
 amdgpu_device *adev, uint32_t vmid,
   {
       struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
       struct dma_fence *fence;
-    struct amdgpu_job *job;
-
-    int r;
         /* flush hdp cache */
       adev->hdp.funcs->flush_hdp(adev, NULL);
@@ -378,34 +376,16 @@ static void gmc_v10_0_flush_gpu_tlb(struct 
 amdgpu_device *adev, uint32_t vmid,
           return;
       }

  -    /* The SDMA on Navi has a bug which can theoretically result in memory -     * corruption if an invalidation happens at the same time as an VA -     * translation. Avoid this by doing the invalidation from the SDMA -     * itself.
-     */
-    r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity, -                     AMDGPU_FENCE_OWNER_UNDEFINED,
-                     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-                     &job);
-    if (r)
-        goto error_alloc;

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0 -7
-
-    job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-    job->vm_needs_flush = true;
-    job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-    amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-    fence = amdgpu_job_submit(job);
-

      mutex_unlock(&adev->mman.gtt_window_lock);
  -    dma_fence_wait(fence, false);
-    dma_fence_put(fence);
-
-    return;
+    fence = amdgpu_sdma_flush_tlb(adev);
+    if (fence) {
+        dma_fence_wait(fence, false);
+        dma_fence_put(fence);
+        return;
+    }
  -error_alloc:
-    mutex_unlock(&adev->mman.gtt_window_lock);
-    DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r); +    DRM_ERROR("Error flushing GPU TLB using the SDMA !\n"); }
    /**

#3 lore raw
In reply to Shashank Sharma
Quoted 31 lines · Shashank Sharma

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41 ++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
  drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------
  3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
  /* SDMA CSA reside in the 3rd page of CSA */
  #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 
+/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *
+ * @adev: amdgpu device pointer
+ *
+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c line 52

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
+ * itself.
+ */
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev)
+{
+	int r;
+	struct dma_fence *fence;
+	struct amdgpu_job *job;
+	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+
+	mutex_lock(&adev->mman.gtt_window_lock);

gtt_window_lock is for GART copy buffer/window sharing, here we use copy ring but don't use copy buffer, and ring ib sa and job submit has spinlock, so I think gtt_window_lock is not needed.

gtt_window_lock should be removed from gmc_v10_0_flush_gpu_tlb too.

Regards,

Philip

Quoted 99 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +21 -0
+	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
+				     AMDGPU_FENCE_OWNER_UNDEFINED,
+				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
+				     &job);
+	if (r)
+		goto error_alloc;
+
+	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+	job->vm_needs_flush = true;
+	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+	fence = amdgpu_job_submit(job);
+
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return fence;
+
+error_alloc:
+	mutex_unlock(&adev->mman.gtt_window_lock);
+	return NULL;
+}
+
  /*
   * GPU SDMA IP block helpers function.
   */
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
          bool duplicate);
  void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev);
  int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
 
  #endif
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +8 -28
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
  #include "athub_v2_1.h"
 
  #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
 
  #if 0
  static const struct soc15_reg_golden golden_settings_navi10_hdp[] =
@@ -333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
  {
  	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
  	struct dma_fence *fence;
-	struct amdgpu_job *job;
-
-	int r;
 
  	/* flush hdp cache */
  	adev->hdp.funcs->flush_hdp(adev, NULL);
@@ -378,34 +376,16 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
  		return;
  	}
 
-	/* The SDMA on Navi has a bug which can theoretically result in memory
-	 * corruption if an invalidation happens at the same time as an VA
-	 * translation. Avoid this by doing the invalidation from the SDMA
-	 * itself.
-	 */
-	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
-				     AMDGPU_FENCE_OWNER_UNDEFINED,
-				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-				     &job);
-	if (r)
-		goto error_alloc;
-
-	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-	job->vm_needs_flush = true;
-	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-	fence = amdgpu_job_submit(job);
-
  	mutex_unlock(&adev->mman.gtt_window_lock);
 
-	dma_fence_wait(fence, false);
-	dma_fence_put(fence);
-
-	return;
+	fence = amdgpu_sdma_flush_tlb(adev);
+	if (fence) {
+		dma_fence_wait(fence, false);
+		dma_fence_put(fence);
+		return;
+	}
 
-error_alloc:
-	mutex_unlock(&adev->mman.gtt_window_lock);
-	DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
+	DRM_ERROR("Error flushing GPU TLB using the SDMA !\n");
  }
 
  /**
maintainer
#4 lore raw

Am 09.06.23 um 15:32 schrieb Philip Yang:

Quoted 36 lines · Shashank Sharma
Quoted 34 lines

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 + drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------

  3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +0 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
   /* SDMA CSA reside in the 3rd page of CSA */
   #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)

  +/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *
+ * @adev: amdgpu device pointer
+ *
+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA
+ * itself.
+ */

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +2 -0
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev)
+{
Quoted 8 lines
Quoted 6 lines

+    int r;
+    struct dma_fence *fence;
+    struct amdgpu_job *job;
+    struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; +
+    mutex_lock(&adev->mman.gtt_window_lock);

gtt_window_lock is for GART copy buffer/window sharing, here we use copy ring but don't use copy buffer, and ring ib sa and job submit has spinlock, so I think gtt_window_lock is not needed.

We need something to protect the entity while pushing jobs to it and that's the gtt_window_lock, so that can't be removed here.

Christian.

Quoted 101 lines

gtt_window_lock should be removed from gmc_v10_0_flush_gpu_tlb too.

Regards,

Philip

Quoted 94 lines

+    r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity, +                     AMDGPU_FENCE_OWNER_UNDEFINED,
+                     16 * 4, AMDGPU_IB_POOL_IMMEDIATE, +                     &job);
+    if (r)
+        goto error_alloc;

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +15 -0
+
+    job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+    job->vm_needs_flush = true;
+    job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+    amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+    fence = amdgpu_job_submit(job);
+
+    mutex_unlock(&adev->mman.gtt_window_lock);
+    return fence;
+
+error_alloc:
+    mutex_unlock(&adev->mman.gtt_window_lock);
+    return NULL;
+}
+

  /*
   * GPU SDMA IP block helpers function.
   */

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct 
 amdgpu_device *adev,
           bool duplicate);
   void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev);
   int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
     #endif
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1 -0
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
   #include "athub_v2_1.h"
     #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
     #if 0

  static const struct soc15_reg_golden golden_settings_navi10_hdp[] =

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0 -3
@@ -333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct 
 amdgpu_device *adev, uint32_t vmid,
   {
       struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
       struct dma_fence *fence;
-    struct amdgpu_job *job;
-
-    int r;
         /* flush hdp cache */
       adev->hdp.funcs->flush_hdp(adev, NULL);
@@ -378,34 +376,16 @@ static void gmc_v10_0_flush_gpu_tlb(struct 
 amdgpu_device *adev, uint32_t vmid,
           return;
       }

  -    /* The SDMA on Navi has a bug which can theoretically result in memory -     * corruption if an invalidation happens at the same time as an VA -     * translation. Avoid this by doing the invalidation from the SDMA -     * itself.
-     */
-    r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity, -                     AMDGPU_FENCE_OWNER_UNDEFINED,
-                     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-                     &job);
-    if (r)
-        goto error_alloc;

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0 -7
-
-    job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-    job->vm_needs_flush = true;
-    job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-    amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-    fence = amdgpu_job_submit(job);
-

      mutex_unlock(&adev->mman.gtt_window_lock);
  -    dma_fence_wait(fence, false);
-    dma_fence_put(fence);
-
-    return;
+    fence = amdgpu_sdma_flush_tlb(adev);
+    if (fence) {
+        dma_fence_wait(fence, false);
+        dma_fence_put(fence);
+        return;
+    }
  -error_alloc:
-    mutex_unlock(&adev->mman.gtt_window_lock);
-    DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r); +    DRM_ERROR("Error flushing GPU TLB using the SDMA !\n"); }
    /**

#5 lore raw
In reply to Shashank Sharma

[Public]

Quoted 43 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Shashank Sharma
Sent: Friday, June 9, 2023 6:03 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Somalapuram, Amaranath <[email protected]>; Koenig, Christian <[email protected]>; Sharma, Shashank <[email protected]>
Subject: [RFC] drm/amdgpu: flush gpu tlb using sdma ring

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41
++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------
 3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
 /* SDMA CSA reside in the 3rd page of CSA */  #define
 AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 
+/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *
+ * @adev: amdgpu device pointer
+ *
+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c line 52

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
+ * itself.
+ */
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev) {
+     int r;
+     struct dma_fence *fence;
+     struct amdgpu_job *job;
+     struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+
+     mutex_lock(&adev->mman.gtt_window_lock);
+     r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,

I guess this patch needs to be reworked on top of recent entity rename to high_pr code.

Regards,
Guchun

Quoted 105 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +20 -0
+                                  AMDGPU_FENCE_OWNER_UNDEFINED,
+                                  16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
+                                  &job);
+     if (r)
+             goto error_alloc;
+
+     job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+     job->vm_needs_flush = true;
+     job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+     amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+     fence = amdgpu_job_submit(job);
+
+     mutex_unlock(&adev->mman.gtt_window_lock);
+     return fence;
+
+error_alloc:
+     mutex_unlock(&adev->mman.gtt_window_lock);
+     return NULL;
+}
+
 /*
  * GPU SDMA IP block helpers function.
  */
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct
 amdgpu_device *adev,
         bool duplicate);
 void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device
 *adev);  int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
 

#endif

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1 -0
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
 #include "athub_v2_1.h"
 
 #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
 
 #if 0
 static const struct soc15_reg_golden golden_settings_navi10_hdp[] = @@ -

333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct

amdgpu_device *adev, uint32_t vmid,  {
      struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
      struct dma_fence *fence;
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0 -3
-     struct amdgpu_job *job;
-
-     int r;
 
      /* flush hdp cache */
      adev->hdp.funcs->flush_hdp(adev, NULL); @@ -378,34 +376,16 @@

static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,

              return;
      }

- /* The SDMA on Navi has a bug which can theoretically result in
memory

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +7 -25
-      * corruption if an invalidation happens at the same time as an VA
-      * translation. Avoid this by doing the invalidation from the SDMA
-      * itself.
-      */
-     r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
-                                  AMDGPU_FENCE_OWNER_UNDEFINED,
-                                  16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-                                  &job);
-     if (r)
-             goto error_alloc;
-
-     job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-     job->vm_needs_flush = true;
-     job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-     amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-     fence = amdgpu_job_submit(job);
-
      mutex_unlock(&adev->mman.gtt_window_lock);
 
-     dma_fence_wait(fence, false);
-     dma_fence_put(fence);
-
-     return;
+     fence = amdgpu_sdma_flush_tlb(adev);
+     if (fence) {
+             dma_fence_wait(fence, false);
+             dma_fence_put(fence);
+             return;
+     }
 
-error_alloc:
-     mutex_unlock(&adev->mman.gtt_window_lock);
-     DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
+     DRM_ERROR("Error flushing GPU TLB using the SDMA !\n");
 }
 
 /**
--

2.40.1

#6 lore raw

Hello Guchun,

Thanks for your comments.

Quoted 48 lines · Guchun

[Public]

Quoted 46 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Shashank Sharma
Sent: Friday, June 9, 2023 6:03 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>; Somalapuram, Amaranath <[email protected]>; Koenig, Christian <[email protected]>; Sharma, Shashank <[email protected]>
Subject: [RFC] drm/amdgpu: flush gpu tlb using sdma ring

This RFC patch moves the code to flush TLB using SDMA ring to a different place like a separate function. The idea is that KFD/KGD code should be able to flush GPU TLB using SDMA if they want to.

Cc Christian Koenig <[email protected]>
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 41
++++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  1 +
  drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c   | 36 +++++----------------
  3 files changed, 50 insertions(+), 28 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +13 -0
index 231ca06bc9c7..20a5dad32bfc 100644
@@ -30,6 +30,47 @@
  /* SDMA CSA reside in the 3rd page of CSA */  #define
 AMDGPU_CSA_SDMA_OFFSET (4096 * 2)
 
+/**
+ * amdgpu_sdma_flush_tlb - flush gpu TLB using SDMA ring
+ *
+ * @adev: amdgpu device pointer
+ *
+ * return: returns dma fence which must be put by caller
+ *
+ * The SDMA on Navi has a bug which can theoretically result in memory
+ * corruption if an invalidation happens at the same time as an VA
+ * translation. Avoid this by doing the invalidation from the SDMA
+ * itself.
+ */
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev) {
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c line 53

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +7 -0
+     int r;
+     struct dma_fence *fence;
+     struct amdgpu_job *job;
+     struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+
+     mutex_lock(&adev->mman.gtt_window_lock);
+     r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,

I guess this patch needs to be reworked on top of recent entity rename to high_pr code.

Probably, my code was based on V6.3, I might have to fast forward.

- Shashank

Quoted 108 lines

Regards,
Guchun

Quoted 105 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +20 -0
+                                  AMDGPU_FENCE_OWNER_UNDEFINED,
+                                  16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
+                                  &job);
+     if (r)
+             goto error_alloc;
+
+     job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
+     job->vm_needs_flush = true;
+     job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
+     amdgpu_ring_pad_ib(ring, &job->ibs[0]);
+     fence = amdgpu_job_submit(job);
+
+     mutex_unlock(&adev->mman.gtt_window_lock);
+     return fence;
+
+error_alloc:
+     mutex_unlock(&adev->mman.gtt_window_lock);
+     return NULL;
+}
+
  /*
   * GPU SDMA IP block helpers function.
   */
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
index fc8528812598..c895948f6e82 100644
@@ -130,5 +130,6 @@ void amdgpu_sdma_destroy_inst_ctx(struct
 amdgpu_device *adev,
          bool duplicate);
  void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device
 *adev);  int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
+struct dma_fence *amdgpu_sdma_flush_tlb(struct amdgpu_device *adev);
 
  #endif
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1 -0
index ab2556ca984e..0bfaee00a838 100644
@@ -52,6 +52,7 @@
  #include "athub_v2_1.h"
 
  #include "amdgpu_reset.h"
+#include "amdgpu_sdma.h"
 
  #if 0
  static const struct soc15_reg_golden golden_settings_navi10_hdp[] = @@ -

333,9 +334,6 @@ static void gmc_v10_0_flush_gpu_tlb(struct

amdgpu_device *adev, uint32_t vmid,  {
       struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
       struct dma_fence *fence;
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +0 -3
-     struct amdgpu_job *job;
-
-     int r;
 
       /* flush hdp cache */
       adev->hdp.funcs->flush_hdp(adev, NULL); @@ -378,34 +376,16 @@

static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,

               return;
       }

- /* The SDMA on Navi has a bug which can theoretically result in
memory

drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +7 -25
-      * corruption if an invalidation happens at the same time as an VA
-      * translation. Avoid this by doing the invalidation from the SDMA
-      * itself.
-      */
-     r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.entity,
-                                  AMDGPU_FENCE_OWNER_UNDEFINED,
-                                  16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
-                                  &job);
-     if (r)
-             goto error_alloc;
-
-     job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
-     job->vm_needs_flush = true;
-     job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
-     amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-     fence = amdgpu_job_submit(job);
-
       mutex_unlock(&adev->mman.gtt_window_lock);
 
-     dma_fence_wait(fence, false);
-     dma_fence_put(fence);
-
-     return;
+     fence = amdgpu_sdma_flush_tlb(adev);
+     if (fence) {
+             dma_fence_wait(fence, false);
+             dma_fence_put(fence);
+             return;
+     }
 
-error_alloc:
-     mutex_unlock(&adev->mman.gtt_window_lock);
-     DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
+     DRM_ERROR("Error flushing GPU TLB using the SDMA !\n");
  }
 
  /**
--

2.40.1