Skip to contentSkip to conversation

drm/amd/amdgpu: add tracing of kernel DMA mappings and enable on init (v2)

PATCH In review Tom St Denis 2 participants 6 messages Aug 14, 2017 – Aug 15, 2017 5 messages not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 drm/amd/amdgpu: add tracing of kernel DMA mappings and enable on init (v2)

This patch adds trace points to the amdgpu bo_kmap/bo/kunmap functions which capture internal allocations.

(v2): Simply add a sleep to the init. Users can use this script to load with map debugging enabled:

#!/bin/bash
modprobe amdgpu map_debug=1 &
sleep 1
echo 1 > /sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_unpopulate/enable echo 1 > /sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_populate/enable

---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    | 12 ++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 ++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    |  4 ++--
 4 files changed, 25 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index d2aaad77c353..2f5781df88c5 100644
@@ -121,6 +121,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
 extern int amdgpu_param_buf_per_se;
 extern int amdgpu_job_hang_limit;
 extern int amdgpu_lbpw;
+extern int amdgpu_enable_map_debugging;
 
 #ifdef CONFIG_DRM_AMDGPU_SI
 extern int amdgpu_si_support;
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +12 -0
index 2cdf8443e7d3..0ed777cc2f63 100644
@@ -120,6 +120,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
 int amdgpu_param_buf_per_se = 0;
 int amdgpu_job_hang_limit = 0;
 int amdgpu_lbpw = -1;
+int amdgpu_enable_map_debugging = 0;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -263,6 +264,9 @@ module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
 MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(lbpw, amdgpu_lbpw, int, 0444);
 
+MODULE_PARM_DESC(map_debug, "Enable map debugging in kernel (1 = on, 0 = off)");
+module_param_named(map_debug, amdgpu_enable_map_debugging, int, 0444);
+
 #ifdef CONFIG_DRM_AMDGPU_SI
 
 #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
@@ -865,11 +869,19 @@ static struct pci_driver amdgpu_kms_pci_driver = {
 };
 
 
+// enable trace events during init
+static void amdgpu_enable_dma_tracers(void)
+{
+	msleep(3000);
+}
 
 static int __init amdgpu_init(void)
 {
 	int r;
 
+	if (amdgpu_enable_map_debugging)
+		amdgpu_enable_dma_tracers();
+
 	r = amdgpu_sync_init();
 	if (r)
 		goto error_sync;
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +10 -1
index e7e899190bef..a292e86fbaa7 100644
@@ -37,6 +37,9 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
+void amdgpu_trace_dma_map(struct ttm_tt *ttm);
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm);
+
 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -625,6 +628,9 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
 	if (r)
 		return r;
 
+	if (unlikely(trace_amdgpu_ttm_tt_populate_enabled()) && bo->tbo.mem.mem_type != TTM_PL_VRAM)
+		amdgpu_trace_dma_map(bo->tbo.ttm);
+
 	if (ptr)
 		*ptr = amdgpu_bo_kptr(bo);
 
@@ -640,8 +646,11 @@ void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
 
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
 {
-	if (bo->kmap.bo)
+	if (bo->kmap.bo) {
+		if (unlikely(trace_amdgpu_ttm_tt_unpopulate_enabled()) && bo->tbo.mem.mem_type != TTM_PL_VRAM)
+			amdgpu_trace_dma_unmap(bo->tbo.ttm);
 		ttm_bo_kunmap(&bo->kmap);
+	}
 }
 
 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +2 -2
index 26665b4baf36..85727be3fbb7 100644
@@ -663,7 +663,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
 	return r;
 }
 
-static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
+void amdgpu_trace_dma_map(struct ttm_tt *ttm)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -679,7 +679,7 @@ static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
 	}
 }
 
-static void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
Signature · 6 lines
2.12.0

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): [email protected]

#2 lore raw

Am 14.08.2017 um 13:37 schrieb Tom St Denis:

Quoted 11 lines

This patch adds trace points to the amdgpu bo_kmap/bo/kunmap functions which capture internal allocations.

(v2): Simply add a sleep to the init. Users can use this script to load with map debugging enabled:

#!/bin/bash
modprobe amdgpu map_debug=1 &
sleep 1
echo 1 > /sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_unpopulate/enable echo 1 > /sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_populate/enable

I've just realized that there is a far simpler method than that or enabling trace points using the module command line.

Assuming your GPU is PCI device 01:00.0 connected through bridge 00:02.1 (use lspci -t to figure the connection out):

# Temporary remove the PCIe device from the bus
echo 1 > /sys/bus/pci/devices/0000\:01\:00.0/remove
# Load amdgpu, this allows you to enable the trace points you want and also sets probes etc.. modprobe amdpgu
# Rescan the bus, the device subsystem will automatically probe amdgpu with this device echo 1 > /sys/bus/pci/devices/0000\:00\:02.1/rescan

Apart from that I would really like to see those trace points in TTM instead. I also don't mind adding a dev or pdev pointer to the ttm_bo_device structure for this.

Regards,
Christian.

Quoted 114 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    | 12 ++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 ++++++++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    |  4 ++--
  4 files changed, 25 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index d2aaad77c353..2f5781df88c5 100644
@@ -121,6 +121,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
  extern int amdgpu_param_buf_per_se;
  extern int amdgpu_job_hang_limit;
  extern int amdgpu_lbpw;
+extern int amdgpu_enable_map_debugging;
 
  #ifdef CONFIG_DRM_AMDGPU_SI
  extern int amdgpu_si_support;
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +12 -0
index 2cdf8443e7d3..0ed777cc2f63 100644
@@ -120,6 +120,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
  int amdgpu_param_buf_per_se = 0;
  int amdgpu_job_hang_limit = 0;
  int amdgpu_lbpw = -1;
+int amdgpu_enable_map_debugging = 0;
 
  MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
  module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -263,6 +264,9 @@ module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
  MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = enable, 0 = disable, -1 = auto)");
  module_param_named(lbpw, amdgpu_lbpw, int, 0444);
 
+MODULE_PARM_DESC(map_debug, "Enable map debugging in kernel (1 = on, 0 = off)");
+module_param_named(map_debug, amdgpu_enable_map_debugging, int, 0444);
+
  #ifdef CONFIG_DRM_AMDGPU_SI
 
  #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
@@ -865,11 +869,19 @@ static struct pci_driver amdgpu_kms_pci_driver = {
  };
 
 
+// enable trace events during init
+static void amdgpu_enable_dma_tracers(void)
+{
+	msleep(3000);
+}
 
  static int __init amdgpu_init(void)
  {
  	int r;
 
+	if (amdgpu_enable_map_debugging)
+		amdgpu_enable_dma_tracers();
+
  	r = amdgpu_sync_init();
  	if (r)
  		goto error_sync;
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +10 -1
index e7e899190bef..a292e86fbaa7 100644
@@ -37,6 +37,9 @@
  #include "amdgpu.h"
  #include "amdgpu_trace.h"
 
+void amdgpu_trace_dma_map(struct ttm_tt *ttm);
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm);
+
  static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  {
  	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -625,6 +628,9 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
  	if (r)
  		return r;
 
+	if (unlikely(trace_amdgpu_ttm_tt_populate_enabled()) && bo->tbo.mem.mem_type != TTM_PL_VRAM)
+		amdgpu_trace_dma_map(bo->tbo.ttm);
+
  	if (ptr)
  		*ptr = amdgpu_bo_kptr(bo);
 
@@ -640,8 +646,11 @@ void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
 
  void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
  {
-	if (bo->kmap.bo)
+	if (bo->kmap.bo) {
+		if (unlikely(trace_amdgpu_ttm_tt_unpopulate_enabled()) && bo->tbo.mem.mem_type != TTM_PL_VRAM)
+			amdgpu_trace_dma_unmap(bo->tbo.ttm);
  		ttm_bo_kunmap(&bo->kmap);
+	}
  }
 
  struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
index 26665b4baf36..85727be3fbb7 100644
@@ -663,7 +663,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
  	return r;
  }
 
-static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
+void amdgpu_trace_dma_map(struct ttm_tt *ttm)
  {
  	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static void amdgpu_trace_dma_map(struct ttm_tt *ttm) · line 685

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
@@ -679,7 +679,7 @@ static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
  	}
  }
 
-static void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
  {
  	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
  	struct amdgpu_ttm_tt *gtt = (void *)ttm;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): e5a32833-bbb0-7ec8-2b42-91cb58c78ca2-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#4 lore raw
Quoted 17 lines · Christian König

Am 14.08.2017 um 13:37 schrieb Tom St Denis:

Quoted 6 lines

This patch adds trace points to the amdgpu bo_kmap/bo/kunmap functions which capture internal allocations.

(v2): Simply add a sleep to the init. Users can use this script to load with map debugging enabled:

Quoted 7 lines

#!/bin/bash
modprobe amdgpu map_debug=1 &
sleep 1
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_unpopulate/enable
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_populate/enable

I've just realized that there is a far simpler method than that or enabling trace points using the module command line.

Quoted 8 lines

Assuming your GPU is PCI device 01:00.0 connected through bridge 00:02.1 (use lspci -t to figure the connection out):

# Temporary remove the PCIe device from the bus
echo 1 > /sys/bus/pci/devices/0000\:01\:00.0/remove
# Load amdgpu, this allows you to enable the trace points you want and also sets probes etc.. modprobe amdpgu
# Rescan the bus, the device subsystem will automatically probe amdgpu with this device echo 1 > /sys/bus/pci/devices/0000\:00\:02.1/rescan

That would definitely be a bunch trickier to script up. Nobody will run this by hand. So we'd need to remove all devices that are AMD vendor ID but GPU only (e.g. not other controllers) which means we need to parse the lspci output for VGA controller related text.

Definitely doable and probably "cleaner" from a race point of view.

Quoted 1 line

Apart from that I would really like to see those trace points in TTM instead. I also don't mind adding a dev or pdev pointer to the ttm_bo_device structure for this.

Not against this in principle. There's a bit of confusion... our ttm amdgpu functions are passed

struct ttm_tt *ttm

which we cast to

struct amdgpu_ttm_tt {
	struct ttm_dma_tt	ttm;
	struct amdgpu_device	*adev;
	u64			offset;

... <snip>

Then we use the "->ttm" of that to get dma_address[] but pages[] from the original ttm_tt structure ... from what I recall (when I looked into this last week) ttm_dma_tt and ttm_tt aren't the same structure right? (and looking at it right now ttm_dma_tt begins with ttm_tt....).

So wherever I put the trace in ttm I need access to dma_address[] which doesn't seem so clear cut. Can I just cast a ttm_tt pointer to ttm_dma_tt?

Tom

Quoted 109 lines

Regards,
Christian.

Quoted 105 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    | 12 ++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 ++++++++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    |  4 ++--
  4 files changed, 25 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index d2aaad77c353..2f5781df88c5 100644
@@ -121,6 +121,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
  extern int amdgpu_param_buf_per_se;
  extern int amdgpu_job_hang_limit;
  extern int amdgpu_lbpw;
+extern int amdgpu_enable_map_debugging;
  #ifdef CONFIG_DRM_AMDGPU_SI
  extern int amdgpu_si_support;
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
index 2cdf8443e7d3..0ed777cc2f63 100644
@@ -120,6 +120,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
  int amdgpu_param_buf_per_se = 0;
  int amdgpu_job_hang_limit = 0;
  int amdgpu_lbpw = -1;
+int amdgpu_enable_map_debugging = 0;
  MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
  module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -263,6 +264,9 @@ module_param_named(job_hang_limit, 
 amdgpu_job_hang_limit, int ,0444);
  MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 = 
 enable, 0 = disable, -1 = auto)");
  module_param_named(lbpw, amdgpu_lbpw, int, 0444);
+MODULE_PARM_DESC(map_debug, "Enable map debugging in kernel (1 = on, 
 0 = off)");
+module_param_named(map_debug, amdgpu_enable_map_debugging, int, 0444);
+
  #ifdef CONFIG_DRM_AMDGPU_SI
  #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +8 -0
@@ -865,11 +869,19 @@ static struct pci_driver amdgpu_kms_pci_driver = {
  };
+// enable trace events during init
+static void amdgpu_enable_dma_tracers(void)
+{
+    msleep(3000);
+}
  static int __init amdgpu_init(void)
  {
      int r;
+    if (amdgpu_enable_map_debugging)
+        amdgpu_enable_dma_tracers();
+
      r = amdgpu_sync_init();
      if (r)
          goto error_sync;
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +10 -1
index e7e899190bef..a292e86fbaa7 100644
@@ -37,6 +37,9 @@
  #include "amdgpu.h"
  #include "amdgpu_trace.h"
+void amdgpu_trace_dma_map(struct ttm_tt *ttm);
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm);
+
  static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  {
      struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -625,6 +628,9 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
      if (r)
          return r;
+    if (unlikely(trace_amdgpu_ttm_tt_populate_enabled()) && 
 bo->tbo.mem.mem_type != TTM_PL_VRAM)
+        amdgpu_trace_dma_map(bo->tbo.ttm);
+
      if (ptr)
          *ptr = amdgpu_bo_kptr(bo);
@@ -640,8 +646,11 @@ void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
  void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
  {
-    if (bo->kmap.bo)
+    if (bo->kmap.bo) {
+        if (unlikely(trace_amdgpu_ttm_tt_unpopulate_enabled()) && 
 bo->tbo.mem.mem_type != TTM_PL_VRAM)
+            amdgpu_trace_dma_unmap(bo->tbo.ttm);
          ttm_bo_kunmap(&bo->kmap);
+    }
  }
  struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
index 26665b4baf36..85727be3fbb7 100644
@@ -663,7 +663,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt 
 *ttm, struct page **pages)
      return r;
  }
-static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
+void amdgpu_trace_dma_map(struct ttm_tt *ttm)
  {
      struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
      struct amdgpu_ttm_tt *gtt = (void *)ttm;
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static void amdgpu_trace_dma_map(struct ttm_tt *ttm) · line 684

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
@@ -679,7 +679,7 @@ static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
      }
  }
-static void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
  {
      struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
      struct amdgpu_ttm_tt *gtt = (void *)ttm;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): e98b3321-d11e-83f0-cd66-184d81764db9-5C7GfCeVMHo@public.gmane.org

#6 lore raw

Am 15.08.2017 um 13:33 schrieb Tom St Denis:

Quoted 35 lines · Christian König
Quoted 17 lines

Am 14.08.2017 um 13:37 schrieb Tom St Denis:

Quoted 6 lines

This patch adds trace points to the amdgpu bo_kmap/bo/kunmap functions which capture internal allocations.

(v2): Simply add a sleep to the init. Users can use this script to load with map debugging enabled:

Quoted 7 lines

#!/bin/bash
modprobe amdgpu map_debug=1 &
sleep 1
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_unpopulate/enable
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_populate/enable

I've just realized that there is a far simpler method than that or enabling trace points using the module command line.

Quoted 8 lines

Assuming your GPU is PCI device 01:00.0 connected through bridge 00:02.1 (use lspci -t to figure the connection out):

# Temporary remove the PCIe device from the bus
echo 1 > /sys/bus/pci/devices/0000\:01\:00.0/remove
# Load amdgpu, this allows you to enable the trace points you want and also sets probes etc.. modprobe amdpgu
# Rescan the bus, the device subsystem will automatically probe amdgpu with this device echo 1 > /sys/bus/pci/devices/0000\:00\:02.1/rescan

That would definitely be a bunch trickier to script up. Nobody will run this by hand. So we'd need to remove all devices that are AMD vendor ID but GPU only (e.g. not other controllers) which means we need to parse the lspci output for VGA controller related text.

Definitely doable and probably "cleaner" from a race point of view.

Quoted 1 line

Apart from that I would really like to see those trace points in TTM instead. I also don't mind adding a dev or pdev pointer to the ttm_bo_device structure for this.

Not against this in principle. There's a bit of confusion... our ttm amdgpu functions are passed

Quoted 7 lines

struct ttm_tt *ttm

which we cast to

struct amdgpu_ttm_tt {
    struct ttm_dma_tt    ttm;
    struct amdgpu_device    *adev;

BTW: Isn't this adev pointer here superflous? I mean the ttm_tt has a bdev pointer to use, don't they?

Quoted 6 lines
    u64 offset;

... <snip>

Then we use the "->ttm" of that to get dma_address[] but pages[] from the original ttm_tt structure ... from what I recall (when I looked into this last week) ttm_dma_tt and ttm_tt aren't the same structure right? (and looking at it right now ttm_dma_tt begins with ttm_tt....).

So wherever I put the trace in ttm I need access to dma_address[] which doesn't seem so clear cut. Can I just cast a ttm_tt pointer to ttm_dma_tt?

Ah, ok now I understand your problem. Well, printing the dma addresses in this situation is probably not a good idea to start with.

When a buffer object is mapped to kernel space there is no guarantee that it already has it's DMA mapping. For amdgpu we probably never run into this situation, but for other drivers that might now be the case.

Do we really need that info?

Christian.

Quoted 127 lines

Tom

Quoted 113 lines

Regards,
Christian.

Quoted 109 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    | 12 ++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 11 ++++++++++-
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    |  4 ++--
  4 files changed, 25 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu.h +1 -0
index d2aaad77c353..2f5781df88c5 100644
@@ -121,6 +121,7 @@ extern int amdgpu_cntl_sb_buf_per_se;
  extern int amdgpu_param_buf_per_se;
  extern int amdgpu_job_hang_limit;
  extern int amdgpu_lbpw;
+extern int amdgpu_enable_map_debugging;
  #ifdef CONFIG_DRM_AMDGPU_SI
  extern int amdgpu_si_support;
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +4 -0
index 2cdf8443e7d3..0ed777cc2f63 100644
@@ -120,6 +120,7 @@ int amdgpu_cntl_sb_buf_per_se = 0;
  int amdgpu_param_buf_per_se = 0;
  int amdgpu_job_hang_limit = 0;
  int amdgpu_lbpw = -1;
+int amdgpu_enable_map_debugging = 0;
  MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in 
 megabytes");
  module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -263,6 +264,9 @@ module_param_named(job_hang_limit, 
 amdgpu_job_hang_limit, int ,0444);
  MODULE_PARM_DESC(lbpw, "Load Balancing Per Watt (LBPW) support (1 
 = enable, 0 = disable, -1 = auto)");
  module_param_named(lbpw, amdgpu_lbpw, int, 0444);
+MODULE_PARM_DESC(map_debug, "Enable map debugging in kernel (1 = 
 on, 0 = off)");
+module_param_named(map_debug, amdgpu_enable_map_debugging, int, 0444);
+
  #ifdef CONFIG_DRM_AMDGPU_SI
  #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +8 -0
@@ -865,11 +869,19 @@ static struct pci_driver amdgpu_kms_pci_driver 
 = {
  };
+// enable trace events during init
+static void amdgpu_enable_dma_tracers(void)
+{
+    msleep(3000);
+}
  static int __init amdgpu_init(void)
  {
      int r;
+    if (amdgpu_enable_map_debugging)
+        amdgpu_enable_dma_tracers();
+
      r = amdgpu_sync_init();
      if (r)
          goto error_sync;
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +10 -1
index e7e899190bef..a292e86fbaa7 100644
@@ -37,6 +37,9 @@
  #include "amdgpu.h"
  #include "amdgpu_trace.h"
+void amdgpu_trace_dma_map(struct ttm_tt *ttm);
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm);
+
  static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  {
      struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -625,6 +628,9 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void 
 **ptr)
      if (r)
          return r;
+    if (unlikely(trace_amdgpu_ttm_tt_populate_enabled()) && 
 bo->tbo.mem.mem_type != TTM_PL_VRAM)
+        amdgpu_trace_dma_map(bo->tbo.ttm);
+
      if (ptr)
          *ptr = amdgpu_bo_kptr(bo);
@@ -640,8 +646,11 @@ void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
  void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
  {
-    if (bo->kmap.bo)
+    if (bo->kmap.bo) {
+        if (unlikely(trace_amdgpu_ttm_tt_unpopulate_enabled()) && 
 bo->tbo.mem.mem_type != TTM_PL_VRAM)
+            amdgpu_trace_dma_unmap(bo->tbo.ttm);
          ttm_bo_kunmap(&bo->kmap);
+    }
  }
  struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
index 26665b4baf36..85727be3fbb7 100644
@@ -663,7 +663,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt 
 *ttm, struct page **pages)
      return r;
  }
-static void amdgpu_trace_dma_map(struct ttm_tt *ttm)
+void amdgpu_trace_dma_map(struct ttm_tt *ttm)
  {
      struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
      struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -679,7 +679,7 @@ static void amdgpu_trace_dma_map(struct ttm_tt 
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static void amdgpu_trace_dma_map(struct ttm_tt *ttm) · line 685

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +1 -1
 *ttm)
      }
  }
-static void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
+void amdgpu_trace_dma_unmap(struct ttm_tt *ttm)
  {
      struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
      struct amdgpu_ttm_tt *gtt = (void *)ttm;
Quoted 4 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 208e2082-cdea-d6dc-9ad8-569b3013df1e-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#8 lore raw
Quoted 38 lines · Christian König

Am 15.08.2017 um 13:33 schrieb Tom St Denis:

Quoted 37 lines · Christian König
Quoted 17 lines

Am 14.08.2017 um 13:37 schrieb Tom St Denis:

Quoted 6 lines

This patch adds trace points to the amdgpu bo_kmap/bo/kunmap functions which capture internal allocations.

(v2): Simply add a sleep to the init. Users can use this script to load with map debugging enabled:

Quoted 7 lines

#!/bin/bash
modprobe amdgpu map_debug=1 &
sleep 1
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_unpopulate/enable
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_populate/enable

I've just realized that there is a far simpler method than that or enabling trace points using the module command line.

Quoted 8 lines

Assuming your GPU is PCI device 01:00.0 connected through bridge 00:02.1 (use lspci -t to figure the connection out):

# Temporary remove the PCIe device from the bus
echo 1 > /sys/bus/pci/devices/0000\:01\:00.0/remove
# Load amdgpu, this allows you to enable the trace points you want and also sets probes etc.. modprobe amdpgu
# Rescan the bus, the device subsystem will automatically probe amdgpu with this device echo 1 > /sys/bus/pci/devices/0000\:00\:02.1/rescan

That would definitely be a bunch trickier to script up. Nobody will run this by hand. So we'd need to remove all devices that are AMD vendor ID but GPU only (e.g. not other controllers) which means we need to parse the lspci output for VGA controller related text.

Definitely doable and probably "cleaner" from a race point of view.

Quoted 1 line

Apart from that I would really like to see those trace points in TTM instead. I also don't mind adding a dev or pdev pointer to the ttm_bo_device structure for this.

Not against this in principle. There's a bit of confusion... our ttm amdgpu functions are passed

struct ttm_tt *ttm

Quoted 7 lines
Quoted 5 lines

which we cast to

struct amdgpu_ttm_tt {
    struct ttm_dma_tt    ttm;
    struct amdgpu_device    *adev;

BTW: Isn't this adev pointer here superflous? I mean the ttm_tt has a bdev pointer to use, don't they?

Honestly without cscoping it I wouldn't know :-)

Quoted 11 lines
Quoted 7 lines
    u64 offset;

... <snip>

Then we use the "->ttm" of that to get dma_address[] but pages[] from the original ttm_tt structure ... from what I recall (when I looked into this last week) ttm_dma_tt and ttm_tt aren't the same structure right? (and looking at it right now ttm_dma_tt begins with ttm_tt....).

So wherever I put the trace in ttm I need access to dma_address[] which doesn't seem so clear cut. Can I just cast a ttm_tt pointer to ttm_dma_tt?

Ah, ok now I understand your problem. Well, printing the dma addresses in this situation is probably not a good idea to start with.

When a buffer object is mapped to kernel space there is no guarantee that it already has it's DMA mapping. For amdgpu we probably never run into this situation, but for other drivers that might now be the case.

At the points I'm putting the traces a "map" has already happened per page like in ttm_populate and our kmap functions.

Quoted 1 line

Do we really need that info?

To reverse the IOMMU translation I need the bus/dma address, physical address, and PCI device information (because the dma <=> phys translation is only valid per PCI device).

Tom
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 0fa42ba8-38d2-0d1a-d914-eb92d98f2e8d-5C7GfCeVMHo@public.gmane.org

#10 lore raw
Quoted 42 lines · Tom St Denis
Quoted 41 lines · Christian König

Am 15.08.2017 um 13:33 schrieb Tom St Denis:

Quoted 40 lines · Christian König
Quoted 18 lines

Am 14.08.2017 um 13:37 schrieb Tom St Denis:

Quoted 6 lines

This patch adds trace points to the amdgpu bo_kmap/bo/kunmap functions which capture internal allocations.

(v2): Simply add a sleep to the init. Users can use this script to load with map debugging enabled:

Quoted 8 lines

#!/bin/bash
modprobe amdgpu map_debug=1 &
sleep 1
echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_unpopulate/enable

echo 1 >
/sys/kernel/debug/tracing/events/amdgpu/amdgpu_ttm_tt_populate/enable

I've just realized that there is a far simpler method than that or enabling trace points using the module command line.

Quoted 8 lines

Assuming your GPU is PCI device 01:00.0 connected through bridge 00:02.1 (use lspci -t to figure the connection out):

# Temporary remove the PCIe device from the bus
echo 1 > /sys/bus/pci/devices/0000\:01\:00.0/remove
# Load amdgpu, this allows you to enable the trace points you want and also sets probes etc.. modprobe amdpgu
# Rescan the bus, the device subsystem will automatically probe amdgpu with this device echo 1 > /sys/bus/pci/devices/0000\:00\:02.1/rescan

That would definitely be a bunch trickier to script up. Nobody will run this by hand. So we'd need to remove all devices that are AMD vendor ID but GPU only (e.g. not other controllers) which means we need to parse the lspci output for VGA controller related text.

Definitely doable and probably "cleaner" from a race point of view.

Quoted 1 line

Apart from that I would really like to see those trace points in TTM instead. I also don't mind adding a dev or pdev pointer to the ttm_bo_device structure for this.

Not against this in principle. There's a bit of confusion... our ttm amdgpu functions are passed

struct ttm_tt *ttm

which we cast to

Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
struct amdgpu_ttm_tt {
    struct ttm_dma_tt    ttm;
    struct amdgpu_device    *adev;

BTW: Isn't this adev pointer here superflous? I mean the ttm_tt has a bdev pointer to use, don't they?

Honestly without cscoping it I wouldn't know :-)

Yes, you can find the adev via

static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)

{
	return container_of(bdev, struct amdgpu_device, mman.bdev);
}

But looking at ttm_bo_device I don't see an obvious path to a device structure (or pci structure). So that would need to be added (as you previously commented).

Tom
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx