Skip to contentSkip to conversation

drm: Implement vm_operations_struct.access

PATCH In review Felix Kuehling 5 participants 13 messages Jul 13, 2017 – Jul 17, 2017 7 messages not in the archive

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 drm: Implement vm_operations_struct.access

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

---
 drivers/gpu/drm/ttm/ttm_bo_vm.c | 66 ++++++++++++++++++++++++++++++++++++++++-
 include/drm/ttm/ttm_bo_driver.h | 12 ++++++++
 2 files changed, 77 insertions(+), 1 deletion(-)
drivers/gpu/drm/ttm/ttm_bo_vm.c +65 -1
index 9f53df9..0ef2eb9 100644
@@ -294,10 +294,74 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
 	vma->vm_private_data = NULL;
 }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long first_page = offset >> PAGE_SHIFT;
+	unsigned long last_page = (offset + len - 1) >> PAGE_SHIFT;
+	unsigned long num_pages = last_page - first_page + 1;
+	struct ttm_bo_kmap_obj map;
+	void *ptr;
+	bool is_iomem;
+	int ret;
+
+	ret = ttm_bo_kmap(bo, first_page, num_pages, &map);
+	if (ret)
+		return ret;
+
+	offset -= first_page << PAGE_SHIFT;
+	ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset;
+	WARN_ON(is_iomem);
+	if (write)
+		memcpy(ptr, buf, len);
+	else
+		memcpy(buf, ptr, len);
+	ttm_bo_kunmap(&map);
+
+	return len;
+}
+
+static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
+			    void *buf, int len, int write)
+{
+	unsigned long offset = (addr) - vma->vm_start;
+	struct ttm_buffer_object *bo = vma->vm_private_data;
+	int ret;
+
+	if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages)
+		return -EIO;
+
+	ret = ttm_bo_reserve(bo, true, false, NULL);
+	if (ret)
+		return ret;
+
+	switch(bo->mem.mem_type) {
+	case TTM_PL_SYSTEM:
+	case TTM_PL_TT:
+		ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write);
+		break;
+	case TTM_PL_VRAM:
+		if (bo->bdev->driver->access_vram)
+			ret = bo->bdev->driver->access_vram(
+				bo, offset, buf, len, write);
+		else
+			ret = -EIO;
+		break;
+	default:
+		ret = -EIO;
+	}
+
+	ttm_bo_unreserve(bo);
+
+	return ret;
+}
+
 static const struct vm_operations_struct ttm_bo_vm_ops = {
 	.fault = ttm_bo_vm_fault,
 	.open = ttm_bo_vm_open,
-	.close = ttm_bo_vm_close
+	.close = ttm_bo_vm_close,
+	.access = ttm_bo_vm_access
 };
 
 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
include/drm/ttm/ttm_bo_driver.h +12 -0
index 6bbd34d..6ce5094 100644
@@ -471,6 +471,18 @@ struct ttm_bo_driver {
 	 */
 	unsigned long (*io_mem_pfn)(struct ttm_buffer_object *bo,
 				    unsigned long page_offset);
+
+	/**
+	 * Read/write VRAM buffers for ptrace access
+	 *
+	 * @bo: the BO to access
+	 * @offset: the offset from the start of the BO
+	 * @buf: pointer to source/destination buffer
+	 * @len: number of bytes to copy
+	 * @write: whether to read (0) from or write (non-0) to BO
+	 */
+	int (*access_vram)(struct ttm_buffer_object *bo, unsigned long offset,
+			   void *buf, int len, int write);
 };
 
 /**
Signature · 6 lines
1.9.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1499980105-7721-1-git-send-email-Felix.Kuehling-5C7GfCeVMHo@public.gmane.org

#2 lore raw

PATCH 2/2 drm/amdgpu: Implement ttm_bo_driver.access_vram callback

Allows gdb to access contents of user mode mapped VRAM BOs.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 59 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h |  2 ++
 2 files changed, 61 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +59 -0
index ff5614b..d65551d 100644
@@ -1115,6 +1115,64 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	return ttm_bo_eviction_valuable(bo, place);
 }
 
+static int amdgpu_ttm_access_vram(struct ttm_buffer_object *bo,
+				  unsigned long offset,
+				  void *buf, int len, int write)
+{
+	struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+	struct drm_mm_node *nodes = abo->tbo.mem.mm_node;
+	uint32_t value = 0;
+	int result = 0;
+	uint64_t pos;
+	unsigned long flags;
+
+	while (offset >= (nodes->size << PAGE_SHIFT)) {
+		offset -= nodes->size << PAGE_SHIFT;
+		++nodes;
+	}
+	pos = (nodes->start << PAGE_SHIFT) + offset;
+
+	while (len && pos < adev->mc.mc_vram_size) {
+		uint64_t aligned_pos = pos & ~(uint64_t)3;
+		uint32_t bytes = 4 - (pos & 3);
+		uint32_t shift = (pos & 3) * 8;
+		uint32_t mask = 0xffffffff << shift;
+
+		if (len < bytes) {
+			mask &= 0xffffffff >> (bytes - len) * 8;
+			bytes = len;
+		}
+
+		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+		WREG32(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
+		WREG32(mmMM_INDEX_HI, aligned_pos >> 31);
+		if (!write || mask != 0xffffffff)
+			value = RREG32(mmMM_DATA);
+		if (write) {
+			value &= ~mask;
+			value |= (*(uint32_t *)buf << shift) & mask;
+			WREG32(mmMM_DATA, value);
+		}
+		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+		if (!write) {
+			value = (value & mask) >> shift;
+			memcpy(buf, &value, bytes);
+		}
+
+		result += bytes;
+		buf = (uint8_t *)buf + bytes;
+		pos += bytes;
+		len -= bytes;
+		if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
+			++nodes;
+			pos = (nodes->start << PAGE_SHIFT);
+		}
+	}
+
+	return result;
+}
+
 static struct ttm_bo_driver amdgpu_bo_driver = {
 	.ttm_tt_create = &amdgpu_ttm_tt_create,
 	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
@@ -1130,6 +1188,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
 	.io_mem_free = &amdgpu_ttm_io_mem_free,
 	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
+	.access_vram = &amdgpu_ttm_access_vram
 };
 
 int amdgpu_ttm_init(struct amdgpu_device *adev)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -0
index f137c24..a22e430 100644
@@ -78,6 +78,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			struct dma_fence **fence);
 
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
+int amdgpu_bo_mmap(struct file *filp, struct vm_area_struct *vma,
+		   struct ttm_bo_device *bdev);
 bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
 int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
Signature · 6 lines
1.9.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 1499980105-7721-2-git-send-email-Felix.Kuehling-5C7GfCeVMHo@public.gmane.org

#4 lore raw

Re: [PATCH 2/2] drm/amdgpu: Implement ttm_bo_driver.access_vram callback

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h int amdgpu_fill_buffer(struct amdgpu_bo *bo, · line 85

Quoted 11 lines · Felix Kuehling
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -0
@@ -78,6 +78,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			struct dma_fence **fence);
 
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
+int amdgpu_bo_mmap(struct file *filp, struct vm_area_struct *vma,
+		   struct ttm_bo_device *bdev);
 bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
 int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);

Oops, this is a remnant from the old version that hacked the ttm_vm_ops in amdgpu. I'll remove this before I submit.
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Reviewed-by
#5 lore raw

Re: [PATCH 2/2] drm/amdgpu: Implement ttm_bo_driver.access_vram callback

Quoted 6 lines · Felix Kuehling
Quoted 5 lines · Felix Kuehling
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +0 -0
@@ -78,6 +78,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			struct dma_fence **fence);
 
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h int amdgpu_fill_buffer(struct amdgpu_bo *bo, · line 85

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -0
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
+int amdgpu_bo_mmap(struct file *filp, struct vm_area_struct *vma,
+		   struct ttm_bo_device *bdev);
 bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
 int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
 int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);

Oops, this is a remnant from the old version that hacked the ttm_vm_ops in amdgpu. I'll remove this before I submit.

With that fixed (and possibly excluding driver-private memory types if necessary, per discussion of patch 1),

Signature · 6 lines
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
#6 lore raw

Re: [PATCH 2/2] drm/amdgpu: Implement ttm_bo_driver.access_vram callback

In reply to Felix Kuehling

Am 13.07.2017 um 23:08 schrieb Felix Kuehling:

Quoted 23 lines

Allows gdb to access contents of user mode mapped VRAM BOs.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 59 +++++++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h |  2 ++
  2 files changed, 61 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +7 -0
index ff5614b..d65551d 100644
@@ -1115,6 +1115,64 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
  	return ttm_bo_eviction_valuable(bo, place);
  }
 
+static int amdgpu_ttm_access_vram(struct ttm_buffer_object *bo,
+				  unsigned long offset,
+				  void *buf, int len, int write)
+{
+	struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+	struct drm_mm_node *nodes = abo->tbo.mem.mm_node;

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, · line 1134

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +10 -0
+	uint32_t value = 0;
+	int result = 0;
+	uint64_t pos;
+	unsigned long flags;
+
+	while (offset >= (nodes->size << PAGE_SHIFT)) {
+		offset -= nodes->size << PAGE_SHIFT;
+		++nodes;
+	}
+	pos = (nodes->start << PAGE_SHIFT) + offset;

This silently assumes that a read would never cross a node boundary, doesn't it?

Christian.

Quoted 57 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +42 -0
+
+	while (len && pos < adev->mc.mc_vram_size) {
+		uint64_t aligned_pos = pos & ~(uint64_t)3;
+		uint32_t bytes = 4 - (pos & 3);
+		uint32_t shift = (pos & 3) * 8;
+		uint32_t mask = 0xffffffff << shift;
+
+		if (len < bytes) {
+			mask &= 0xffffffff >> (bytes - len) * 8;
+			bytes = len;
+		}
+
+		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+		WREG32(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
+		WREG32(mmMM_INDEX_HI, aligned_pos >> 31);
+		if (!write || mask != 0xffffffff)
+			value = RREG32(mmMM_DATA);
+		if (write) {
+			value &= ~mask;
+			value |= (*(uint32_t *)buf << shift) & mask;
+			WREG32(mmMM_DATA, value);
+		}
+		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+		if (!write) {
+			value = (value & mask) >> shift;
+			memcpy(buf, &value, bytes);
+		}
+
+		result += bytes;
+		buf = (uint8_t *)buf + bytes;
+		pos += bytes;
+		len -= bytes;
+		if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
+			++nodes;
+			pos = (nodes->start << PAGE_SHIFT);
+		}
+	}
+
+	return result;
+}
+
  static struct ttm_bo_driver amdgpu_bo_driver = {
  	.ttm_tt_create = &amdgpu_ttm_tt_create,
  	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
@@ -1130,6 +1188,7 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
  	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
  	.io_mem_free = &amdgpu_ttm_io_mem_free,
  	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
+	.access_vram = &amdgpu_ttm_access_vram
  };
 
  int amdgpu_ttm_init(struct amdgpu_device *adev)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +0 -0
index f137c24..a22e430 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h int amdgpu_fill_buffer(struct amdgpu_bo *bo, · line 85

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -0
@@ -78,6 +78,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
  			struct dma_fence **fence);
 
  int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
+int amdgpu_bo_mmap(struct file *filp, struct vm_area_struct *vma,
+		   struct ttm_bo_device *bdev);
  bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
  int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
  int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

A message that is not in the archive (sent off-list or not yet received): c25a37a9-8fae-11f0-cce6-59ca13412801-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#8 lore raw

Re: [PATCH 2/2] drm/amdgpu: Implement ttm_bo_driver.access_vram callback

Quoted 31 lines · Christian König

Am 13.07.2017 um 23:08 schrieb Felix Kuehling:

Quoted 30 lines

Allows gdb to access contents of user mode mapped VRAM BOs.

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 59
+++++++++++++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h |  2 ++
  2 files changed, 61 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +11 -0
index ff5614b..d65551d 100644
@@ -1115,6 +1115,64 @@ static bool
 amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
      return ttm_bo_eviction_valuable(bo, place);
  }
  +static int amdgpu_ttm_access_vram(struct ttm_buffer_object *bo,
+                  unsigned long offset,
+                  void *buf, int len, int write)
+{
+    struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
+    struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
+    struct drm_mm_node *nodes = abo->tbo.mem.mm_node;
+    uint32_t value = 0;
+    int result = 0;
+    uint64_t pos;
+    unsigned long flags;
+
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, · line 1134

Quoted 5 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +5 -0
+    while (offset >= (nodes->size << PAGE_SHIFT)) {
+        offset -= nodes->size << PAGE_SHIFT;
+        ++nodes;
+    }
+    pos = (nodes->start << PAGE_SHIFT) + offset;

This silently assumes that a read would never cross a node boundary, doesn't it?

It doesn't. See below ...

Quoted 30 lines

Christian.

Quoted 27 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +27 -0
+
+    while (len && pos < adev->mc.mc_vram_size) {
+        uint64_t aligned_pos = pos & ~(uint64_t)3;
+        uint32_t bytes = 4 - (pos & 3);
+        uint32_t shift = (pos & 3) * 8;
+        uint32_t mask = 0xffffffff << shift;
+
+        if (len < bytes) {
+            mask &= 0xffffffff >> (bytes - len) * 8;
+            bytes = len;
+        }
+
+        spin_lock_irqsave(&adev->mmio_idx_lock, flags);
+        WREG32(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
+        WREG32(mmMM_INDEX_HI, aligned_pos >> 31);
+        if (!write || mask != 0xffffffff)
+            value = RREG32(mmMM_DATA);
+        if (write) {
+            value &= ~mask;
+            value |= (*(uint32_t *)buf << shift) & mask;
+            WREG32(mmMM_DATA, value);
+        }
+        spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
+        if (!write) {
+            value = (value & mask) >> shift;
+            memcpy(buf, &value, bytes);
+        }
Quoted 8 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, · line 1169

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +8 -0
+
+        result += bytes;
+        buf = (uint8_t *)buf + bytes;
+        pos += bytes;
+        len -= bytes;
+        if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
+            ++nodes;
+            pos = (nodes->start << PAGE_SHIFT);

... Here I handle crossing a node boundary. Yes, I actually added this case to my kfdtest unit test and made sure it works, along with all odd alignments that the code above handles.

Regards,
Felix

Quoted 23 lines
Quoted 23 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +7 -0
+        }
+    }
+
+    return result;
+}
+
  static struct ttm_bo_driver amdgpu_bo_driver = {
      .ttm_tt_create = &amdgpu_ttm_tt_create,
      .ttm_tt_populate = &amdgpu_ttm_tt_populate,
@@ -1130,6 +1188,7 @@ static bool
 amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
      .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
      .io_mem_free = &amdgpu_ttm_io_mem_free,
      .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
+    .access_vram = &amdgpu_ttm_access_vram
  };
    int amdgpu_ttm_init(struct amdgpu_device *adev)
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +0 -0
index f137c24..a22e430 100644
@@ -78,6 +78,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
Quoted 10 lines

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h int amdgpu_fill_buffer(struct amdgpu_bo *bo, · line 85

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +2 -0
              struct dma_fence **fence);
    int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
+int amdgpu_bo_mmap(struct file *filp, struct vm_area_struct *vma,
+           struct ttm_bo_device *bdev);
  bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
  int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct
 ttm_mem_reg *bo_mem);
  int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#9 lore raw

Re: [PATCH 2/2] drm/amdgpu: Implement ttm_bo_driver.access_vram callback

Am 14.07.2017 um 21:44 schrieb Felix Kuehling:

Quoted 10 lines · Christian König
Quoted 9 lines

Am 13.07.2017 um 23:08 schrieb Felix Kuehling:
[SNIP]

Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +7 -0
+        result += bytes;
+        buf = (uint8_t *)buf + bytes;
+        pos += bytes;
+        len -= bytes;
+        if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
+            ++nodes;
+            pos = (nodes->start << PAGE_SHIFT);
Quoted 3 lines

... Here I handle crossing a node boundary. Yes, I actually added this case to my kfdtest unit test and made sure it works, along with all odd alignments that the code above handles.

Ah, I see. Sorry totally missed that chunk. In this case the patch is Acked-by: Christian König <[email protected]>

Regards,
Christian.

Quoted 3 lines

Regards,
Felix

_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel

#10 lore raw
In reply to a message not in the archive
Quoted 2 lines · Felix Kuehling

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

Thanks for doing this. Looks mostly good, but I still have some comments.

The shortlog prefix should be "drm/ttm:".

Quoted 13 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +5 -0
index 9f53df9..0ef2eb9 100644
@@ -294,10 +294,74 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
 	vma->vm_private_data = NULL;
 }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long first_page = offset >> PAGE_SHIFT;

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 311

Quoted 10 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +10 -0
+	unsigned long last_page = (offset + len - 1) >> PAGE_SHIFT;
+	unsigned long num_pages = last_page - first_page + 1;
+	struct ttm_bo_kmap_obj map;
+	void *ptr;
+	bool is_iomem;
+	int ret;
+
+	ret = ttm_bo_kmap(bo, first_page, num_pages, &map);
+	if (ret)
+		return ret;

Could there be cases (e.g. on 32-bit) where mapping all pages at once fails, but mapping one page at a time would work?

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 315

Quoted 3 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +3 -0
+	offset -= first_page << PAGE_SHIFT;
+	ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset;
+	WARN_ON(is_iomem);

I suggest making this WARN_ON_ONCE, to avoid flooding dmesg if it ever triggers in practice.

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 476

Quoted 12 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +0 -0
 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
include/drm/ttm/ttm_bo_driver.h +3 -0
index 6bbd34d..6ce5094 100644
@@ -471,6 +471,18 @@ struct ttm_bo_driver {
 	 */
 	unsigned long (*io_mem_pfn)(struct ttm_buffer_object *bo,
 				    unsigned long page_offset);
+
+	/**
+	 * Read/write VRAM buffers for ptrace access

Is there any reason for making this specific to VRAM? ttm_bo_vm_access could just call this for anything except TTM_PL_SYSTEM / TTM_PL_TT, and let the driver return an error if it can't handle the memory type.

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 483

Quoted 6 lines
include/drm/ttm/ttm_bo_driver.h +6 -0
+	 * @bo: the BO to access
+	 * @offset: the offset from the start of the BO
+	 * @buf: pointer to source/destination buffer
+	 * @len: number of bytes to copy
+	 * @write: whether to read (0) from or write (non-0) to BO
+	 */

The meaning of the return value should also be documented here.

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 486

Quoted 3 lines
include/drm/ttm/ttm_bo_driver.h +2 -0
+	int (*access_vram)(struct ttm_buffer_object *bo, unsigned long offset,
+			   void *buf, int len, int write);
 };

I suggest making the write parameter a bool.

Signature · 6 lines
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): b377c150-e843-29ff-1be2-0e82f200abd6-otUistvHUpPR7s880joybQ@public.gmane.org

#12 lore raw
Quoted 25 lines · Michel Dänzer
Quoted 2 lines · Felix Kuehling

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

Thanks for doing this. Looks mostly good, but I still have some comments.

The shortlog prefix should be "drm/ttm:".

Quoted 17 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +9 -0
index 9f53df9..0ef2eb9 100644
@@ -294,10 +294,74 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
 	vma->vm_private_data = NULL;
 }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long first_page = offset >> PAGE_SHIFT;
+	unsigned long last_page = (offset + len - 1) >> PAGE_SHIFT;
+	unsigned long num_pages = last_page - first_page + 1;
+	struct ttm_bo_kmap_obj map;
+	void *ptr;
Quoted 8 lines

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 311

Quoted 6 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +6 -0
+	bool is_iomem;
+	int ret;
+
+	ret = ttm_bo_kmap(bo, first_page, num_pages, &map);
+	if (ret)
+		return ret;

Could there be cases (e.g. on 32-bit) where mapping all pages at once fails, but mapping one page at a time would work?

Maybe. I'm not really familiar with ttm_bo_kmap limitations on 32-bit. I guess the the access would have to be really big. I think in practice GDB doesn't do very big accesses. So I'm not sure it's worth the trouble.

Quoted 9 lines

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 315

Quoted 5 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +3 -0
+	offset -= first_page << PAGE_SHIFT;
+	ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset;
+	WARN_ON(is_iomem);

I suggest making this WARN_ON_ONCE, to avoid flooding dmesg if it ever triggers in practice.

Quoted 15 lines

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 476

Quoted 12 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +0 -0
 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
include/drm/ttm/ttm_bo_driver.h +3 -0
index 6bbd34d..6ce5094 100644
@@ -471,6 +471,18 @@ struct ttm_bo_driver {
 	 */
 	unsigned long (*io_mem_pfn)(struct ttm_buffer_object *bo,
 				    unsigned long page_offset);
+
+	/**
+	 * Read/write VRAM buffers for ptrace access

Is there any reason for making this specific to VRAM? ttm_bo_vm_access could just call this for anything except TTM_PL_SYSTEM / TTM_PL_TT, and let the driver return an error if it can't handle the memory type.

Good point. I'll change that.

Quoted 11 lines

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 483

Quoted 8 lines
include/drm/ttm/ttm_bo_driver.h +6 -0
+	 * @bo: the BO to access
+	 * @offset: the offset from the start of the BO
+	 * @buf: pointer to source/destination buffer
+	 * @len: number of bytes to copy
+	 * @write: whether to read (0) from or write (non-0) to BO
+	 */

The meaning of the return value should also be documented here.

Quoted 4 lines

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 486

Quoted 3 lines
include/drm/ttm/ttm_bo_driver.h +2 -0
+	int (*access_vram)(struct ttm_buffer_object *bo, unsigned long offset,
+			   void *buf, int len, int write);
 };

I suggest making the write parameter a bool.

I made this as similar as possible to the vm_ops->access API, where write is also an integer.

Regards,
Felix

Quoted 2 lines

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): a6bb7cda-5090-c88d-3309-003d601f1c46-5C7GfCeVMHo@public.gmane.org

#14 lore raw
Quoted 35 lines · Felix Kuehling
Quoted 25 lines · Michel Dänzer
Quoted 2 lines · Felix Kuehling

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

Thanks for doing this. Looks mostly good, but I still have some comments.

The shortlog prefix should be "drm/ttm:".

Quoted 17 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +9 -0
index 9f53df9..0ef2eb9 100644
@@ -294,10 +294,74 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
 	vma->vm_private_data = NULL;
 }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long first_page = offset >> PAGE_SHIFT;
+	unsigned long last_page = (offset + len - 1) >> PAGE_SHIFT;
+	unsigned long num_pages = last_page - first_page + 1;
+	struct ttm_bo_kmap_obj map;
+	void *ptr;
Quoted 8 lines

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 311

Quoted 6 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +6 -0
+	bool is_iomem;
+	int ret;
+
+	ret = ttm_bo_kmap(bo, first_page, num_pages, &map);
+	if (ret)
+		return ret;

Could there be cases (e.g. on 32-bit) where mapping all pages at once fails, but mapping one page at a time would work?

Quoted 3 lines

Maybe. I'm not really familiar with ttm_bo_kmap limitations on 32-bit. I guess the the access would have to be really big. I think in practice GDB doesn't do very big accesses. So I'm not sure it's worth the trouble.

Okay, I guess it can always be changed later if necessary.

Quoted 7 lines
Quoted 4 lines

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 314

Quoted 3 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +2 -0
+	int (*access_vram)(struct ttm_buffer_object *bo, unsigned long offset,
+			   void *buf, int len, int write);
 };

I suggest making the write parameter a bool.

I made this as similar as possible to the vm_ops->access API, where write is also an integer.

I see, makes sense.

Signature · 6 lines
Earthling Michel Dänzer               |               http://www.amd.com
Libre software enthusiast             |             Mesa and X developer
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 48dc79df-f846-925c-ddf9-52852578201b-otUistvHUpPR7s880joybQ@public.gmane.org

#16 lore raw

Am 15.07.2017 um 05:32 schrieb Michel Dänzer:

Quoted 35 lines · Felix Kuehling
Quoted 34 lines · Michel Dänzer
Quoted 33 lines · Felix Kuehling
Quoted 2 lines

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

Thanks for doing this. Looks mostly good, but I still have some comments.

The shortlog prefix should be "drm/ttm:".

Quoted 13 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +5 -0
index 9f53df9..0ef2eb9 100644
@@ -294,10 +294,74 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
  	vma->vm_private_data = NULL;
  }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long first_page = offset >> PAGE_SHIFT;

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 311

Quoted 10 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +10 -0
+	unsigned long last_page = (offset + len - 1) >> PAGE_SHIFT;
+	unsigned long num_pages = last_page - first_page + 1;
+	struct ttm_bo_kmap_obj map;
+	void *ptr;
+	bool is_iomem;
+	int ret;
+
+	ret = ttm_bo_kmap(bo, first_page, num_pages, &map);
+	if (ret)
+		return ret;

Could there be cases (e.g. on 32-bit) where mapping all pages at once fails, but mapping one page at a time would work?

Quoted 4 lines
Quoted 3 lines

Maybe. I'm not really familiar with ttm_bo_kmap limitations on 32-bit. I guess the the access would have to be really big. I think in practice GDB doesn't do very big accesses. So I'm not sure it's worth the trouble.

Okay, I guess it can always be changed later if necessary.

It would still be better to do this page by page.

See the issue is that when you only map one page ttm_bo_kmap() is clever and returns a pointer to only that page.

But when you map at least two pages ttm_bo_kmap() needs to allocate virtual address space, map the pages and flush the TLBs (which is a very heavy operation) just to make those two pages look continuously to the CPU.

Not that I expect that this function is performance critical, but that is complexity I would try to avoid.

Quoted 9 lines
Quoted 8 lines
Quoted 6 lines

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 314

Quoted 5 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +2 -0
+	int (*access_vram)(struct ttm_buffer_object *bo, unsigned long offset,
+			   void *buf, int len, int write);
  };

I suggest making the write parameter a bool.

I made this as similar as possible to the vm_ops->access API, where write is also an integer.

I see, makes sense.

Yeah, agree as well. Please keep the style of the upstream interface here.

Christian.

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#17 lore raw
In reply to a message not in the archive

Am 13.07.2017 um 23:08 schrieb Felix Kuehling:

Quoted 2 lines

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

One more comment additionally to what Michel already wrote below, apart from that it looks good to me.

Quoted 49 lines
---
  drivers/gpu/drm/ttm/ttm_bo_vm.c | 66 ++++++++++++++++++++++++++++++++++++++++-
  include/drm/ttm/ttm_bo_driver.h | 12 ++++++++
  2 files changed, 77 insertions(+), 1 deletion(-)
drivers/gpu/drm/ttm/ttm_bo_vm.c +34 -0
index 9f53df9..0ef2eb9 100644
@@ -294,10 +294,74 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
  	vma->vm_private_data = NULL;
  }
 
+static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
+				 unsigned long offset,
+				 void *buf, int len, int write)
+{
+	unsigned long first_page = offset >> PAGE_SHIFT;
+	unsigned long last_page = (offset + len - 1) >> PAGE_SHIFT;
+	unsigned long num_pages = last_page - first_page + 1;
+	struct ttm_bo_kmap_obj map;
+	void *ptr;
+	bool is_iomem;
+	int ret;
+
+	ret = ttm_bo_kmap(bo, first_page, num_pages, &map);
+	if (ret)
+		return ret;
+
+	offset -= first_page << PAGE_SHIFT;
+	ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset;
+	WARN_ON(is_iomem);
+	if (write)
+		memcpy(ptr, buf, len);
+	else
+		memcpy(buf, ptr, len);
+	ttm_bo_kunmap(&map);
+
+	return len;
+}
+
+static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
+			    void *buf, int len, int write)
+{
+	unsigned long offset = (addr) - vma->vm_start;
+	struct ttm_buffer_object *bo = vma->vm_private_data;
+	int ret;

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 340

Quoted 10 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +10 -0
+
+	if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages)
+		return -EIO;
+
+	ret = ttm_bo_reserve(bo, true, false, NULL);
+	if (ret)
+		return ret;
+
+	switch(bo->mem.mem_type) {
+	case TTM_PL_SYSTEM:

When the BO is in the system domain you need to add this as well I think:

        if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
                ret = ttm_tt_swapin(bo->ttm);
                if (unlikely(ret != 0))
                        return ret;
        }

Regards,
Christian.

Quoted 41 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +21 -1
+	case TTM_PL_TT:
+		ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write);
+		break;
+	case TTM_PL_VRAM:
+		if (bo->bdev->driver->access_vram)
+			ret = bo->bdev->driver->access_vram(
+				bo, offset, buf, len, write);
+		else
+			ret = -EIO;
+		break;
+	default:
+		ret = -EIO;
+	}
+
+	ttm_bo_unreserve(bo);
+
+	return ret;
+}
+
  static const struct vm_operations_struct ttm_bo_vm_ops = {
  	.fault = ttm_bo_vm_fault,
  	.open = ttm_bo_vm_open,
-	.close = ttm_bo_vm_close
+	.close = ttm_bo_vm_close,
+	.access = ttm_bo_vm_access
  };
 
  static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
include/drm/ttm/ttm_bo_driver.h +5 -0
index 6bbd34d..6ce5094 100644
@@ -471,6 +471,18 @@ struct ttm_bo_driver {
  	 */
  	unsigned long (*io_mem_pfn)(struct ttm_buffer_object *bo,
  				    unsigned long page_offset);
+
+	/**
+	 * Read/write VRAM buffers for ptrace access
+	 *
+	 * @bo: the BO to access

include/drm/ttm/ttm_bo_driver.h struct ttm_bo_driver { · line 488

Quoted 10 lines
include/drm/ttm/ttm_bo_driver.h +7 -0
+	 * @offset: the offset from the start of the BO
+	 * @buf: pointer to source/destination buffer
+	 * @len: number of bytes to copy
+	 * @write: whether to read (0) from or write (non-0) to BO
+	 */
+	int (*access_vram)(struct ttm_buffer_object *bo, unsigned long offset,
+			   void *buf, int len, int write);
  };
 
  /**

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

A message that is not in the archive (sent off-list or not yet received): 8b708b8e-cf4f-d553-811f-a7849fbf6eff-ANTagKRnAhcb1SvskN2V4Q@public.gmane.org

#19 lore raw
Quoted 11 lines · Christian König

Am 13.07.2017 um 23:08 schrieb Felix Kuehling:

Quoted 2 lines

Allows gdb to access contents of user mode mapped BOs. VRAM access requires the driver to implement a new callback in ttm_bo_driver.

One more comment additionally to what Michel already wrote below, apart from that it looks good to me.

drivers/gpu/drm/ttm/ttm_bo_vm.c static void ttm_bo_vm_close(struct vm_area_struct *vma) · line 340

Quoted 3 lines
drivers/gpu/drm/ttm/ttm_bo_vm.c +2 -0
+    switch(bo->mem.mem_type) {
+    case TTM_PL_SYSTEM:
Quoted 7 lines

When the BO is in the system domain you need to add this as well I think:

        if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
                ret = ttm_tt_swapin(bo->ttm);
                if (unlikely(ret != 0))
                        return ret;
        }

OK, thanks for pointing that out.

Regards,
Felix

Quoted 3 lines

Regards,
Christian.

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx