Skip to contentSkip to conversation

drm/amdkfd: use iosys_map for CWSR buffer access

PATCH In review James Zhu 5 participants 5 messages Jun 29

Subsystems (3)
lore
Share & tools
lore

James Zhu

author
#0 lore raw

PATCH 1/1 drm/amdkfd: use iosys_map for CWSR buffer access

After moving TBA/TMA from GTT to VRAM for GFX9.4.2+ in commit 5088a1ba6d6d, direct pointer dereferences to CWSR buffers became unsafe because VRAM is accessed via MMIO (PCI BAR mappings).

Direct writes like 'tma[2] = enabled' and memcpy() can fail or produce incorrect results on non-x86 architectures because:
- MMIO requires specific accessor functions (writeq/readq)
- Compiler optimizations may generate invalid instruction sequences
- No guarantee of proper memory barriers or atomic access

This patch converts CWSR buffer access to use struct iosys_map, which automatically handles both system memory (GTT) and MMIO (VRAM) correctly by:
- Using writeq/writel/memcpy_toio for MMIO regions
- Using WRITE_ONCE/memcpy for system memory
- Providing proper memory barriers and access guarantees

Changes:
- Replace void *cwsr_kaddr with struct iosys_map cwsr_map
- Detect MMIO vs system memory using TTM_BO_MAP_IOMEM_MASK
- Use iosys_map_wr() for writing trap handler addresses and flags
- Use iosys_map_memcpy_to() for copying CWSR ISA code

This ensures correct operation on all architectures while maintaining backward compatibility with older GPUs and APUs that use GTT.

---
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  3 +-
 drivers/gpu/drm/amd/amdkfd/kfd_process.c | 63 +++++++++++++++++-------
 2 files changed, 47 insertions(+), 19 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -1
index ad4897f094a2..6e559aab4009 100644
@@ -32,6 +32,7 @@
 #include <linux/atomic.h>
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
 #include <uapi/linux/kfd_ioctl.h>
 #include <linux/idr.h>
 #include <linux/kfifo.h>
@@ -710,7 +711,7 @@ struct qcm_process_device {
 
 	/* CWSR memory */
 	struct kgd_mem *cwsr_mem;
-	void *cwsr_kaddr;
+	struct iosys_map cwsr_map;
 	uint64_t cwsr_base;
 	uint64_t tba_addr;
 	uint64_t tma_addr;
drivers/gpu/drm/amd/amdkfd/kfd_process.c +45 -18

123 lines not on this page · raw message

Signature · 1 line
2.43.0
maintainer
#1 lore raw
Quoted 23 lines · James Zhu

After moving TBA/TMA from GTT to VRAM for GFX9.4.2+ in commit 5088a1ba6d6d, direct pointer dereferences to CWSR buffers became unsafe because VRAM is accessed via MMIO (PCI BAR mappings).

Direct writes like 'tma[2] = enabled' and memcpy() can fail or produce incorrect results on non-x86 architectures because:
- MMIO requires specific accessor functions (writeq/readq)
- Compiler optimizations may generate invalid instruction sequences
- No guarantee of proper memory barriers or atomic access

This patch converts CWSR buffer access to use struct iosys_map, which automatically handles both system memory (GTT) and MMIO (VRAM) correctly by:
- Using writeq/writel/memcpy_toio for MMIO regions
- Using WRITE_ONCE/memcpy for system memory
- Providing proper memory barriers and access guarantees

Changes:
- Replace void *cwsr_kaddr with struct iosys_map cwsr_map
- Detect MMIO vs system memory using TTM_BO_MAP_IOMEM_MASK
- Use iosys_map_wr() for writing trap handler addresses and flags
- Use iosys_map_memcpy_to() for copying CWSR ISA code

Quoted 6 lines

This ensures correct operation on all architectures while maintaining backward compatibility with older GPUs and APUs that use GTT.

Looks valid of hand, but I have no time for an in deep review. Somebody who knows the code should take another look as well.

But feel free to add Acked-by: Christian König <[email protected]>.

Regards,
Christian.

Quoted 160 lines
---
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  3 +-
 drivers/gpu/drm/amd/amdkfd/kfd_process.c | 63 +++++++++++++++++-------
 2 files changed, 47 insertions(+), 19 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -1
index ad4897f094a2..6e559aab4009 100644
@@ -32,6 +32,7 @@
 #include <linux/atomic.h>
 #include <linux/workqueue.h>
 #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
 #include <uapi/linux/kfd_ioctl.h>
 #include <linux/idr.h>
 #include <linux/kfifo.h>
@@ -710,7 +711,7 @@ struct qcm_process_device {
 
 	/* CWSR memory */
 	struct kgd_mem *cwsr_mem;
-	void *cwsr_kaddr;
+	struct iosys_map cwsr_map;
 	uint64_t cwsr_base;
 	uint64_t tba_addr;
 	uint64_t tma_addr;
drivers/gpu/drm/amd/amdkfd/kfd_process.c +45 -18
index 8e701dcda8ec..7fd65c31afa2 100644
@@ -33,6 +33,7 @@
 #include <linux/mman.h>
 #include <linux/file.h>
 #include <linux/pm_runtime.h>
+#include <drm/ttm/ttm_bo.h>
 #include "amdgpu_amdkfd.h"
 #include "amdgpu.h"
 #include "amdgpu_reset.h"
@@ -745,6 +746,21 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
 					       NULL);
 }
 
+static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
+			struct kfd_process_device *pdd, struct iosys_map *map)
+{
+	struct kfd_node *dev = pdd->dev;
+
+	if (map && !iosys_map_is_null(map)) {
+		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
+		iosys_map_clear(map);
+	}
+
+	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
+	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
+					       NULL);
+}
+
 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
  *	This function should be only called right after the process
  *	is created and when kfd_processes_mutex is still being held
@@ -1192,8 +1208,8 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
 		if (pdd->drm_file)
 			fput(pdd->drm_file);
 
-		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
-			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
+		if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
+			free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
 				get_order(KFD_CWSR_TBA_TMA_SIZE));
 
 		idr_destroy(&pdd->alloc_idr);
@@ -1501,7 +1517,7 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
 	void *kaddr;
 	int ret;
 
-	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
 		return 0;
 
 	if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
@@ -1516,17 +1532,28 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
 		return ret;
 
 	qpd->cwsr_mem = mem;
-	qpd->cwsr_kaddr = kaddr;
+
+	/* Set up iosys_map based on whether memory is MMIO or system memory */
+	if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+		iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
+	else
+		iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
+
 	qpd->tba_addr = qpd->cwsr_base;
 
-	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
+	/* Copy CWSR ISA to buffer using appropriate accessor */
+	iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
+			    dev->kfd->cwsr_isa_size);
 
 	kfd_process_set_trap_debug_flag(&pdd->qpd,
 					pdd->process->debug_trap_enabled);
 
 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
-		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
+		 qpd->tba_addr, qpd->tma_addr,
+		 qpd->cwsr_map.is_iomem ? "iomem" : "system",
+		 qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
+					  qpd->cwsr_map.vaddr);
 
 	return 0;
 }
@@ -1536,24 +1563,24 @@ static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
 	struct kfd_node *dev = pdd->dev;
 	struct qcm_process_device *qpd = &pdd->qpd;
 
-	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
 		return;
 
-	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
+	kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
 }
 
 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
 				  uint64_t tba_addr,
 				  uint64_t tma_addr)
 {
-	if (qpd->cwsr_kaddr) {
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
 		/* KFD trap handler is bound, record as second-level TBA/TMA
 		 * in first-level TMA. First-level trap will jump to second.
 		 */
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[0] = tba_addr;
-		tma[1] = tma_addr;
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
+			     uint64_t, tba_addr);
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
+			     uint64_t, tma_addr);
 	} else {
 		/* No trap handler bound, bind as first-level TBA/TMA. */
 		qpd->tba_addr = tba_addr;
@@ -1619,10 +1646,10 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
 				     bool enabled)
 {
-	if (qpd->cwsr_kaddr) {
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[2] = enabled;
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
+		iosys_map_wr(&qpd->cwsr_map,
+			     KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
+			     uint64_t, enabled);
 	}
 }
 
maintainer Reviewed-by on v1 1/1
#2 lore raw
In reply to James Zhu
Quoted 23 lines · James Zhu

After moving TBA/TMA from GTT to VRAM for GFX9.4.2+ in commit 5088a1ba6d6d, direct pointer dereferences to CWSR buffers became unsafe because VRAM is accessed via MMIO (PCI BAR mappings).

Direct writes like 'tma[2] = enabled' and memcpy() can fail or produce incorrect results on non-x86 architectures because:
- MMIO requires specific accessor functions (writeq/readq)
- Compiler optimizations may generate invalid instruction sequences
- No guarantee of proper memory barriers or atomic access

This patch converts CWSR buffer access to use struct iosys_map, which automatically handles both system memory (GTT) and MMIO (VRAM) correctly by:
- Using writeq/writel/memcpy_toio for MMIO regions
- Using WRITE_ONCE/memcpy for system memory
- Providing proper memory barriers and access guarantees

Changes:
- Replace void *cwsr_kaddr with struct iosys_map cwsr_map
- Detect MMIO vs system memory using TTM_BO_MAP_IOMEM_MASK
- Use iosys_map_wr() for writing trap handler addresses and flags
- Use iosys_map_memcpy_to() for copying CWSR ISA code

Quoted 6 lines

This ensures correct operation on all architectures while maintaining backward compatibility with older GPUs and APUs that use GTT.

Quoted 160 lines
---
  drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  3 +-
  drivers/gpu/drm/amd/amdkfd/kfd_process.c | 63 +++++++++++++++++-------
  2 files changed, 47 insertions(+), 19 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -1
index ad4897f094a2..6e559aab4009 100644
@@ -32,6 +32,7 @@
  #include <linux/atomic.h>
  #include <linux/workqueue.h>
  #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
  #include <uapi/linux/kfd_ioctl.h>
  #include <linux/idr.h>
  #include <linux/kfifo.h>
@@ -710,7 +711,7 @@ struct qcm_process_device {
 
  	/* CWSR memory */
  	struct kgd_mem *cwsr_mem;
-	void *cwsr_kaddr;
+	struct iosys_map cwsr_map;
  	uint64_t cwsr_base;
  	uint64_t tba_addr;
  	uint64_t tma_addr;
drivers/gpu/drm/amd/amdkfd/kfd_process.c +45 -18
index 8e701dcda8ec..7fd65c31afa2 100644
@@ -33,6 +33,7 @@
  #include <linux/mman.h>
  #include <linux/file.h>
  #include <linux/pm_runtime.h>
+#include <drm/ttm/ttm_bo.h>
  #include "amdgpu_amdkfd.h"
  #include "amdgpu.h"
  #include "amdgpu_reset.h"
@@ -745,6 +746,21 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
  					       NULL);
  }
 
+static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
+			struct kfd_process_device *pdd, struct iosys_map *map)
+{
+	struct kfd_node *dev = pdd->dev;
+
+	if (map && !iosys_map_is_null(map)) {
+		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
+		iosys_map_clear(map);
+	}
+
+	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
+	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
+					       NULL);
+}
+
  /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
   *	This function should be only called right after the process
   *	is created and when kfd_processes_mutex is still being held
@@ -1192,8 +1208,8 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
  		if (pdd->drm_file)
  			fput(pdd->drm_file);
 
-		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
-			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
+		if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
+			free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
  				get_order(KFD_CWSR_TBA_TMA_SIZE));
 
  		idr_destroy(&pdd->alloc_idr);
@@ -1501,7 +1517,7 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
  	void *kaddr;
  	int ret;
 
-	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
  		return 0;
 
  	if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
@@ -1516,17 +1532,28 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
  		return ret;
 
  	qpd->cwsr_mem = mem;
-	qpd->cwsr_kaddr = kaddr;
+
+	/* Set up iosys_map based on whether memory is MMIO or system memory */
+	if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+		iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
+	else
+		iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
+
  	qpd->tba_addr = qpd->cwsr_base;
 
-	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
+	/* Copy CWSR ISA to buffer using appropriate accessor */
+	iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
+			    dev->kfd->cwsr_isa_size);
 
  	kfd_process_set_trap_debug_flag(&pdd->qpd,
  					pdd->process->debug_trap_enabled);
 
  	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
-		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
+		 qpd->tba_addr, qpd->tma_addr,
+		 qpd->cwsr_map.is_iomem ? "iomem" : "system",
+		 qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
+					  qpd->cwsr_map.vaddr);
 
  	return 0;
  }
@@ -1536,24 +1563,24 @@ static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
  	struct kfd_node *dev = pdd->dev;
  	struct qcm_process_device *qpd = &pdd->qpd;
 
-	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
  		return;
 
-	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
+	kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
  }
 
  void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
  				  uint64_t tba_addr,
  				  uint64_t tma_addr)
  {
-	if (qpd->cwsr_kaddr) {
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
  		/* KFD trap handler is bound, record as second-level TBA/TMA
  		 * in first-level TMA. First-level trap will jump to second.
  		 */
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[0] = tba_addr;
-		tma[1] = tma_addr;
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
+			     uint64_t, tba_addr);
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
+			     uint64_t, tma_addr);
  	} else {
  		/* No trap handler bound, bind as first-level TBA/TMA. */
  		qpd->tba_addr = tba_addr;
@@ -1619,10 +1646,10 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
  void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
  				     bool enabled)
  {
-	if (qpd->cwsr_kaddr) {
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[2] = enabled;
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
+		iosys_map_wr(&qpd->cwsr_map,
+			     KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
+			     uint64_t, enabled);
  	}
  }
 
#3 lore raw
In reply to James Zhu
Quoted 23 lines · James Zhu

After moving TBA/TMA from GTT to VRAM for GFX9.4.2+ in commit 5088a1ba6d6d, direct pointer dereferences to CWSR buffers became unsafe because VRAM is accessed via MMIO (PCI BAR mappings).

Direct writes like 'tma[2] = enabled' and memcpy() can fail or produce incorrect results on non-x86 architectures because:
- MMIO requires specific accessor functions (writeq/readq)
- Compiler optimizations may generate invalid instruction sequences
- No guarantee of proper memory barriers or atomic access

This patch converts CWSR buffer access to use struct iosys_map, which automatically handles both system memory (GTT) and MMIO (VRAM) correctly by:
- Using writeq/writel/memcpy_toio for MMIO regions
- Using WRITE_ONCE/memcpy for system memory
- Providing proper memory barriers and access guarantees

Changes:
- Replace void *cwsr_kaddr with struct iosys_map cwsr_map
- Detect MMIO vs system memory using TTM_BO_MAP_IOMEM_MASK
- Use iosys_map_wr() for writing trap handler addresses and flags
- Use iosys_map_memcpy_to() for copying CWSR ISA code

Quoted 5 lines

This ensures correct operation on all architectures while maintaining backward compatibility with older GPUs and APUs that use GTT.

The correct tags would be Co-developed-by and Assisted-by.

Quoted 161 lines
---
  drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  3 +-
  drivers/gpu/drm/amd/amdkfd/kfd_process.c | 63 +++++++++++++++++-------
  2 files changed, 47 insertions(+), 19 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -1
index ad4897f094a2..6e559aab4009 100644
@@ -32,6 +32,7 @@
  #include <linux/atomic.h>
  #include <linux/workqueue.h>
  #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
  #include <uapi/linux/kfd_ioctl.h>
  #include <linux/idr.h>
  #include <linux/kfifo.h>
@@ -710,7 +711,7 @@ struct qcm_process_device {
 
  	/* CWSR memory */
  	struct kgd_mem *cwsr_mem;
-	void *cwsr_kaddr;
+	struct iosys_map cwsr_map;
  	uint64_t cwsr_base;
  	uint64_t tba_addr;
  	uint64_t tma_addr;
drivers/gpu/drm/amd/amdkfd/kfd_process.c +45 -18
index 8e701dcda8ec..7fd65c31afa2 100644
@@ -33,6 +33,7 @@
  #include <linux/mman.h>
  #include <linux/file.h>
  #include <linux/pm_runtime.h>
+#include <drm/ttm/ttm_bo.h>
  #include "amdgpu_amdkfd.h"
  #include "amdgpu.h"
  #include "amdgpu_reset.h"
@@ -745,6 +746,21 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem,
  					       NULL);
  }
 
+static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
+			struct kfd_process_device *pdd, struct iosys_map *map)
+{
+	struct kfd_node *dev = pdd->dev;
+
+	if (map && !iosys_map_is_null(map)) {
+		amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
+		iosys_map_clear(map);
+	}
+
+	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
+	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
+					       NULL);
+}
+
  /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
   *	This function should be only called right after the process
   *	is created and when kfd_processes_mutex is still being held
@@ -1192,8 +1208,8 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
  		if (pdd->drm_file)
  			fput(pdd->drm_file);
 
-		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
-			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
+		if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
+			free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
  				get_order(KFD_CWSR_TBA_TMA_SIZE));
 
  		idr_destroy(&pdd->alloc_idr);
@@ -1501,7 +1517,7 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
  	void *kaddr;
  	int ret;
 
-	if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
  		return 0;
 
  	if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
@@ -1516,17 +1532,28 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
  		return ret;
 
  	qpd->cwsr_mem = mem;
-	qpd->cwsr_kaddr = kaddr;
+
+	/* Set up iosys_map based on whether memory is MMIO or system memory */
+	if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+		iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
+	else
+		iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
+
  	qpd->tba_addr = qpd->cwsr_base;
 
-	memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
+	/* Copy CWSR ISA to buffer using appropriate accessor */
+	iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
+			    dev->kfd->cwsr_isa_size);
 
  	kfd_process_set_trap_debug_flag(&pdd->qpd,
  					pdd->process->debug_trap_enabled);
 
  	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
-		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
+		 qpd->tba_addr, qpd->tma_addr,
+		 qpd->cwsr_map.is_iomem ? "iomem" : "system",
+		 qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
+					  qpd->cwsr_map.vaddr);
 
  	return 0;
  }
@@ -1536,24 +1563,24 @@ static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
  	struct kfd_node *dev = pdd->dev;
  	struct qcm_process_device *qpd = &pdd->qpd;
 
-	if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+	if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
  		return;
 
-	kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
+	kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
  }
 
  void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
  				  uint64_t tba_addr,
  				  uint64_t tma_addr)
  {
-	if (qpd->cwsr_kaddr) {
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
  		/* KFD trap handler is bound, record as second-level TBA/TMA
  		 * in first-level TMA. First-level trap will jump to second.
  		 */
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[0] = tba_addr;
-		tma[1] = tma_addr;
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
+			     uint64_t, tba_addr);
+		iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
+			     uint64_t, tma_addr);
  	} else {
  		/* No trap handler bound, bind as first-level TBA/TMA. */
  		qpd->tba_addr = tba_addr;
@@ -1619,10 +1646,10 @@ bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
  void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
  				     bool enabled)
  {
-	if (qpd->cwsr_kaddr) {
-		uint64_t *tma =
-			(uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-		tma[2] = enabled;
+	if (!iosys_map_is_null(&qpd->cwsr_map)) {
+		iosys_map_wr(&qpd->cwsr_map,
+			     KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
+			     uint64_t, enabled);
  	}
  }
 
#4 lore raw
Quoted 26 lines · Mario Limonciello
Quoted 23 lines · James Zhu

After moving TBA/TMA from GTT to VRAM for GFX9.4.2+ in commit 5088a1ba6d6d, direct pointer dereferences to CWSR buffers became unsafe because VRAM is accessed via MMIO (PCI BAR mappings).

Direct writes like 'tma[2] = enabled' and memcpy() can fail or produce incorrect results on non-x86 architectures because:
- MMIO requires specific accessor functions (writeq/readq)
- Compiler optimizations may generate invalid instruction sequences
- No guarantee of proper memory barriers or atomic access

This patch converts CWSR buffer access to use struct iosys_map, which automatically handles both system memory (GTT) and MMIO (VRAM) correctly by:
- Using writeq/writel/memcpy_toio for MMIO regions
- Using WRITE_ONCE/memcpy for system memory
- Providing proper memory barriers and access guarantees

Changes:
- Replace void *cwsr_kaddr with struct iosys_map cwsr_map
- Detect MMIO vs system memory using TTM_BO_MAP_IOMEM_MASK
- Use iosys_map_wr() for writing trap handler addresses and flags
- Use iosys_map_memcpy_to() for copying CWSR ISA code

Quoted 6 lines
Quoted 5 lines

This ensures correct operation on all architectures while maintaining backward compatibility with older GPUs and APUs that use GTT.

The correct tags would be Co-developed-by and Assisted-by.

JZ] Thanks! I will correct it.

Quoted 161 lines
Quoted 160 lines

---
  drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  3 +- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 63 +++++++++++++++++-------

  2 files changed, 47 insertions(+), 19 deletions(-)
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +2 -1
index ad4897f094a2..6e559aab4009 100644
@@ -32,6 +32,7 @@
   #include <linux/atomic.h>
   #include <linux/workqueue.h>
   #include <linux/spinlock.h>
+#include <linux/iosys-map.h>
   #include <uapi/linux/kfd_ioctl.h>
   #include <linux/idr.h>
   #include <linux/kfifo.h>
@@ -710,7 +711,7 @@ struct qcm_process_device {
         /* CWSR memory */
       struct kgd_mem *cwsr_mem;
-    void *cwsr_kaddr;
+    struct iosys_map cwsr_map;
       uint64_t cwsr_base;
       uint64_t tba_addr;
       uint64_t tma_addr;
drivers/gpu/drm/amd/amdkfd/kfd_process.c +15 -0
index 8e701dcda8ec..7fd65c31afa2 100644
@@ -33,6 +33,7 @@
   #include <linux/mman.h>
   #include <linux/file.h>
   #include <linux/pm_runtime.h>
+#include <drm/ttm/ttm_bo.h>
   #include "amdgpu_amdkfd.h"
   #include "amdgpu.h"
   #include "amdgpu_reset.h"
@@ -745,6 +746,21 @@ static void kfd_process_free_gpuvm(struct 
 kgd_mem *mem,
                              NULL);
   }
   +static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
+            struct kfd_process_device *pdd, struct iosys_map *map)
+{
+    struct kfd_node *dev = pdd->dev;
+
+    if (map && !iosys_map_is_null(map)) {
+        amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
+        iosys_map_clear(map);
+    }
+
+    amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, 
 pdd->drm_priv);
+    amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, 
 pdd->drm_priv,
+                           NULL);
+}
+

  /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process *    This function should be only called right after the process *    is created and when kfd_processes_mutex is still being held

drivers/gpu/drm/amd/amdkfd/kfd_process.c +2 -1
@@ -1192,8 +1208,8 @@ static void kfd_process_destroy_pdds(struct 
 kfd_process *p)
           if (pdd->drm_file)
               fput(pdd->drm_file);
   -        if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
-            free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
+        if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && 
 !pdd->qpd.cwsr_base)
+            free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
                   get_order(KFD_CWSR_TBA_TMA_SIZE));

            idr_destroy(&pdd->alloc_idr);

drivers/gpu/drm/amd/amdkfd/kfd_process.c +1 -0
@@ -1501,7 +1517,7 @@ static int 
 kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
       void *kaddr;
       int ret;
   -    if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
+    if (!dev->kfd->cwsr_enabled || 
 !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
           return 0;

        if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)

drivers/gpu/drm/amd/amdkfd/kfd_process.c +2 -1
@@ -1516,17 +1532,28 @@ static int 
 kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
           return ret;
         qpd->cwsr_mem = mem;
-    qpd->cwsr_kaddr = kaddr;
+
+    /* Set up iosys_map based on whether memory is MMIO or system 

memory */
+    if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
+        iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
+    else
+        iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
+
      qpd->tba_addr = qpd->cwsr_base;
  -    memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size); +    /* Copy CWSR ISA to buffer using appropriate accessor */
+    iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa, +                dev->kfd->cwsr_isa_size); kfd_process_set_trap_debug_flag(&pdd->qpd, pdd->process->debug_trap_enabled); qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-    pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", -         qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+    pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
+         qpd->tba_addr, qpd->tma_addr,
+         qpd->cwsr_map.is_iomem ? "iomem" : "system",
+         qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
+                      qpd->cwsr_map.vaddr);
        return 0;
  }

drivers/gpu/drm/amd/amdkfd/kfd_process.c +7 -5
@@ -1536,24 +1563,24 @@ static void 
 kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
       struct kfd_node *dev = pdd->dev;
       struct qcm_process_device *qpd = &pdd->qpd;
   -    if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
+    if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) 
 || !qpd->cwsr_base)
           return;
   -    kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
+    kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
   }
     void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
                     uint64_t tba_addr,
                     uint64_t tma_addr)
   {
-    if (qpd->cwsr_kaddr) {
+    if (!iosys_map_is_null(&qpd->cwsr_map)) {
           /* KFD trap handler is bound, record as second-level TBA/TMA
            * in first-level TMA. First-level trap will jump to second.
            */
-        uint64_t *tma =
-            (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-        tma[0] = tba_addr;
-        tma[1] = tma_addr;
+        iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
+                 uint64_t, tba_addr);
+        iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + 
 sizeof(uint64_t),
+                 uint64_t, tma_addr);
       } else {

          /* No trap handler bound, bind as first-level TBA/TMA. */ qpd->tba_addr = tba_addr;

drivers/gpu/drm/amd/amdkfd/kfd_process.c +4 -4
@@ -1619,10 +1646,10 @@ bool kfd_process_xnack_mode(struct 
 kfd_process *p, bool supported)
   void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
                        bool enabled)
   {
-    if (qpd->cwsr_kaddr) {
-        uint64_t *tma =
-            (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
-        tma[2] = enabled;
+    if (!iosys_map_is_null(&qpd->cwsr_map)) {
+        iosys_map_wr(&qpd->cwsr_map,
+                 KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
+                 uint64_t, enabled);
       }
   }