Skip to contentSkip to conversation

drm/amdgpu/sdma: add SDMA usermode-queue doorbell pool infra

PATCH In review Jesse Zhang 3 participants 14 messages Apr 28 – Apr 29

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v3 1/11 drm/amdgpu/sdma: add SDMA usermode-queue doorbell pool infra

Add a per-device qword-slot pool covering the firmware-managed NBIO SDMA decode window (BAR dwords [sdma_engine[0], sdma_engine[0] + sdma_doorbell_range * num_instances)) — the only range whose writes are routed to the SDMA back-end. Kernel SDMA ring slots are pre-masked at init.

Exposed to userspace via a fixed mmap pgoff sentinel (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET, bit 56 of the offset, well above DRM's vma_offset_manager range so it can never collide with a real GEM mmap offset). amdgpu_sdma_userq_doorbell_mmap() does io_remap_pfn_range() straight into the SDMA decode window. The pgoff-sentinel dispatcher in amdgpu_drm_mmap() and the AMDGPU_INFO_USERQ_DOORBELL ioctl that hands the offset to userspace land in subsequent patches. Same general pattern as KFD's KFD_MMAP_TYPE_* .

Slots are allocated/freed via amdgpu_sdma_userq_doorbell_alloc/free. The init/fini hooks land in the per-engine sdma6/sdma7 patches.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 154 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  61 +++++++++
 2 files changed, 215 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +154 -0
index 321310ba2c08..be7e9de0630e 100644
@@ -22,6 +22,8 @@
  */
 
 #include <linux/firmware.h>
+#include <linux/io.h>
+#include <linux/mm.h>
 #include "amdgpu.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
@@ -200,6 +202,158 @@ void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
 	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
 }
 
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_sdma_userq_db *db;
+	u32 base_dw, size_dw, nslots, ring_dw;
+	int i;
+
+	if (!adev->userq_funcs[AMDGPU_HW_IP_DMA])
+		return 0;
+
+	base_dw = adev->doorbell_index.sdma_engine[0] << 1;
+	size_dw = adev->doorbell_index.sdma_doorbell_range *
+		  adev->sdma.num_instances;
+	nslots  = size_dw / 2;	/* qword slots */
+	if (!nslots)
+		return 0;
+
+	db = kzalloc(sizeof(*db), GFP_KERNEL);
+	if (!db)
+		return -ENOMEM;
+
+	db->phys_base = adev->doorbell.base +
+			(resource_size_t)base_dw * sizeof(u32);
+	db->size      = size_dw * sizeof(u32);
+
+	mutex_init(&adev->sdma.userq_db_mutex);
+	adev->sdma.userq_db_bitmap = bitmap_zalloc(nslots, GFP_KERNEL);
+	if (!adev->sdma.userq_db_bitmap) {
+		kfree(db);
+		return -ENOMEM;
+	}
+
+	adev->sdma.userq_db        = db;
+	adev->sdma.userq_db_nslots = nslots;
+
+	/*
+	 * Mask out the qword slots used by the kernel SDMA rings
+	 * (sdma_engine[i] << 1 in absolute BAR dwords ⇒ qword slot
+	 * (sdma_engine[i] - sdma_engine[0]) within this window).
+	 */
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring_dw = adev->doorbell_index.sdma_engine[i] << 1;
+		if (ring_dw >= base_dw && ring_dw < base_dw + size_dw)
+			set_bit((ring_dw - base_dw) / 2,
+				adev->sdma.userq_db_bitmap);
+	}
+
+	dev_info(adev->dev,
+		 "SDMA UMQ doorbell pool: %u qword slots in BAR dword [%u, %u)\n",
+		 nslots, base_dw, base_dw + size_dw);
+	return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev)
+{
+	if (!adev->sdma.userq_db)
+		return;
+	bitmap_free(adev->sdma.userq_db_bitmap);
+	adev->sdma.userq_db_bitmap = NULL;
+	adev->sdma.userq_db_nslots = 0;
+	kfree(adev->sdma.userq_db);
+	adev->sdma.userq_db = NULL;
+}
+
+/*
+ * Allocate one qword doorbell slot.  On success, *out_slot receives the
+ * slot id (also the qword index inside the userspace mmap of the window)
+ * which the caller passes back to free.
+ */
+int amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32 *out_slot)
+{
+	u32 slot;
+
+	if (!adev->sdma.userq_db || !adev->sdma.userq_db_nslots)
+		return -ENODEV;
+
+	mutex_lock(&adev->sdma.userq_db_mutex);
+	slot = find_first_zero_bit(adev->sdma.userq_db_bitmap,
+				   adev->sdma.userq_db_nslots);
+	if (slot >= adev->sdma.userq_db_nslots) {
+		mutex_unlock(&adev->sdma.userq_db_mutex);
+		return -ENOSPC;
+	}
+	set_bit(slot, adev->sdma.userq_db_bitmap);
+	mutex_unlock(&adev->sdma.userq_db_mutex);
+
+	*out_slot = slot;
+	return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device *adev, u32 slot)
+{
+	if (!adev->sdma.userq_db)
+		return;
+	if (slot >= adev->sdma.userq_db_nslots)
+		return;
+	mutex_lock(&adev->sdma.userq_db_mutex);
+	clear_bit(slot, adev->sdma.userq_db_bitmap);
+	mutex_unlock(&adev->sdma.userq_db_mutex);
+}
+
+/*
+ * Report the fake mmap offset/size for the SDMA UMQ doorbell window.
+ *
+ * No drm_gem_object, no GEM handle.  Userspace passes the returned offset
+ * directly to mmap(drm_fd, ...); amdgpu_drm_mmap() recognises the
+ * sentinel pgoff and routes the call to amdgpu_sdma_userq_doorbell_mmap()
+ * below, which io_remap_pfn_range()s the routable SDMA doorbell BAR window.
+ *
+ * Same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL — the doorbell window
+ * is unreachable from any drm_gem_object_lookup() / gem_to_amdgpu_bo()
+ * code path because it never enters the GEM object space at all.
+ */
+int amdgpu_sdma_userq_doorbell_get_mmap(struct amdgpu_device *adev,
+					u64 *mmap_offset, u64 *mmap_size)
+{
+	if (!adev->sdma.userq_db)
+		return -ENODEV;
+
+	*mmap_offset = AMDGPU_USERQ_DOORBELL_MMAP_OFFSET;
+	*mmap_size   = adev->sdma.userq_db->size;
+	return 0;
+}
+
+bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned long pgoff)
+{
+	return pgoff == (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET >> PAGE_SHIFT);
+}
+
+int amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+				    struct vm_area_struct *vma)
+{
+	struct drm_file *file_priv = filp->private_data;
+	struct drm_device *dev = file_priv->minor->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_sdma_userq_db *db = adev->sdma.userq_db;
+
+	if (!db)
+		return -ENODEV;
+
+	if (vma->vm_end - vma->vm_start > round_up(db->size, PAGE_SIZE))
+		return -EINVAL;
+
+	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
+			  VM_DONTDUMP | VM_PFNMAP);
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	return io_remap_pfn_range(vma, vma->vm_start,
+				  db->phys_base >> PAGE_SHIFT,
+				  vma->vm_end - vma->vm_start,
+				  vma->vm_page_prot);
+}
+
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
 			       u32 instance, bool duplicate)
 {
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +61 -0

79 lines not on this page · raw message

Signature · 1 line
2.49.0
#1 lore raw

PATCH v3 2/11 drm/amdgpu/userq: route SDMA UMQ doorbells through the kernel pool

From: "Jesse.zhang" <[email protected]>

User-allocated DOORBELL BOs land at BAR offsets outside the firmware-managed NBIO SDMA decode window and cannot reach the SDMA back-end. For AMDGPU_HW_IP_DMA queues, ignore the user-supplied doorbell index and allocate one from the per-device pool added in the previous patch. Track the assigned slot id on the queue so it can be returned to the bitmap on destroy.

Add a new sdma_doorbell_offset_bytes field to drm_amdgpu_userq_out that tells userspace where its kernel-allocated qword slot lives inside the SDMA doorbell window. Userspace gets the window's mmap offset/size from the AMDGPU_INFO_USERQ_DOORBELL ioctl added later in this series and mmap()s the drm fd at that offset.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 46 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h |  7 ++++
 include/uapi/drm/amdgpu_drm.h             |  8 ++++
 3 files changed, 61 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +46 -0
index 8f48520cb822..94ec686049ac 100644
@@ -662,6 +662,10 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que
 	amdgpu_bo_unpin(queue->wptr_obj.obj);
 	amdgpu_bo_unreserve(queue->wptr_obj.obj);
 	amdgpu_bo_unref(&queue->wptr_obj.obj);
+
+	if (queue->sdma_userq_db_slot >= 0)
+		amdgpu_sdma_userq_doorbell_free(adev,
+					(u32)queue->sdma_userq_db_slot);
 	kfree(queue);
 
 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
@@ -762,6 +766,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	queue->queue_type = args->in.ip_type;
 	queue->vm = &fpriv->vm;
 	queue->priority = priority;
+	queue->sdma_userq_db_slot = -1;
 
 	db_info.queue_type = queue->queue_type;
 	db_info.doorbell_handle = queue->doorbell_handle;
@@ -792,6 +797,38 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 		goto clean_mapping;
 	}
 
+	/*
+	 * SDMA UMQ doorbell override:  user-allocated DOORBELL BOs land at
+	 * BAR offsets outside the firmware-managed NBIO SDMA decode window
+	 * and cannot reach the SDMA back-end.  Replace the user-supplied
+	 * doorbell index with one allocated from the per-device
+	 * sdma.userq_db pool that sits inside the routable window.
+	 * Userspace fetches the mmap offset for that pool via
+	 * AMDGPU_INFO_USERQ_DOORBELL with AMDGPU_HW_IP_DMA and mmap()s the
+	 * drm fd at that offset (no GEM handle); sdma_doorbell_offset_bytes
+	 * (returned in args->out) tells userspace where inside that mapping
+	 * its slot lives.
+	 */
+	if (queue->queue_type == AMDGPU_HW_IP_DMA &&
+	    adev->sdma.userq_db) {
+		u32 slot_id;
+
+		r = amdgpu_sdma_userq_doorbell_alloc(adev, &slot_id);
+		if (r) {
+			drm_file_err(uq_mgr->file,
+				     "SDMA UMQ doorbell pool exhausted (err=%d)\n",
+				     r);
+			goto clean_mapping;
+		}
+		/*
+		 * Slot id is a qword index inside the routable window;
+		 * convert to absolute BAR dword index.
+		 */
+		index = (u64)(adev->doorbell_index.sdma_engine[0] << 1) +
+			(u64)slot_id * 2;
+		queue->sdma_userq_db_slot = (int)slot_id;
+	}
+
 	queue->doorbell_index = index;
 	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
 	r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
@@ -851,6 +888,15 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	amdgpu_userq_init_hang_detect_work(queue);
 
 	args->out.queue_id = qid;
+	if (queue->sdma_userq_db_slot >= 0) {
+		/*
+		 * Tell userspace where inside its mmap of the SDMA UMQ
+		 * doorbell BO (handle returned by AMDGPU_INFO_USERQ_DOORBELL with
+		 * AMDGPU_HW_IP_DMA) the assigned qword slot lives.
+		 */
+		args->out.sdma_doorbell_offset_bytes =
+			(u64)queue->sdma_userq_db_slot * sizeof(u64);
+	}
 	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
 	mutex_unlock(&uq_mgr->userq_mutex);
 	return 0;
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +7 -0
index 675fe6395ac8..cdfced627dec 100644
@@ -59,6 +59,13 @@ struct amdgpu_usermode_queue {
 	uint64_t		doorbell_handle;
 	uint64_t		doorbell_index;
 	uint64_t		flags;
+	/*
+	 * For SDMA UMQs whose doorbell came from the kernel-managed pool
+	 * (amdgpu_sdma_userq_doorbell_alloc), record the slot id so it can
+	 * be returned to the bitmap on queue destroy.  -1 means the queue
+	 * is using a user-supplied doorbell BO.
+	 */
+	int			sdma_userq_db_slot;
 	struct amdgpu_mqd_prop	*userq_prop;
 	struct amdgpu_userq_mgr *userq_mgr;
 	struct amdgpu_vm	*vm;
include/uapi/drm/amdgpu_drm.h +8 -0
index 9f3090db2f16..79e8bbda046b 100644
@@ -421,6 +421,14 @@ struct drm_amdgpu_userq_out {
 	 */
 	__u32	queue_id;
 	__u32 _pad;
+	/**
+	 * For SDMA usermode queues whose doorbell was assigned by the
+	 * kernel from the per-device pool (see AMDGPU_INFO_SDMA_USERQ_DOORBELL),
+	 * this field carries the byte offset of the assigned slot inside
+	 * the routable doorbell window so userspace can write there.
+	 * 0 means the kernel did not override the user's doorbell.
+	 */
+	__u64	sdma_doorbell_offset_bytes;
 };
 
 union drm_amdgpu_userq {
Signature · 1 line
2.49.0
#2 lore raw

PATCH v3 3/11 drm/amdgpu/sdma7: register SDMA UMQ doorbell pool

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

Call amdgpu_sdma_userq_doorbell_init() at sw_init time (gated on userq_funcs[AMDGPU_HW_IP_DMA] being set so we only run on chips that actually expose SDMA UMQs) and amdgpu_sdma_userq_doorbell_fini() at sw_fini, so SDMA usermode queues get doorbells from the firmware-managed NBIO routable window.

---
 drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c | 9 +++++++++
 1 file changed, 9 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +9 -0
index 85d98a0e1bff..5f6c51ba7ac1 100644
@@ -1378,6 +1378,14 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block)
 		break;
 	}
 
+	/*
+	 * Init the SDMA usermode-queue doorbell pool inside the firmware-
+	 * managed NBIO S2A SDMA decode window so user SDMA UMQs get
+	 * doorbells that are actually routable to the SDMA back-end.
+	 */
+	if (adev->userq_funcs[AMDGPU_HW_IP_DMA])
+		amdgpu_sdma_userq_doorbell_init(adev);
+
 	return r;
 }
 
@@ -1389,6 +1397,7 @@ static int sdma_v7_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	for (i = 0; i < adev->sdma.num_instances; i++)
 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 
+	amdgpu_sdma_userq_doorbell_fini(adev);
 	amdgpu_sdma_sysfs_reset_mask_fini(adev);
 	amdgpu_sdma_destroy_inst_ctx(adev, true);
 
Signature · 1 line
2.49.0
#3 lore raw

PATCH v3 4/11 drm/amdgpu/sdma6: register SDMA UMQ doorbell pool

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

Call amdgpu_sdma_userq_doorbell_init() at sw_init time (gated on userq_funcs[AMDGPU_HW_IP_DMA] being set so we only run on chips that actually expose SDMA UMQs) and amdgpu_sdma_userq_doorbell_fini() at sw_fini, so SDMA usermode queues get doorbells from the firmware-managed NBIO routable window.

---
 drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 9 +++++++++
 1 file changed, 9 insertions(+)
drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +9 -0
index de329b76a00c..02eeac3b2e11 100644
@@ -1426,6 +1426,14 @@ static int sdma_v6_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
+	/*
+	 * Init the SDMA usermode-queue doorbell pool inside the firmware-
+	 * managed NBIO SDMA decode window so user SDMA UMQs get doorbells
+	 * that are actually routable to the SDMA back-end.
+	 */
+	if (adev->userq_funcs[AMDGPU_HW_IP_DMA])
+		amdgpu_sdma_userq_doorbell_init(adev);
+
 	return r;
 }
 
@@ -1437,6 +1445,7 @@ static int sdma_v6_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	for (i = 0; i < adev->sdma.num_instances; i++)
 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 
+	amdgpu_sdma_userq_doorbell_fini(adev);
 	amdgpu_sdma_sysfs_reset_mask_fini(adev);
 	amdgpu_sdma_destroy_inst_ctx(adev, true);
 
Signature · 1 line
2.49.0
#4 lore raw

PATCH v3 5/11 drm/amdgpu: add AMDGPU_INFO_USERQ_DOORBELL ioctl

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

Per-IP doorbell layout query. Input: info->query_hw_ip.type selects the IP (AMDGPU_HW_IP_GFX / COMPUTE / DMA / VCN_ENC). Output: the doorbell-BAR dword range usable by usermode queues for that IP, the aggregated doorbell offset (when MES owns one), and — for AMDGPU_HW_IP_DMA only — a fixed mmap pgoff sentinel plus byte size of the kernel-owned routable SDMA doorbell window.

For SDMA usermode queues the kernel hands out doorbell slots from the per-device pool (the only BAR range whose writes are routed to the SDMA back-end) instead of accepting user-allocated doorbell BOs. The returned mmap_offset is the AMDGPU_USERQ_DOORBELL_MMAP_OFFSET sentinel; userspace mmap()s the drm fd at that offset (no GEM handle, no drm_gem_object_lookup() consumer can ever reach this window) and the per-queue qword-slot offset inside the mapping is reported in drm_amdgpu_userq_out.sdma_doorbell_offset_bytes.

amdgpu_drm_mmap() (a thin wrapper installed in amdgpu_drv's file_operations in place of drm_gem_mmap) dispatches on the pgoff:
sentinel hits go to amdgpu_sdma_userq_doorbell_mmap() which does io_remap_pfn_range() straight into the SDMA decode window; everything else falls through to drm_gem_mmap().

V2: Picks up David Wu's [PATCH 11/14] AMDGPU_INFO_DOORBELL design (struct
shape + per-IP dispatch) and extends it with the SDMA-only mmap_offset/ mmap_size fields.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 16 +++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 35 ++++++++++++++++++++++
 include/uapi/drm/amdgpu_drm.h           | 40 +++++++++++++++++++++++--
 3 files changed, 87 insertions(+), 4 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +15 -1
index c6546a858597..6e0acd761e94 100644
@@ -3023,13 +3023,27 @@ static int amdgpu_flush(struct file *f, fl_owner_t id)
 	return timeout >= 0 ? 0 : timeout;
 }
 
+/*
+ * mmap dispatcher.  We override the standard drm_gem_mmap because the
+ * SDMA UMQ doorbell window is reached via a fixed sentinel pgoff
+ * (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET) — it has no drm_gem_object so
+ * drm_gem_mmap()'s vma_offset_manager lookup would fail.  Same pattern
+ * as KFD's KFD_MMAP_TYPE_DOORBELL.
+ */
+static int amdgpu_drm_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	if (amdgpu_sdma_userq_doorbell_is_our_pgoff(vma->vm_pgoff))
+		return amdgpu_sdma_userq_doorbell_mmap(filp, vma);
+	return drm_gem_mmap(filp, vma);
+}
+
 static const struct file_operations amdgpu_driver_kms_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
 	.flush = amdgpu_flush,
 	.release = amdgpu_drm_release,
 	.unlocked_ioctl = amdgpu_drm_ioctl,
-	.mmap = drm_gem_mmap,
+	.mmap = amdgpu_drm_mmap,
 	.poll = drm_poll,
 	.read = drm_read,
 #ifdef CONFIG_COMPAT
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +35 -0
index d88e4994c8c1..63e387bb691a 100644
@@ -1425,6 +1425,41 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 			return -EINVAL;
 		}
 	}
+	case AMDGPU_INFO_USERQ_DOORBELL: {
+		struct drm_amdgpu_info_userq_doorbell db_info = {};
+		u32 agdb = adev->enable_mes ?
+			adev->mes.aggregated_doorbells[AMDGPU_MES_PRIORITY_LEVEL_NORMAL] : 0;
+		int r;
+
+		switch (info->query_hw_ip.type) {
+		case AMDGPU_HW_IP_DMA:
+			if (!adev->sdma.userq_db)
+				return -ENODEV;
+			db_info.index_start =
+				adev->doorbell_index.sdma_engine[0] << 1;
+			db_info.index_end = db_info.index_start +
+				adev->doorbell_index.sdma_doorbell_range *
+				adev->sdma.num_instances - 1;
+			if (agdb) {
+				db_info.agdb_enable = 1;
+				db_info.agdb_offset = agdb;
+			}
+			r = amdgpu_sdma_userq_doorbell_get_mmap(adev,
+					&db_info.mmap_offset,
+					&db_info.mmap_size);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_HW_IP_VCN_ENC:
+		case AMDGPU_HW_IP_GFX:
+		case AMDGPU_HW_IP_COMPUTE:
+		default:
+			return -EINVAL;
+		}
+		return copy_to_user(out, &db_info,
+				    min((size_t)size, sizeof(db_info)))
+			? -EFAULT : 0;
+	}
 	default:
 		DRM_DEBUG_KMS("Invalid request %d\n", info->query);
 		return -EINVAL;
include/uapi/drm/amdgpu_drm.h +37 -3
index 79e8bbda046b..458219a42da3 100644
@@ -423,9 +423,10 @@ struct drm_amdgpu_userq_out {
 	__u32 _pad;
 	/**
 	 * For SDMA usermode queues whose doorbell was assigned by the
-	 * kernel from the per-device pool (see AMDGPU_INFO_SDMA_USERQ_DOORBELL),
-	 * this field carries the byte offset of the assigned slot inside
-	 * the routable doorbell window so userspace can write there.
+	 * kernel from the per-device pool (see AMDGPU_INFO_USERQ_DOORBELL with
+	 * AMDGPU_HW_IP_DMA), this field carries the byte offset of the
+	 * assigned slot inside the routable doorbell window so userspace
+	 * can write there.
 	 * 0 means the kernel did not override the user's doorbell.
 	 */
 	__u64	sdma_doorbell_offset_bytes;
@@ -1280,6 +1281,39 @@ struct drm_amdgpu_cs_chunk_cp_gfx_shadow {
 #define AMDGPU_INFO_GPUVM_FAULT			0x23
 /* query FW object size and alignment */
 #define AMDGPU_INFO_UQ_FW_AREAS			0x24
+/*
+ * Per-IP doorbell layout query.  Input: info->query_hw_ip.type selects
+ * the IP (AMDGPU_HW_IP_GFX / COMPUTE / DMA / VCN_ENC).  Output:
+ * doorbell-BAR dword range usable by usermode queues for that IP, the
+ * aggregated doorbell offset (when MES/UMSCH owns one for the IP), and
+ * — for AMDGPU_HW_IP_DMA only — a per-fpriv GEM handle for a
+ * kernel-owned BO that backs the routable SDMA doorbell window so
+ * userspace can mmap() its assigned slot.
+ */
+#define AMDGPU_INFO_USERQ_DOORBELL			0x25
+
+struct drm_amdgpu_info_userq_doorbell {
+	/* BAR dword index of the start of the per-IP doorbell range. */
+	__u32 index_start;
+	/* BAR dword index of the last doorbell in the range (inclusive). */
+	__u32 index_end;
+	/* 1 if an aggregated doorbell exists for this IP. */
+	__u32 agdb_enable;
+	/* If agdb_enable, BAR dword index of the aggregated doorbell. */
+	__u32 agdb_offset;
+	/*
+	 * AMDGPU_HW_IP_DMA only: per-fpriv GEM handle for the kernel-owned
+	 * BO backing the routable SDMA doorbell window.  Userspace mmap()s
+	 * it through the standard AMDGPU_GEM_OP_MMAP / mmap() flow to get
+	 * a CPU pointer; each created SDMA usermode queue's qword-slot
+	 * offset inside that mapping is reported in
+	 * drm_amdgpu_userq_out.sdma_doorbell_offset_bytes.  0 for IPs that
+	 * do not need a kernel-managed doorbell BO.
+	 */
+	__u64 mmap_offset;
+	/* Byte size of the mmap region; 0 when mmap_offset is 0. */
+	__u64 mmap_size;
+};
 
 #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT	0
 #define AMDGPU_INFO_MMR_SE_INDEX_MASK	0xff
Signature · 1 line
2.49.0
#5 lore raw

PATCH v3 6/11 drm/amdgpu/mes: add NOTIFY_WORK_ON_UNMAPPED_QUEUE op + ADD_QUEUE fields

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

Kernel-side abstraction work for the SDMA usermode-queue plumbing landed in subsequent per-engine patches:

- mes_add_queue_input gains is_user_mode_submission and unmap_flag_addr. Without is_user_mode_submission MES treats SDMA queues as kernel-managed and uses the end-of-MQD slot for the unmap flag, so PROTECTED_FENCE at the tail of every SDMA IB looks like a "queue done" signal and MES gangs the queue out forever.

- mes_misc_opcode gains MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE with a notify_work.priority_level payload. This wakes a gangs-out SDMA UMQ so subsequent IBs get re-mapped (SDMA has no CP_UNMAPPED_DOORBELL HW intercept).

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h       |  7 +++++++
 drivers/gpu/drm/amd/include/mes_v12_api_def.h | 12 +++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +7 -0
index cafc5caae822..705056de94b0 100644
@@ -265,6 +265,8 @@ struct mes_add_queue_input {
 	uint32_t	exclusively_scheduled;
 	uint32_t	sh_mem_config_data;
 	uint32_t	vm_cntx_cntl;
+	uint32_t	is_user_mode_submission;
+	uint64_t	unmap_flag_addr;
 };
 
 struct mes_remove_queue_input {
@@ -343,6 +345,7 @@ enum mes_misc_opcode {
 	MES_MISC_OP_WRM_REG_WR_WAIT,
 	MES_MISC_OP_SET_SHADER_DEBUGGER,
 	MES_MISC_OP_CHANGE_CONFIG,
+	MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE,
 };
 
 struct mes_misc_op_input {
@@ -397,6 +400,10 @@ struct mes_misc_op_input {
 				uint32_t tdr_delay;
 			} tdr_config;
 		} change_config;
+
+		struct {
+			uint32_t priority_level;
+		} notify_work;
 	};
 };
 
drivers/gpu/drm/amd/include/mes_v12_api_def.h +11 -1
index e541a43714a1..cd6e60184a06 100644
@@ -381,7 +381,11 @@ union MESAPI__ADD_QUEUE {
 			uint32_t exclusively_scheduled : 1;
 			uint32_t is_long_running : 1;
 			uint32_t is_dwm_queue : 1;
-			uint32_t reserved	 : 15;
+			uint32_t is_video_blit_queue : 1;
+			uint32_t is_user_mode_submission : 1;
+			uint32_t enable_perf_profiling : 1;
+			uint32_t exclude_process_limit : 1;
+			uint32_t reserved	 : 11;
 		};
 		struct MES_API_STATUS	api_status;
 		uint64_t		tma_addr;
@@ -393,6 +397,12 @@ union MESAPI__ADD_QUEUE {
 		uint32_t		queue_id;
 		uint32_t		alignment_mode_setting;
 		uint32_t		full_sh_mem_config_data;
+		/*
+		 * MC addr where MES writes 1 when it unmaps the queue.  Used
+		 * by user-mode SDMA UMQs so the kernel/userspace can detect
+		 * the unmapped state and re-arm work via NOTIFY_WORK_ON_UNMAPPED_QUEUE.
+		 */
+		uint64_t		unmap_flag_addr;
 	};
 
 	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
Signature · 1 line
2.49.0
#6 lore raw

PATCH v3 7/11 drm/amdgpu/mes11: plumb unmap_flag_addr + NOTIFY_WORK_ON_UNMAPPED_QUEUE

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

Pass the new mes_add_queue_input.unmap_flag_addr through to the MESAPI__ADD_QUEUE packet, and route MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE to the matching MESAPI_MISC opcode.

Note: the MES v11 firmware spec does not (yet) carry a per-queue is_user_mode_submission bit, so SDMA UMQs on chips with MES v11 may still see PROTECTED_FENCE-as-queue-done behaviour after the first IB until firmware adds the bit. The wakeup mechanism (NOTIFY) is wired up so that path is ready when firmware lands.

---
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +14 -0
index a926a330700e..575cc4a684b1 100644
@@ -362,6 +362,16 @@ static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
 
 	mes_add_queue_pkt.exclusively_scheduled = input->exclusively_scheduled;
 
+	/*
+	 * unmap_flag_addr is plumbed through but only honoured by MES when
+	 * the global use_add_queue_unmap_flag_addr flag is set in
+	 * SET_HW_RESOURCES.  MES v11 firmware spec does not carry a
+	 * per-queue is_user_mode_submission bit, so SDMA UMQs on chips with
+	 * MES v11 may still see PROTECTED_FENCE-as-queue-done behaviour
+	 * until firmware adds the bit.
+	 */
+	mes_add_queue_pkt.unmap_flag_addr = input->unmap_flag_addr;
+
 	return mes_v11_0_submit_pkt_and_poll_completion(mes,
 			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 			offsetof(union MESAPI__ADD_QUEUE, api_status));
@@ -660,6 +670,10 @@ static int mes_v11_0_misc_op(struct amdgpu_mes *mes,
 		misc_pkt.change_config.option.bits.limit_single_process =
 				input->change_config.option.limit_single_process;
 		break;
+	case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE:
+		misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+		misc_pkt.queue_sch_level = input->notify_work.priority_level;
+		break;
 
 	default:
 		drm_err(adev_to_drm(mes->adev), "unsupported misc op (%d)\n", input->op);
Signature · 1 line
2.49.0
#7 lore raw

PATCH v3 8/11 drm/amdgpu/mes12: plumb is_user_mode_submission, unmap_flag_addr, NOTIFY

In reply to Jesse Zhang

Pass is_user_mode_submission and unmap_flag_addr from mes_add_queue_input through to MESAPI__ADD_QUEUE in both mes_v12_0 add_hw_queue paths, and route MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE to the matching MESAPI_MISC opcode.

The kernel-side caller that actually sets is_user_mode_submission for SDMA UMQs lives in a later patch; this one is just the engine-level plumbing.

---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 6 ++++++
 1 file changed, 6 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +6 -0
index 023c7345ea54..5acc505533f3 100644
@@ -342,6 +342,8 @@ static int mes_v12_0_add_hw_queue(struct amdgpu_mes *mes,
 	mes_add_queue_pkt.trap_en = input->trap_en;
 	mes_add_queue_pkt.skip_process_ctx_clear = input->skip_process_ctx_clear;
 	mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
+	mes_add_queue_pkt.is_user_mode_submission = input->is_user_mode_submission;
+	mes_add_queue_pkt.unmap_flag_addr = input->unmap_flag_addr;
 
 	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
 	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
@@ -697,6 +699,10 @@ static int mes_v12_0_misc_op(struct amdgpu_mes *mes,
 		misc_pkt.change_config.option.bits.limit_single_process =
 				input->change_config.option.limit_single_process;
 		break;
+	case MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE:
+		misc_pkt.opcode = MESAPI_MISC__NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+		misc_pkt.queue_sch_level = input->notify_work.priority_level;
+		break;
 
 	default:
 		DRM_ERROR("unsupported misc op (%d)\n", input->op);
Signature · 1 line
2.49.0
#8 lore raw

PATCH v3 9/11 drm/amdgpu/mes_userqueue: mark SDMA UMQs as user-mode submission

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

For AMDGPU_HW_IP_DMA queues, set mes_add_queue_input.is_user_mode_submission and a stable unmap_flag_addr (a kernel-owned dword in the MQD object's tail padding). This tells MES to use the new wptr_mc / unmap_flag scheme so the PROTECTED_FENCE at the tail of every SDMA IB no longer terminates the queue. Combined with the NOTIFY_WORK_ON_UNMAPPED_QUEUE wakeup added in a follow-up patch, this lets multi-IB submissions on a single SDMA UMQ work end-to-end.

---
 drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +22 -0
index d12cd1b7790b..3dbcddb46b24 100644
@@ -165,6 +165,28 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue)
 	queue_input.doorbell_offset = userq_props->doorbell_index;
 	queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
 	queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
+	/*
+	 * SDMA UMQs need is_user_mode_submission so MES treats them as user
+	 * queues (using the new wptr_mc_addr / unmap_flag_addr scheme).
+	 * Without this MES uses end-of-MQD for unmap_flag, sees PROTECTED_FENCE
+	 * as a "queue done" signal, and gangs the queue out forever.  Combined
+	 * with NOTIFY_WORK_ON_UNMAPPED_QUEUE poke from amdgpu_userq_signal_ioctl
+	 * this lets multi-IB submissions work.  Use queue->mqd.gpu_addr +
+	 * mqd_size as a stable kernel-owned location for unmap_flag — userspace
+	 * never reads it; the kernel just needs SOMETHING valid to give MES.
+	 */
+	if (queue->queue_type == AMDGPU_HW_IP_DMA) {
+		queue_input.is_user_mode_submission = 1;
+		/*
+		 * Same offset MES would derive in legacy mode
+		 * (get_unmap_flag_addr_from_end_of_mqd in MES src 12).  Lives
+		 * inside the allocated MQD object's tail padding so it's a
+		 * valid MC address; the kernel never reads it back — its only
+		 * purpose is to keep MES happy.
+		 */
+		queue_input.unmap_flag_addr = queue->mqd.gpu_addr +
+			adev->mqds[queue->queue_type].mqd_size + sizeof(u32);
+	}
 
 	amdgpu_mes_lock(&adev->mes);
 	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
Signature · 1 line
2.49.0
#9 lore raw

PATCH v3 10/11 drm/amdgpu/userq_fence: wake gangs-out SDMA UMQs via NOTIFY

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

SDMA has no CP_UNMAPPED_DOORBELL HW intercept, so once MES gangs the queue out (after the first IB idles it) per-queue doorbell rings from userspace hit a mapped-out HW slot and are silently dropped: rptr stops advancing and FENCE IRQ never fires.

After the SDMA UMQ's first IB has actually completed (fence_drv->cpu_addr != 0), issue MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE and ring the priority's aggregated doorbell so MES re-evaluates scheduling and re-maps the queue for the next IB. The first submission is intentionally skipped — the queue is still mapped from MAP_QUEUE then, and an extra notify would race the initial scheduling.

---
 .../gpu/drm/amd/amdgpu/amdgpu_userq_fence.c   | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +33 -0
index a58342c2ac44..6ef4cbd5d5da 100644
@@ -598,6 +598,39 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	/* drop the reference acquired in fence creation function */
 	dma_fence_put(fence);
 
+	/*
+	 * SDMA UMQ wake: SDMA has no CP_UNMAPPED_DOORBELL HW intercept, so
+	 * once MES gangs the queue out (after the first IB's PROTECTED_FENCE
+	 * idles the queue), subsequent per-queue doorbell rings hit a
+	 * mapped-out HW slot and are silently ignored — rptr stops
+	 * advancing, FENCE IRQ never fires.  The MES MISC API
+	 * NOTIFY_WORK_ON_UNMAPPED_QUEUE flips MES's hasReadyQueues flag for
+	 * the queue's priority level, which makes MES re-evaluate
+	 * scheduling and re-map our SDMA UMQ for the next IB.
+	 *
+	 * Skip on the very first submission (fence_drv->cpu_addr == 0
+	 * means SDMA hasn't completed any IB yet, so MES still has the
+	 * queue mapped from MAP_QUEUE — calling NOTIFY here would race the
+	 * initial scheduling and starve the first IB).
+	 */
+	if (queue && queue->queue_type == AMDGPU_HW_IP_DMA &&
+	    adev->enable_mes && adev->mes.funcs->misc_op &&
+	    queue->fence_drv && queue->fence_drv->cpu_addr &&
+	    le64_to_cpu(*queue->fence_drv->cpu_addr) != 0) {
+		struct mes_misc_op_input op = { 0 };
+		u32 agg_db = adev->mes.aggregated_doorbells[
+				AMDGPU_MES_PRIORITY_LEVEL_NORMAL];
+
+		op.op = MES_MISC_OP_NOTIFY_WORK_ON_UNMAPPED_QUEUE;
+		op.notify_work.priority_level = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
+		amdgpu_mes_lock(&adev->mes);
+		(void)adev->mes.funcs->misc_op(&adev->mes, &op);
+		amdgpu_mes_unlock(&adev->mes);
+
+		if (agg_db)
+			WDOORBELL64(agg_db, queue->doorbell_index);
+	}
+
 exec_fini:
 	drm_exec_fini(&exec);
 put_gobj_write:
Signature · 1 line
2.49.0
#10 lore raw

PATCH v3 11/11 drm/amdgpu/userq_fence: workaround SDMA UMQ first-submit doorbell race

In reply to Jesse Zhang

From: "Jesse.zhang" <[email protected]>

SDMA UMQ can hang on the first submission after creation because userspace may ring the doorbell before LSDMA has programmed the HQD doorbell offset. The doorbell write is silently dropped, and SDMA's subsequent MCU_WPTR_POLL leaves the trailing PROTECTED_FENCE unread, so the fence never fires.

Re-ring the doorbell from amdgpu_userq_signal_ioctl() after updating wptr to ensure SDMA switches to burst-read mode and consumes all packets. This is a workaround; remove it once the firmware issue is fixed.

Verified by 100/100 successful iterations of
amd_basic --run-subtest cs-sdma-with-IP-DMA-UMQ run back-to-back on navi48; previously hung intermittently between iter 7 and iter 30+.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 3 +++
 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +3 -0
index 6ef4cbd5d5da..d00ed5d4bbc3 100644
@@ -535,6 +535,9 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 	if (r)
 		goto put_gobj_write;
 
+	if (queue->queue_type == AMDGPU_HW_IP_DMA && queue->doorbell_index)
+		WDOORBELL64(queue->doorbell_index, wptr);
+
 	r = amdgpu_userq_fence_alloc(&userq_fence);
 	if (r)
 		goto put_gobj_write;
Signature · 1 line
2.49.0
maintainer
#11 lore raw
In reply to Jesse Zhang
Quoted 10 lines · Jesse Zhang

Add a per-device qword-slot pool covering the firmware-managed NBIO SDMA decode window (BAR dwords [sdma_engine[0], sdma_engine[0] + sdma_doorbell_range * num_instances)) — the only range whose writes are routed to the SDMA back-end. Kernel SDMA ring slots are pre-masked at init.

Exposed to userspace via a fixed mmap pgoff sentinel (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET, bit 56 of the offset, well above DRM's vma_offset_manager range so it can never collide with a real GEM mmap offset).

That is a pretty bad idea, we should use the approach we came up with for the MMIO BAR as well.

Quoted 2 lines

amdgpu_sdma_userq_doorbell_mmap() does io_remap_pfn_range() straight into the SDMA decode window.

That is also a pretty big no-go. What exactly is the requirement here?

Do we need to use a global portion of the doorbell to keep the SDMA doorbells together?

If yes where the heck does that requirement comes from because that clearly goes against upstream requirements on managing the doorbells.

Regards,
Christian.

Quoted 275 lines

The
pgoff-sentinel dispatcher in amdgpu_drm_mmap() and the AMDGPU_INFO_USERQ_DOORBELL ioctl that hands the offset to userspace land in subsequent patches. Same general pattern as KFD's KFD_MMAP_TYPE_* .

Slots are allocated/freed via amdgpu_sdma_userq_doorbell_alloc/free. The init/fini hooks land in the per-engine sdma6/sdma7 patches.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 154 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  61 +++++++++
 2 files changed, 215 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +154 -0
index 321310ba2c08..be7e9de0630e 100644
@@ -22,6 +22,8 @@
  */
 
 #include <linux/firmware.h>
+#include <linux/io.h>
+#include <linux/mm.h>
 #include "amdgpu.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
@@ -200,6 +202,158 @@ void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
 	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
 }
 
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_sdma_userq_db *db;
+	u32 base_dw, size_dw, nslots, ring_dw;
+	int i;
+
+	if (!adev->userq_funcs[AMDGPU_HW_IP_DMA])
+		return 0;
+
+	base_dw = adev->doorbell_index.sdma_engine[0] << 1;
+	size_dw = adev->doorbell_index.sdma_doorbell_range *
+		  adev->sdma.num_instances;
+	nslots  = size_dw / 2;	/* qword slots */
+	if (!nslots)
+		return 0;
+
+	db = kzalloc(sizeof(*db), GFP_KERNEL);
+	if (!db)
+		return -ENOMEM;
+
+	db->phys_base = adev->doorbell.base +
+			(resource_size_t)base_dw * sizeof(u32);
+	db->size      = size_dw * sizeof(u32);
+
+	mutex_init(&adev->sdma.userq_db_mutex);
+	adev->sdma.userq_db_bitmap = bitmap_zalloc(nslots, GFP_KERNEL);
+	if (!adev->sdma.userq_db_bitmap) {
+		kfree(db);
+		return -ENOMEM;
+	}
+
+	adev->sdma.userq_db        = db;
+	adev->sdma.userq_db_nslots = nslots;
+
+	/*
+	 * Mask out the qword slots used by the kernel SDMA rings
+	 * (sdma_engine[i] << 1 in absolute BAR dwords ⇒ qword slot
+	 * (sdma_engine[i] - sdma_engine[0]) within this window).
+	 */
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		ring_dw = adev->doorbell_index.sdma_engine[i] << 1;
+		if (ring_dw >= base_dw && ring_dw < base_dw + size_dw)
+			set_bit((ring_dw - base_dw) / 2,
+				adev->sdma.userq_db_bitmap);
+	}
+
+	dev_info(adev->dev,
+		 "SDMA UMQ doorbell pool: %u qword slots in BAR dword [%u, %u)\n",
+		 nslots, base_dw, base_dw + size_dw);
+	return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev)
+{
+	if (!adev->sdma.userq_db)
+		return;
+	bitmap_free(adev->sdma.userq_db_bitmap);
+	adev->sdma.userq_db_bitmap = NULL;
+	adev->sdma.userq_db_nslots = 0;
+	kfree(adev->sdma.userq_db);
+	adev->sdma.userq_db = NULL;
+}
+
+/*
+ * Allocate one qword doorbell slot.  On success, *out_slot receives the
+ * slot id (also the qword index inside the userspace mmap of the window)
+ * which the caller passes back to free.
+ */
+int amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32 *out_slot)
+{
+	u32 slot;
+
+	if (!adev->sdma.userq_db || !adev->sdma.userq_db_nslots)
+		return -ENODEV;
+
+	mutex_lock(&adev->sdma.userq_db_mutex);
+	slot = find_first_zero_bit(adev->sdma.userq_db_bitmap,
+				   adev->sdma.userq_db_nslots);
+	if (slot >= adev->sdma.userq_db_nslots) {
+		mutex_unlock(&adev->sdma.userq_db_mutex);
+		return -ENOSPC;
+	}
+	set_bit(slot, adev->sdma.userq_db_bitmap);
+	mutex_unlock(&adev->sdma.userq_db_mutex);
+
+	*out_slot = slot;
+	return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device *adev, u32 slot)
+{
+	if (!adev->sdma.userq_db)
+		return;
+	if (slot >= adev->sdma.userq_db_nslots)
+		return;
+	mutex_lock(&adev->sdma.userq_db_mutex);
+	clear_bit(slot, adev->sdma.userq_db_bitmap);
+	mutex_unlock(&adev->sdma.userq_db_mutex);
+}
+
+/*
+ * Report the fake mmap offset/size for the SDMA UMQ doorbell window.
+ *
+ * No drm_gem_object, no GEM handle.  Userspace passes the returned offset
+ * directly to mmap(drm_fd, ...); amdgpu_drm_mmap() recognises the
+ * sentinel pgoff and routes the call to amdgpu_sdma_userq_doorbell_mmap()
+ * below, which io_remap_pfn_range()s the routable SDMA doorbell BAR window.
+ *
+ * Same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL — the doorbell window
+ * is unreachable from any drm_gem_object_lookup() / gem_to_amdgpu_bo()
+ * code path because it never enters the GEM object space at all.
+ */
+int amdgpu_sdma_userq_doorbell_get_mmap(struct amdgpu_device *adev,
+					u64 *mmap_offset, u64 *mmap_size)
+{
+	if (!adev->sdma.userq_db)
+		return -ENODEV;
+
+	*mmap_offset = AMDGPU_USERQ_DOORBELL_MMAP_OFFSET;
+	*mmap_size   = adev->sdma.userq_db->size;
+	return 0;
+}
+
+bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned long pgoff)
+{
+	return pgoff == (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET >> PAGE_SHIFT);
+}
+
+int amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+				    struct vm_area_struct *vma)
+{
+	struct drm_file *file_priv = filp->private_data;
+	struct drm_device *dev = file_priv->minor->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+	struct amdgpu_sdma_userq_db *db = adev->sdma.userq_db;
+
+	if (!db)
+		return -ENODEV;
+
+	if (vma->vm_end - vma->vm_start > round_up(db->size, PAGE_SIZE))
+		return -EINVAL;
+
+	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
+			  VM_DONTDUMP | VM_PFNMAP);
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	return io_remap_pfn_range(vma, vma->vm_start,
+				  db->phys_base >> PAGE_SHIFT,
+				  vma->vm_end - vma->vm_start,
+				  vma->vm_page_prot);
+}
+
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
 			       u32 instance, bool duplicate)
 {
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +61 -0
index 2bf365609775..a308d5e6ee54 100644
@@ -146,6 +146,20 @@ struct amdgpu_sdma {
 	bool			disable_uq;
 	void (*get_csa_info)(struct amdgpu_device *adev,
 			     struct amdgpu_sdma_csa_info *csa_info);
+
+	/*
+	 * SDMA usermode-queue doorbell pool.  The window covers
+	 * BAR dwords [sdma_engine[0], sdma_engine[0] +
+	 * sdma_doorbell_range * num_instances) — the only range that NBIO
+	 * routes to the SDMA back-end.  Each bit in the bitmap represents
+	 * one qword slot; kernel SDMA ring slots are pre-masked at init.
+	 * Exposed to userspace via a fixed mmap pgoff sentinel handled by
+	 * amdgpu_drm_mmap() — see AMDGPU_USERQ_DOORBELL_MMAP_OFFSET below.
+	 */
+	struct amdgpu_sdma_userq_db *userq_db;
+	struct mutex		userq_db_mutex;
+	unsigned long		*userq_db_bitmap;
+	u32			userq_db_nslots;	/* qword slots */
 };
 
 /*
@@ -185,6 +199,42 @@ struct amdgpu_buffer_funcs {
 				 uint32_t byte_count);
 };
 
+/*
+ * SDMA usermode-queue doorbell pool.
+ *
+ * The pool re-uses qword doorbell slots inside the firmware-managed NBIO
+ * SDMA decode window (BAR dwords [sdma_engine[0],
+ * sdma_engine[0] + sdma_doorbell_range * num_instances)) — that range is
+ * the only one whose writes are routed to the SDMA back-end.  The kernel
+ * SDMA ring slots are pre-marked so they keep working alongside any
+ * number of SDMA UMQs.
+ *
+ * Exposed to userspace via a fixed mmap pgoff sentinel
+ * (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET) handled in amdgpu_drm_mmap()'s
+ * dispatcher — same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL.  No
+ * drm_gem_object, no GEM handle, no GEM consumer can ever look this
+ * up via drm_gem_object_lookup() / gem_to_amdgpu_bo().
+ */
+
+struct amdgpu_sdma_userq_db {
+	resource_size_t		phys_base;	/* BAR phys addr of window start */
+	u32			size;		/* window size in bytes */
+};
+
+/*
+ * Fake mmap offsets returned to userspace for amdgpu private mmap regions.
+ *
+ * The high bits (>= bit 56) form a "type" field that amdgpu_drm_mmap()
+ * uses to dispatch to a private handler instead of drm_gem_mmap().  Lives
+ * well above DRM's vma_offset_manager range so any sentinel can never
+ * collide with a real GEM mmap offset.  Internal-only — userspace MUST
+ * get the value via AMDGPU_INFO_USERQ_DOORBELL.  Same general pattern
+ * as KFD's KFD_MMAP_TYPE_* (kfd_priv.h).
+ */
+#define AMDGPU_USERQ_MMAP_TYPE_MASK		(0xFULL << 56)
+#define AMDGPU_USERQ_MMAP_TYPE_SDMA_DOORBELL	(0x1ULL << 56)
+#define AMDGPU_USERQ_DOORBELL_MMAP_OFFSET	AMDGPU_USERQ_MMAP_TYPE_SDMA_DOORBELL
+
 int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 			     bool caller_handles_kernel_queues);
 
@@ -205,6 +255,17 @@ int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
 				      struct amdgpu_iv_entry *entry);
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, u32 instance,
 			       bool duplicate);
+struct file;
+struct vm_area_struct;
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev);
+void amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev);
+int amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32 *out_slot);
+void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device *adev, u32 slot);
+int amdgpu_sdma_userq_doorbell_get_mmap(struct amdgpu_device *adev,
+					u64 *mmap_offset, u64 *mmap_size);
+bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned long pgoff);
+int  amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+				     struct vm_area_struct *vma);
 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
         bool duplicate);
 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
#12 lore raw
In reply to Jesse Zhang
Quoted 17 lines · Jesse Zhang

Add a per-device qword-slot pool covering the firmware-managed NBIO SDMA decode window (BAR dwords [sdma_engine[0], sdma_engine[0] + sdma_doorbell_range * num_instances)) — the only range whose writes are routed to the SDMA back-end. Kernel SDMA ring slots are pre-masked at init.

Exposed to userspace via a fixed mmap pgoff sentinel (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET, bit 56 of the offset, well above DRM's vma_offset_manager range so it can never collide with a real GEM mmap offset). amdgpu_sdma_userq_doorbell_mmap() does io_remap_pfn_range() straight into the SDMA decode window. The pgoff-sentinel dispatcher in amdgpu_drm_mmap() and the AMDGPU_INFO_USERQ_DOORBELL ioctl that hands the offset to userspace land in subsequent patches. Same general pattern as KFD's KFD_MMAP_TYPE_* .

Quoted 2 lines

Slots are allocated/freed via amdgpu_sdma_userq_doorbell_alloc/free. The init/fini hooks land in the per-engine sdma6/sdma7 patches.

This shouldn't be necessary. The doorbell offsets are relative to each 4K page in the doorbell aperture.

Alex

Quoted 270 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 154 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |  61 +++++++++
 2 files changed, 215 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +154 -0
index 321310ba2c08..be7e9de0630e 100644
@@ -22,6 +22,8 @@
  */
 
 #include <linux/firmware.h>
+#include <linux/io.h>
+#include <linux/mm.h>
 #include "amdgpu.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
@@ -200,6 +202,158 @@ void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
               sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
 }
 
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev)
+{
+       struct amdgpu_sdma_userq_db *db;
+       u32 base_dw, size_dw, nslots, ring_dw;
+       int i;
+
+       if (!adev->userq_funcs[AMDGPU_HW_IP_DMA])
+               return 0;
+
+       base_dw = adev->doorbell_index.sdma_engine[0] << 1;
+       size_dw = adev->doorbell_index.sdma_doorbell_range *
+                 adev->sdma.num_instances;
+       nslots  = size_dw / 2;  /* qword slots */
+       if (!nslots)
+               return 0;
+
+       db = kzalloc(sizeof(*db), GFP_KERNEL);
+       if (!db)
+               return -ENOMEM;
+
+       db->phys_base = adev->doorbell.base +
+                       (resource_size_t)base_dw * sizeof(u32);
+       db->size      = size_dw * sizeof(u32);
+
+       mutex_init(&adev->sdma.userq_db_mutex);
+       adev->sdma.userq_db_bitmap = bitmap_zalloc(nslots, GFP_KERNEL);
+       if (!adev->sdma.userq_db_bitmap) {
+               kfree(db);
+               return -ENOMEM;
+       }
+
+       adev->sdma.userq_db        = db;
+       adev->sdma.userq_db_nslots = nslots;
+
+       /*
+        * Mask out the qword slots used by the kernel SDMA rings
+        * (sdma_engine[i] << 1 in absolute BAR dwords ⇒ qword slot
+        * (sdma_engine[i] - sdma_engine[0]) within this window).
+        */
+       for (i = 0; i < adev->sdma.num_instances; i++) {
+               ring_dw = adev->doorbell_index.sdma_engine[i] << 1;
+               if (ring_dw >= base_dw && ring_dw < base_dw + size_dw)
+                       set_bit((ring_dw - base_dw) / 2,
+                               adev->sdma.userq_db_bitmap);
+       }
+
+       dev_info(adev->dev,
+                "SDMA UMQ doorbell pool: %u qword slots in BAR dword [%u, %u)\n",
+                nslots, base_dw, base_dw + size_dw);
+       return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev)
+{
+       if (!adev->sdma.userq_db)
+               return;
+       bitmap_free(adev->sdma.userq_db_bitmap);
+       adev->sdma.userq_db_bitmap = NULL;
+       adev->sdma.userq_db_nslots = 0;
+       kfree(adev->sdma.userq_db);
+       adev->sdma.userq_db = NULL;
+}
+
+/*
+ * Allocate one qword doorbell slot.  On success, *out_slot receives the
+ * slot id (also the qword index inside the userspace mmap of the window)
+ * which the caller passes back to free.
+ */
+int amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32 *out_slot)
+{
+       u32 slot;
+
+       if (!adev->sdma.userq_db || !adev->sdma.userq_db_nslots)
+               return -ENODEV;
+
+       mutex_lock(&adev->sdma.userq_db_mutex);
+       slot = find_first_zero_bit(adev->sdma.userq_db_bitmap,
+                                  adev->sdma.userq_db_nslots);
+       if (slot >= adev->sdma.userq_db_nslots) {
+               mutex_unlock(&adev->sdma.userq_db_mutex);
+               return -ENOSPC;
+       }
+       set_bit(slot, adev->sdma.userq_db_bitmap);
+       mutex_unlock(&adev->sdma.userq_db_mutex);
+
+       *out_slot = slot;
+       return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device *adev, u32 slot)
+{
+       if (!adev->sdma.userq_db)
+               return;
+       if (slot >= adev->sdma.userq_db_nslots)
+               return;
+       mutex_lock(&adev->sdma.userq_db_mutex);
+       clear_bit(slot, adev->sdma.userq_db_bitmap);
+       mutex_unlock(&adev->sdma.userq_db_mutex);
+}
+
+/*
+ * Report the fake mmap offset/size for the SDMA UMQ doorbell window.
+ *
+ * No drm_gem_object, no GEM handle.  Userspace passes the returned offset
+ * directly to mmap(drm_fd, ...); amdgpu_drm_mmap() recognises the
+ * sentinel pgoff and routes the call to amdgpu_sdma_userq_doorbell_mmap()
+ * below, which io_remap_pfn_range()s the routable SDMA doorbell BAR window.
+ *
+ * Same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL — the doorbell window
+ * is unreachable from any drm_gem_object_lookup() / gem_to_amdgpu_bo()
+ * code path because it never enters the GEM object space at all.
+ */
+int amdgpu_sdma_userq_doorbell_get_mmap(struct amdgpu_device *adev,
+                                       u64 *mmap_offset, u64 *mmap_size)
+{
+       if (!adev->sdma.userq_db)
+               return -ENODEV;
+
+       *mmap_offset = AMDGPU_USERQ_DOORBELL_MMAP_OFFSET;
+       *mmap_size   = adev->sdma.userq_db->size;
+       return 0;
+}
+
+bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned long pgoff)
+{
+       return pgoff == (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET >> PAGE_SHIFT);
+}
+
+int amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+                                   struct vm_area_struct *vma)
+{
+       struct drm_file *file_priv = filp->private_data;
+       struct drm_device *dev = file_priv->minor->dev;
+       struct amdgpu_device *adev = drm_to_adev(dev);
+       struct amdgpu_sdma_userq_db *db = adev->sdma.userq_db;
+
+       if (!db)
+               return -ENODEV;
+
+       if (vma->vm_end - vma->vm_start > round_up(db->size, PAGE_SIZE))
+               return -EINVAL;
+
+       vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
+                         VM_DONTDUMP | VM_PFNMAP);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+       return io_remap_pfn_range(vma, vma->vm_start,
+                                 db->phys_base >> PAGE_SHIFT,
+                                 vma->vm_end - vma->vm_start,
+                                 vma->vm_page_prot);
+}
+
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
                               u32 instance, bool duplicate)
 {
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +61 -1
index 2bf365609775..a308d5e6ee54 100644
@@ -146,6 +146,20 @@ struct amdgpu_sdma {
        bool                    disable_uq;
        void (*get_csa_info)(struct amdgpu_device *adev,
                             struct amdgpu_sdma_csa_info *csa_info);
+
+       /*
+        * SDMA usermode-queue doorbell pool.  The window covers
+        * BAR dwords [sdma_engine[0], sdma_engine[0] +
+        * sdma_doorbell_range * num_instances) — the only range that NBIO
+        * routes to the SDMA back-end.  Each bit in the bitmap represents
+        * one qword slot; kernel SDMA ring slots are pre-masked at init.
+        * Exposed to userspace via a fixed mmap pgoff sentinel handled by
+        * amdgpu_drm_mmap() — see AMDGPU_USERQ_DOORBELL_MMAP_OFFSET below.
+        */
+       struct amdgpu_sdma_userq_db *userq_db;
+       struct mutex            userq_db_mutex;
+       unsigned long           *userq_db_bitmap;
+       u32                     userq_db_nslots;        /* qword slots */
 };
 
 /*
@@ -185,6 +199,42 @@ struct amdgpu_buffer_funcs {
                                 uint32_t byte_count);
 };
 
+/*
+ * SDMA usermode-queue doorbell pool.
+ *
+ * The pool re-uses qword doorbell slots inside the firmware-managed NBIO
+ * SDMA decode window (BAR dwords [sdma_engine[0],
+ * sdma_engine[0] + sdma_doorbell_range * num_instances)) — that range is
+ * the only one whose writes are routed to the SDMA back-end.  The kernel
+ * SDMA ring slots are pre-marked so they keep working alongside any
+ * number of SDMA UMQs.
+ *
+ * Exposed to userspace via a fixed mmap pgoff sentinel
+ * (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET) handled in amdgpu_drm_mmap()'s
+ * dispatcher — same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL.  No
+ * drm_gem_object, no GEM handle, no GEM consumer can ever look this
+ * up via drm_gem_object_lookup() / gem_to_amdgpu_bo().
+ */
+
+struct amdgpu_sdma_userq_db {
+       resource_size_t         phys_base;      /* BAR phys addr of window start */
+       u32                     size;           /* window size in bytes */
+};
+
+/*
+ * Fake mmap offsets returned to userspace for amdgpu private mmap regions.
+ *
+ * The high bits (>= bit 56) form a "type" field that amdgpu_drm_mmap()
+ * uses to dispatch to a private handler instead of drm_gem_mmap().  Lives
+ * well above DRM's vma_offset_manager range so any sentinel can never
+ * collide with a real GEM mmap offset.  Internal-only — userspace MUST
+ * get the value via AMDGPU_INFO_USERQ_DOORBELL.  Same general pattern
+ * as KFD's KFD_MMAP_TYPE_* (kfd_priv.h).
+ */
+#define AMDGPU_USERQ_MMAP_TYPE_MASK            (0xFULL << 56)
+#define AMDGPU_USERQ_MMAP_TYPE_SDMA_DOORBELL   (0x1ULL << 56)
+#define AMDGPU_USERQ_DOORBELL_MMAP_OFFSET      AMDGPU_USERQ_MMAP_TYPE_SDMA_DOORBELL
+
 int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
                             bool caller_handles_kernel_queues);
 
@@ -205,6 +255,17 @@ int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
                                      struct amdgpu_iv_entry *entry);
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, u32 instance,
                               bool duplicate);
+struct file;
+struct vm_area_struct;
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev);
+void amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev);
+int amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32 *out_slot);
+void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device *adev, u32 slot);
+int amdgpu_sdma_userq_doorbell_get_mmap(struct amdgpu_device *adev,
+                                       u64 *mmap_offset, u64 *mmap_size);
+bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned long pgoff);
+int  amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+                                    struct vm_area_struct *vma);
 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
         bool duplicate);
 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
--

2.49.0

#13 lore raw

AMD General

Hi Alex,

Quoted 29 lines

-----Original Message-----
From: Alex Deucher <[email protected]>
Sent: Wednesday, April 29, 2026 12:00 AM
To: Zhang, Jesse(Jie) <[email protected]>
Cc: [email protected]; Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Liang, Prike <[email protected]>
Subject: Re: [PATCH v3 01/11] drm/amdgpu/sdma: add SDMA usermode-queue doorbell pool infra

Quoted 9 lines · Jesse Zhang

Add a per-device qword-slot pool covering the firmware-managed NBIO SDMA decode window (BAR dwords [sdma_engine[0], sdma_engine[0] + sdma_doorbell_range * num_instances)) — the only range whose writes are routed to the SDMA back-end. Kernel SDMA ring slots are pre-masked at init.

Exposed to userspace via a fixed mmap pgoff sentinel (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET, bit 56 of the offset, well

above

Quoted 8 lines

DRM's vma_offset_manager range so it can never collide with a real GEM mmap offset). amdgpu_sdma_userq_doorbell_mmap() does io_remap_pfn_range() straight into the SDMA decode window. The pgoff-sentinel dispatcher in amdgpu_drm_mmap() and the AMDGPU_INFO_USERQ_DOORBELL ioctl that hands the offset to userspace land in subsequent patches. Same general pattern as KFD's KFD_MMAP_TYPE_* .

Quoted 5 lines
Quoted 2 lines

Slots are allocated/freed via amdgpu_sdma_userq_doorbell_alloc/free. The init/fini hooks land in the per-engine sdma6/sdma7 patches.

This shouldn't be necessary. The doorbell offsets are relative to each 4K page in the doorbell aperture.

The doorbell offset being relative to each 4K page is fine for addressing once userspace maps the right doorbell pages. The bug we saw was that the SDMA UMQ path used amdgpu_userq_get_doorbell_index,
and the absolute doorbell dword fell outside the NBIO SDMA decode window, so writes never reached SDMA except via polling.

Thanks
Jesse

Quoted 295 lines

Alex

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c void amdgpu_sdma_destroy_inst_ctx(struct

Quoted 25 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c | 154
+++++++++++++++++++++++  drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h |
61 +++++++++
 2 files changed, 215 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +2 -0
index 321310ba2c08..be7e9de0630e 100644
@@ -22,6 +22,8 @@
  */
 
 #include <linux/firmware.h>
+#include <linux/io.h>
+#include <linux/mm.h>
 #include "amdgpu.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
@@ -200,6 +202,158 @@ void amdgpu_sdma_destroy_inst_ctx(struct

amdgpu_device *adev,

Quoted 40 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +37 -0
               sizeof(struct amdgpu_sdma_instance) *
 AMDGPU_MAX_SDMA_INSTANCES);  }
 
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev) {
+       struct amdgpu_sdma_userq_db *db;
+       u32 base_dw, size_dw, nslots, ring_dw;
+       int i;
+
+       if (!adev->userq_funcs[AMDGPU_HW_IP_DMA])
+               return 0;
+
+       base_dw = adev->doorbell_index.sdma_engine[0] << 1;
+       size_dw = adev->doorbell_index.sdma_doorbell_range *
+                 adev->sdma.num_instances;
+       nslots  = size_dw / 2;  /* qword slots */
+       if (!nslots)
+               return 0;
+
+       db = kzalloc(sizeof(*db), GFP_KERNEL);
+       if (!db)
+               return -ENOMEM;
+
+       db->phys_base = adev->doorbell.base +
+                       (resource_size_t)base_dw * sizeof(u32);
+       db->size      = size_dw * sizeof(u32);
+
+       mutex_init(&adev->sdma.userq_db_mutex);
+       adev->sdma.userq_db_bitmap = bitmap_zalloc(nslots, GFP_KERNEL);
+       if (!adev->sdma.userq_db_bitmap) {
+               kfree(db);
+               return -ENOMEM;
+       }
+
+       adev->sdma.userq_db        = db;
+       adev->sdma.userq_db_nslots = nslots;
+
+       /*
+        * Mask out the qword slots used by the kernel SDMA rings
+        * (sdma_engine[i] << 1 in absolute BAR dwords ⇒ qword slot
+        * (sdma_engine[i] - sdma_engine[0]) within this window).

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
+        */
+       for (i = 0; i < adev->sdma.num_instances; i++) {
+               ring_dw = adev->doorbell_index.sdma_engine[i] << 1;
+               if (ring_dw >= base_dw && ring_dw < base_dw + size_dw)
+                       set_bit((ring_dw - base_dw) / 2,
+                               adev->sdma.userq_db_bitmap);
+       }
+
+       dev_info(adev->dev,
+                "SDMA UMQ doorbell pool: %u qword slots in BAR dword

[%u, %u)\n",

Quoted 52 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +52 -0
+                nslots, base_dw, base_dw + size_dw);
+       return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev) {
+       if (!adev->sdma.userq_db)
+               return;
+       bitmap_free(adev->sdma.userq_db_bitmap);
+       adev->sdma.userq_db_bitmap = NULL;
+       adev->sdma.userq_db_nslots = 0;
+       kfree(adev->sdma.userq_db);
+       adev->sdma.userq_db = NULL;
+}
+
+/*
+ * Allocate one qword doorbell slot.  On success, *out_slot receives
+the
+ * slot id (also the qword index inside the userspace mmap of the
+window)
+ * which the caller passes back to free.
+ */
+int amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32
+*out_slot) {
+       u32 slot;
+
+       if (!adev->sdma.userq_db || !adev->sdma.userq_db_nslots)
+               return -ENODEV;
+
+       mutex_lock(&adev->sdma.userq_db_mutex);
+       slot = find_first_zero_bit(adev->sdma.userq_db_bitmap,
+                                  adev->sdma.userq_db_nslots);
+       if (slot >= adev->sdma.userq_db_nslots) {
+               mutex_unlock(&adev->sdma.userq_db_mutex);
+               return -ENOSPC;
+       }
+       set_bit(slot, adev->sdma.userq_db_bitmap);
+       mutex_unlock(&adev->sdma.userq_db_mutex);
+
+       *out_slot = slot;
+       return 0;
+}
+
+void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device *adev, u32
+slot) {
+       if (!adev->sdma.userq_db)
+               return;
+       if (slot >= adev->sdma.userq_db_nslots)
+               return;
+       mutex_lock(&adev->sdma.userq_db_mutex);
+       clear_bit(slot, adev->sdma.userq_db_bitmap);
+       mutex_unlock(&adev->sdma.userq_db_mutex);
+}

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
+
+/*
+ * Report the fake mmap offset/size for the SDMA UMQ doorbell window.
+ *
+ * No drm_gem_object, no GEM handle.  Userspace passes the returned
+offset
+ * directly to mmap(drm_fd, ...); amdgpu_drm_mmap() recognises the
+ * sentinel pgoff and routes the call to
+amdgpu_sdma_userq_doorbell_mmap()
+ * below, which io_remap_pfn_range()s the routable SDMA doorbell BAR

window.

Quoted 26 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +26 -0
+ *
+ * Same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL — the doorbell
+window
+ * is unreachable from any drm_gem_object_lookup() /
+gem_to_amdgpu_bo()
+ * code path because it never enters the GEM object space at all.
+ */
+int amdgpu_sdma_userq_doorbell_get_mmap(struct amdgpu_device *adev,
+                                       u64 *mmap_offset, u64
+*mmap_size) {
+       if (!adev->sdma.userq_db)
+               return -ENODEV;
+
+       *mmap_offset = AMDGPU_USERQ_DOORBELL_MMAP_OFFSET;
+       *mmap_size   = adev->sdma.userq_db->size;
+       return 0;
+}
+
+bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned long pgoff) {
+       return pgoff == (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET >>
+PAGE_SHIFT); }
+
+int amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+                                   struct vm_area_struct *vma) {
+       struct drm_file *file_priv = filp->private_data;
+       struct drm_device *dev = file_priv->minor->dev;

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +10 -0
+       struct amdgpu_device *adev = drm_to_adev(dev);
+       struct amdgpu_sdma_userq_db *db = adev->sdma.userq_db;
+
+       if (!db)
+               return -ENODEV;
+
+       if (vma->vm_end - vma->vm_start > round_up(db->size, PAGE_SIZE))
+               return -EINVAL;
+
+       vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND |

VM_NORESERVE |

Quoted 18 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +8 -1
+                         VM_DONTDUMP | VM_PFNMAP);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+       return io_remap_pfn_range(vma, vma->vm_start,
+                                 db->phys_base >> PAGE_SHIFT,
+                                 vma->vm_end - vma->vm_start,
+                                 vma->vm_page_prot); }
+
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
                               u32 instance, bool duplicate)  { diff
--git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h

b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 2bf365609775..a308d5e6ee54 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +0 -0
@@ -146,6 +146,20 @@ struct amdgpu_sdma {
        bool                    disable_uq;
        void (*get_csa_info)(struct amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h struct amdgpu_sdma { · line 157

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +9 -0
                             struct amdgpu_sdma_csa_info *csa_info);
+
+       /*
+        * SDMA usermode-queue doorbell pool.  The window covers
+        * BAR dwords [sdma_engine[0], sdma_engine[0] +
+        * sdma_doorbell_range * num_instances) — the only range that NBIO
+        * routes to the SDMA back-end.  Each bit in the bitmap represents
+        * one qword slot; kernel SDMA ring slots are pre-masked at init.
+        * Exposed to userspace via a fixed mmap pgoff sentinel handled by
+        * amdgpu_drm_mmap() — see

AMDGPU_USERQ_DOORBELL_MMAP_OFFSET below.

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +10 -0
+        */
+       struct amdgpu_sdma_userq_db *userq_db;
+       struct mutex            userq_db_mutex;
+       unsigned long           *userq_db_bitmap;
+       u32                     userq_db_nslots;        /* qword slots */
 };
 
 /*
@@ -185,6 +199,42 @@ struct amdgpu_buffer_funcs {
                                 uint32_t byte_count);  };
 
+/*
+ * SDMA usermode-queue doorbell pool.
+ *
+ * The pool re-uses qword doorbell slots inside the firmware-managed
+NBIO

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h struct amdgpu_buffer_funcs { · line 215

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +10 -0
+ * SDMA decode window (BAR dwords [sdma_engine[0],
+ * sdma_engine[0] + sdma_doorbell_range * num_instances)) — that
+range is
+ * the only one whose writes are routed to the SDMA back-end.  The
+kernel
+ * SDMA ring slots are pre-marked so they keep working alongside any
+ * number of SDMA UMQs.
+ *
+ * Exposed to userspace via a fixed mmap pgoff sentinel
+ * (AMDGPU_USERQ_DOORBELL_MMAP_OFFSET) handled in

amdgpu_drm_mmap()'s

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h struct amdgpu_buffer_funcs { · line 216

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +1 -0
+ * dispatcher — same pattern KFD uses for KFD_MMAP_TYPE_DOORBELL.

No

Quoted 16 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +16 -0
+ * drm_gem_object, no GEM handle, no GEM consumer can ever look this
+ * up via drm_gem_object_lookup() / gem_to_amdgpu_bo().
+ */
+
+struct amdgpu_sdma_userq_db {
+       resource_size_t         phys_base;      /* BAR phys addr of window start */
+       u32                     size;           /* window size in bytes */
+};
+
+/*
+ * Fake mmap offsets returned to userspace for amdgpu private mmap regions.
+ *
+ * The high bits (>= bit 56) form a "type" field that
+amdgpu_drm_mmap()
+ * uses to dispatch to a private handler instead of drm_gem_mmap().
+Lives

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h struct amdgpu_buffer_funcs {

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +10 -0
+ * well above DRM's vma_offset_manager range so any sentinel can
+never
+ * collide with a real GEM mmap offset.  Internal-only — userspace
+MUST
+ * get the value via AMDGPU_INFO_USERQ_DOORBELL.  Same general
+pattern
+ * as KFD's KFD_MMAP_TYPE_* (kfd_priv.h).
+ */
+#define AMDGPU_USERQ_MMAP_TYPE_MASK            (0xFULL << 56)
+#define AMDGPU_USERQ_MMAP_TYPE_SDMA_DOORBELL   (0x1ULL <<

56)

Quoted 1 line

+#define AMDGPU_USERQ_DOORBELL_MMAP_OFFSET

AMDGPU_USERQ_MMAP_TYPE_SDMA_DOORBELL

Quoted 2 lines

+
int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t

instance_id,

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h int amdgpu_sdma_process_ecc_irq(struct

Quoted 3 lines
                             bool caller_handles_kernel_queues);
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +0 -0
@@ -205,6 +255,17 @@ int amdgpu_sdma_process_ecc_irq(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +7 -0
                                      struct amdgpu_iv_entry *entry);
 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, u32 instance,
                               bool duplicate);
+struct file;
+struct vm_area_struct;
+int amdgpu_sdma_userq_doorbell_init(struct amdgpu_device *adev); void
+amdgpu_sdma_userq_doorbell_fini(struct amdgpu_device *adev); int
+amdgpu_sdma_userq_doorbell_alloc(struct amdgpu_device *adev, u32
+*out_slot); void amdgpu_sdma_userq_doorbell_free(struct amdgpu_device
+*adev, u32 slot); int amdgpu_sdma_userq_doorbell_get_mmap(struct

amdgpu_device *adev,

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +4 -1
+                                       u64 *mmap_offset, u64
+*mmap_size); bool amdgpu_sdma_userq_doorbell_is_our_pgoff(unsigned
+long pgoff); int  amdgpu_sdma_userq_doorbell_mmap(struct file *filp,
+                                    struct vm_area_struct *vma);
 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
         bool duplicate);
 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev);
--

2.49.0