Skip to contentSkip to conversation

drm/amdgpu: add UAPI to create user queue gangs

PATCH In review Alex Deucher 4 participants 14 messages Apr 25, 2025 – Apr 28, 2025

Subsystems (3)
lore
Share & tools
lore

Alex Deucher

author maintainer
#0 lore raw

PATCH 1/8 drm/amdgpu: add UAPI to create user queue gangs

Queues in a gang will schedule together.

---
 include/uapi/drm/amdgpu_drm.h | 11 +++++++++++
 1 file changed, 11 insertions(+)
include/uapi/drm/amdgpu_drm.h +11 -0
index 56f052a10ff38..83414563779fb 100644
@@ -329,6 +329,7 @@ union drm_amdgpu_ctx {
 #define AMDGPU_USERQ_OP_CREATE	1
 #define AMDGPU_USERQ_OP_FREE	2
 #define AMDGPU_USERQ_OP_QUERY_STATUS	3
+#define AMDGPU_USERQ_OP_CREATE_GANG	4
 
 /* queue priority levels */
 /* low < normal low < normal high < high */
@@ -417,6 +418,15 @@ struct drm_amdgpu_userq_in {
 	__u64 mqd_size;
 };
 
+struct drm_amdgpu_userq_in_create_gang {
+	/** AMDGPU_USERQ_OP_* */
+	__u32	op;
+	__u32	pad;
+	/** Queue ids passed for operation USERQ_OP_CREATE_GANG */
+	__u32	primary_queue_id;
+	__u32	secondary_queue_id;
+};
+
 /* The structure to carry output of userqueue ops */
 struct drm_amdgpu_userq_out {
 	/**
@@ -436,6 +446,7 @@ struct drm_amdgpu_userq_out_query_state {
 
 union drm_amdgpu_userq {
 	struct drm_amdgpu_userq_in in;
+	struct drm_amdgpu_userq_in_create_gang in_cg;
 	struct drm_amdgpu_userq_out out;
 	struct drm_amdgpu_userq_out_query_state out_qs;
 };
Signature · 1 line
2.49.0
author maintainer
#1 lore raw

PATCH 2/8 drm/amdgpu/mes11: add API for gang submit

Add API for setting up a gang. Gangs schedule together.

---
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c        |  2 +-
 drivers/gpu/drm/amd/include/mes_v11_api_def.h | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +1 -1
index c9eba537de094..8a31bd81bcb9d 100644
@@ -125,7 +125,7 @@ static const char *mes_v11_0_opcodes[] = {
 	"UPDATE_ROOT_PAGE_TABLE",
 	"AMD_LOG",
 	"unused",
-	"unused",
+	"SET_GANG_SUBMIT",
 	"SET_HW_RSRC_1",
 };
 
drivers/gpu/drm/amd/include/mes_v11_api_def.h +18 -0
index 15680c3f49704..d1933bc3669d7 100644
@@ -64,6 +64,7 @@ enum MES_SCH_API_OPCODE {
 	MES_SCH_API_MISC			= 14,
 	MES_SCH_API_UPDATE_ROOT_PAGE_TABLE      = 15,
 	MES_SCH_API_AMD_LOG                     = 16,
+	MES_SCH_API_SET_GANG_SUBMIT             = 18,
 	MES_SCH_API_SET_HW_RSRC_1               = 19,
 	MES_SCH_API_MAX				= 0xFF
 };
@@ -633,6 +634,13 @@ struct SET_SHADER_DEBUGGER {
 	uint32_t trap_en;
 };
 
+struct SET_GANG_SUBMIT {
+	uint64_t gang_context_addr;
+	uint64_t slave_gang_context_addr;
+	uint32_t gang_context_array_index;
+	uint32_t slave_gang_context_array_index;
+};
+
 enum MESAPI_MISC__CHANGE_CONFIG_OPTION {
 	MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS = 0,
 	MESAPI_MISC__CHANGE_CONFIG_OPTION_ENABLE_HWS_LOGGING_BUFFER = 1,
@@ -703,5 +711,15 @@ union MESAPI_AMD_LOG {
 	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
 };
 
+union MESAPI__SET_GANG_SUBMIT {
+	struct {
+		union MES_API_HEADER    header;
+		struct MES_API_STATUS   api_status;
+		struct SET_GANG_SUBMIT  set_gang_submit;
+	};
+
+	uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS];
+};
+
 #pragma pack(pop)
 #endif
Signature · 1 line
2.49.0
author maintainer
#2 lore raw

PATCH 3/8 drm/amdgpu/mes: add callback for gang submit

In reply to Alex Deucher

Add function pointer for gang submit setup.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 8 ++++++++
 1 file changed, 8 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +8 -0
index e98b0d892a593..bd07b02521414 100644
@@ -279,6 +279,11 @@ struct mes_reset_queue_input {
 	bool                               legacy_gfx;
 };
 
+struct mes_set_gang_submit_input {
+	uint64_t	primary_gang_context_addr;
+	uint64_t	secondary_gang_context_addr;
+};
+
 enum mes_misc_opcode {
 	MES_MISC_OP_WRITE_REG,
 	MES_MISC_OP_READ_REG,
@@ -366,6 +371,9 @@ struct amdgpu_mes_funcs {
 
 	int (*reset_hw_queue)(struct amdgpu_mes *mes,
 			      struct mes_reset_queue_input *input);
+
+	int (*set_gang_submit)(struct amdgpu_mes *mes,
+			       struct mes_set_gang_submit_input *input);
 };
 
 #define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev))
Signature · 1 line
2.49.0
author maintainer
#3 lore raw

PATCH 4/8 drm/amdgpu/mes11: add support for setting gang submit

In reply to Alex Deucher

Enable a primary and secondary queue that schedule together.

---
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +22 -0
index 8a31bd81bcb9d..df4aa6ad6a29e 100644
@@ -783,6 +783,27 @@ static int mes_v11_0_reset_hw_queue(struct amdgpu_mes *mes,
 			offsetof(union MESAPI__RESET, api_status));
 }
 
+static int mes_v11_0_set_gang_submit(struct amdgpu_mes *mes,
+				     struct mes_set_gang_submit_input *input)
+{
+	union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt;
+
+	memset(&mes_gang_submit_pkt, 0, sizeof(mes_gang_submit_pkt));
+
+	mes_gang_submit_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_gang_submit_pkt.header.opcode = MES_SCH_API_SET_GANG_SUBMIT;
+	mes_gang_submit_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_gang_submit_pkt.set_gang_submit.gang_context_addr =
+		input->primary_gang_context_addr;
+	mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr =
+		input->secondary_gang_context_addr;
+
+	return mes_v11_0_submit_pkt_and_poll_completion(mes,
+			&mes_gang_submit_pkt, sizeof(mes_gang_submit_pkt),
+			offsetof(union MESAPI__RESET, api_status));
+}
+
 static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
 	.add_hw_queue = mes_v11_0_add_hw_queue,
 	.remove_hw_queue = mes_v11_0_remove_hw_queue,
@@ -792,6 +813,7 @@ static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
 	.resume_gang = mes_v11_0_resume_gang,
 	.misc_op = mes_v11_0_misc_op,
 	.reset_hw_queue = mes_v11_0_reset_hw_queue,
+	.set_gang_submit = mes_v11_0_set_gang_submit,
 };
 
 static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
Signature · 1 line
2.49.0
#4 lore raw

RE: [PATCH 4/8] drm/amdgpu/mes11: add support for setting gang submit

[Public]

Quoted 34 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 4/8] drm/amdgpu/mes11: add support for setting gang submit

Enable a primary and secondary queue that schedule together.

---
 drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +9 -0
index 8a31bd81bcb9d..df4aa6ad6a29e 100644
@@ -783,6 +783,27 @@ static int mes_v11_0_reset_hw_queue(struct
 amdgpu_mes *mes,
                      offsetof(union MESAPI__RESET, api_status));  }
 
+static int mes_v11_0_set_gang_submit(struct amdgpu_mes *mes,
+                                  struct mes_set_gang_submit_input *input) {
+     union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt;
+
+     memset(&mes_gang_submit_pkt, 0, sizeof(mes_gang_submit_pkt));
+
+     mes_gang_submit_pkt.header.type = MES_API_TYPE_SCHEDULER;
+     mes_gang_submit_pkt.header.opcode =
 MES_SCH_API_SET_GANG_SUBMIT;
+     mes_gang_submit_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;

drivers/gpu/drm/amd/amdgpu/mes_v11_0.c static int mes_v11_0_reset_hw_queue(struct · line 805

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +10 -0
+
+     mes_gang_submit_pkt.set_gang_submit.gang_context_addr =
+             input->primary_gang_context_addr;
+     mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr =
+             input->secondary_gang_context_addr;
+
+     return mes_v11_0_submit_pkt_and_poll_completion(mes,
+                     &mes_gang_submit_pkt, sizeof(mes_gang_submit_pkt),
+                     offsetof(union MESAPI__RESET, api_status)); }
+

Please use the MESAPI__SET_GANG_SUBMIT type to get the API status. With that fix, the patch is Reviewed-by: Prike Liang <[email protected]>

Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +0 -0
 static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
      .add_hw_queue = mes_v11_0_add_hw_queue,
      .remove_hw_queue = mes_v11_0_remove_hw_queue, @@ -792,6 +813,7
@@ static const struct amdgpu_mes_funcs mes_v11_0_funcs = {
      .resume_gang = mes_v11_0_resume_gang,
      .misc_op = mes_v11_0_misc_op,
      .reset_hw_queue = mes_v11_0_reset_hw_queue,

+ .set_gang_submit = mes_v11_0_set_gang_submit,
};

static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
--
2.49.0

author maintainer
#5 lore raw

PATCH 5/8 drm/amdgpu/mes12: add support for setting gang submit

In reply to Alex Deucher

Enable a primary and secondary queue that schedule together.

---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 28 ++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +28 -0
index f9f2fbc0a7166..e5eb4b923f24e 100644
@@ -879,6 +879,33 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
 			offsetof(union MESAPI__RESET, api_status));
 }
 
+static int mes_v12_0_set_gang_submit(struct amdgpu_mes *mes,
+				     struct mes_set_gang_submit_input *input)
+{
+	union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt;
+	int pipe;
+
+	memset(&mes_gang_submit_pkt, 0, sizeof(mes_gang_submit_pkt));
+
+	mes_gang_submit_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_gang_submit_pkt.header.opcode = MES_SCH_API_SET_GANG_SUBMIT;
+	mes_gang_submit_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_gang_submit_pkt.set_gang_submit.gang_context_addr =
+		input->primary_gang_context_addr;
+	mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr =
+		input->secondary_gang_context_addr;
+
+	if (mes->adev->enable_uni_mes)
+		pipe = AMDGPU_MES_KIQ_PIPE;
+	else
+		pipe = AMDGPU_MES_SCHED_PIPE;
+
+	return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
+			&mes_gang_submit_pkt, sizeof(mes_gang_submit_pkt),
+			offsetof(union MESAPI__RESET, api_status));
+}
+
 static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
 	.add_hw_queue = mes_v12_0_add_hw_queue,
 	.remove_hw_queue = mes_v12_0_remove_hw_queue,
@@ -888,6 +915,7 @@ static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
 	.resume_gang = mes_v12_0_resume_gang,
 	.misc_op = mes_v12_0_misc_op,
 	.reset_hw_queue = mes_v12_0_reset_hw_queue,
+	.set_gang_submit = mes_v12_0_set_gang_submit,
 };
 
 static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
Signature · 1 line
2.49.0
#6 lore raw

RE: [PATCH 5/8] drm/amdgpu/mes12: add support for setting gang submit

[Public]

Quoted 41 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 5/8] drm/amdgpu/mes12: add support for setting gang submit

Enable a primary and secondary queue that schedule together.

---
 drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 28
++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +15 -0
index f9f2fbc0a7166..e5eb4b923f24e 100644
@@ -879,6 +879,33 @@ static int mes_v12_0_reset_hw_queue(struct
 amdgpu_mes *mes,
                      offsetof(union MESAPI__RESET, api_status));  }
 
+static int mes_v12_0_set_gang_submit(struct amdgpu_mes *mes,
+                                  struct mes_set_gang_submit_input *input) {
+     union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt;
+     int pipe;
+
+     memset(&mes_gang_submit_pkt, 0, sizeof(mes_gang_submit_pkt));
+
+     mes_gang_submit_pkt.header.type = MES_API_TYPE_SCHEDULER;
+     mes_gang_submit_pkt.header.opcode =
 MES_SCH_API_SET_GANG_SUBMIT;
+     mes_gang_submit_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+     mes_gang_submit_pkt.set_gang_submit.gang_context_addr =
+             input->primary_gang_context_addr;
+     mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr =
+             input->secondary_gang_context_addr;

drivers/gpu/drm/amd/amdgpu/mes_v12_0.c static int mes_v12_0_reset_hw_queue(struct · line 907

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +10 -0
+
+     if (mes->adev->enable_uni_mes)
+             pipe = AMDGPU_MES_KIQ_PIPE;
+     else
+             pipe = AMDGPU_MES_SCHED_PIPE;
+
+     return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe,
+                     &mes_gang_submit_pkt, sizeof(mes_gang_submit_pkt),
+                     offsetof(union MESAPI__RESET, api_status)); }
+

This also requires using the MESAPI__SET_GANG_SUBMIT type to get the API status offset.

With this update, the patch is Reviewed-by: Prike Liang <[email protected]>

Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +0 -0
 static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
      .add_hw_queue = mes_v12_0_add_hw_queue,
      .remove_hw_queue = mes_v12_0_remove_hw_queue, @@ -888,6 +915,7
@@ static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
      .resume_gang = mes_v12_0_resume_gang,
      .misc_op = mes_v12_0_misc_op,
      .reset_hw_queue = mes_v12_0_reset_hw_queue,

+ .set_gang_submit = mes_v12_0_set_gang_submit,
};

static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
--
2.49.0

author maintainer
#7 lore raw

PATCH 6/8 drm/amdgpu/userq: add callback for gang setup

In reply to Alex Deucher

Two queues can be ganged and will schedule together.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 3 +++
 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +3 -0
index 35a08ee59d893..ca11f7748d031 100644
@@ -80,6 +80,9 @@ struct amdgpu_userq_funcs {
 		   struct amdgpu_usermode_queue *queue);
 	int (*reset)(struct amdgpu_userq_mgr *uq_mgr,
 		     struct amdgpu_usermode_queue *queue);
+	int (*set_gang)(struct amdgpu_userq_mgr *uq_mgr,
+			struct amdgpu_usermode_queue *primary_queue,
+			struct amdgpu_usermode_queue *secondary_queue);
 };
 
 /* Usermode queues for gfx */
Signature · 1 line
2.49.0
author maintainer
#8 lore raw

PATCH 7/8 drm/amdgpu/userq/mes: implement set gang submit

In reply to Alex Deucher

Enable the userq callback for setting up gang submission.

---
 drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 26 ++++++++++++++++++++++
 1 file changed, 26 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +26 -0
index 650fdb68db127..98d08bda5dbc7 100644
@@ -367,10 +367,36 @@ static int mes_userq_reset(struct amdgpu_userq_mgr *uq_mgr,
 	return r;
 }
 
+static int mes_set_gang(struct amdgpu_userq_mgr *uq_mgr,
+			struct amdgpu_usermode_queue *primary_queue,
+			struct amdgpu_usermode_queue *secondary_queue)
+{
+	struct amdgpu_userq_obj *primary_ctx = &primary_queue->fw_obj;
+	struct amdgpu_userq_obj *secondary_ctx = &secondary_queue->fw_obj;
+	struct amdgpu_device *adev = uq_mgr->adev;
+	struct mes_set_gang_submit_input input;
+	int r;
+
+	memset(&input, 0x0, sizeof(struct mes_set_gang_submit_input));
+
+	input.primary_gang_context_addr =
+		primary_ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
+	input.secondary_gang_context_addr =
+		secondary_ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
+
+	amdgpu_mes_lock(&adev->mes);
+	r = adev->mes.funcs->set_gang_submit(&adev->mes, &input);
+	amdgpu_mes_unlock(&adev->mes);
+	if (r)
+		dev_err(adev->dev, "Failed to set gang submit, err (%d)\n", r);
+	return r;
+}
+
 const struct amdgpu_userq_funcs userq_mes_funcs = {
 	.mqd_create = mes_userq_mqd_create,
 	.mqd_destroy = mes_userq_mqd_destroy,
 	.unmap = mes_userq_unmap,
 	.map = mes_userq_map,
 	.reset = mes_userq_reset,
+	.set_gang = mes_set_gang,
 };
Signature · 1 line
2.49.0
author maintainer
#9 lore raw

PATCH 8/8 drm/amdgpu/userq: enable support for creating gangs

In reply to Alex Deucher

Gangs are sets of userqs that schedule together. You specify the primary and secondary queues and the scheduler will make sure they always run at the same time.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 52 +++++++++++++++++++++++
 1 file changed, 52 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +52 -0
index e56fae10400db..95e1495d9a24c 100644
@@ -561,6 +561,51 @@ amdgpu_userq_query_status(struct drm_file *filp, union drm_amdgpu_userq *args)
 	return 0;
 }
 
+static int
+amdgpu_userq_create_gang(struct drm_file *filp, union drm_amdgpu_userq *args)
+{
+	struct amdgpu_usermode_queue *primary_queue, *secondary_queue;
+	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
+	const struct amdgpu_userq_funcs *userq_funcs;
+	struct amdgpu_device *adev = uq_mgr->adev;
+	int primary_queue_id = args->in_cg.primary_queue_id;
+	int secondary_queue_id = args->in_cg.secondary_queue_id;
+	int r;
+
+	mutex_lock(&uq_mgr->userq_mutex);
+	primary_queue = amdgpu_userq_find(uq_mgr, primary_queue_id);
+	if (!primary_queue) {
+		dev_err(adev->dev, "Invalid gang primary queue id\n");
+		mutex_unlock(&uq_mgr->userq_mutex);
+		return -EINVAL;
+	}
+	if ((primary_queue->queue_type != AMDGPU_HW_IP_GFX) &&
+	    (primary_queue->queue_type != AMDGPU_HW_IP_COMPUTE)) {
+		dev_err(adev->dev, "Invalid gang primary queue type\n");
+		mutex_unlock(&uq_mgr->userq_mutex);
+		return -EINVAL;
+	}
+	secondary_queue = amdgpu_userq_find(uq_mgr, secondary_queue_id);
+	if (!secondary_queue) {
+		dev_err(adev->dev, "Invalid gang secondary queue id\n");
+		mutex_unlock(&uq_mgr->userq_mutex);
+		return -EINVAL;
+	}
+	if ((secondary_queue->queue_type != AMDGPU_HW_IP_GFX) &&
+	    (secondary_queue->queue_type != AMDGPU_HW_IP_COMPUTE)) {
+		dev_err(adev->dev, "Invalid gang secondary queue type\n");
+		mutex_unlock(&uq_mgr->userq_mutex);
+		return -EINVAL;
+	}
+
+	userq_funcs = adev->userq_funcs[primary_queue->queue_type];
+	r = userq_funcs->set_gang(uq_mgr, primary_queue, secondary_queue);
+	mutex_unlock(&uq_mgr->userq_mutex);
+
+	return r;
+}
+
 int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *filp)
 {
@@ -611,6 +656,13 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
 		if (r)
 			DRM_ERROR("Failed to query usermode queue status\n");
 		break;
+	case AMDGPU_USERQ_OP_CREATE_GANG:
+		if (args->in_cg.pad)
+			return -EINVAL;
+		r = amdgpu_userq_create_gang(filp, args);
+		if (r)
+			DRM_ERROR("Failed to create usermode queue gang\n");
+		break;
 	default:
 		DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n", args->in.op);
 		return -EINVAL;
Signature · 1 line
2.49.0
#10 lore raw

RE: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangs

[Public]

Except for the patch4, 5, and the other patches set is Reviewed-by: Prike Liang <[email protected]>
Question:
1) I don't find the user queue gang submission code in the mesa, could you point out where is the flight mesa counterpart code for supporting user queue gang submission?
2) Do we plan to support user queue gang priority set and change support in the further?

Regards,
Prike

Quoted 94 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangs

Gangs are sets of userqs that schedule together. You specify the primary and secondary queues and the scheduler will make sure they always run at the same time.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 52
+++++++++++++++++++++++
 1 file changed, 52 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +45 -0
index e56fae10400db..95e1495d9a24c 100644
@@ -561,6 +561,51 @@ amdgpu_userq_query_status(struct drm_file *filp, union
 drm_amdgpu_userq *args)
      return 0;
 }
 
+static int
+amdgpu_userq_create_gang(struct drm_file *filp, union drm_amdgpu_userq
+*args) {
+     struct amdgpu_usermode_queue *primary_queue, *secondary_queue;
+     struct amdgpu_fpriv *fpriv = filp->driver_priv;
+     struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
+     const struct amdgpu_userq_funcs *userq_funcs;
+     struct amdgpu_device *adev = uq_mgr->adev;
+     int primary_queue_id = args->in_cg.primary_queue_id;
+     int secondary_queue_id = args->in_cg.secondary_queue_id;
+     int r;
+
+     mutex_lock(&uq_mgr->userq_mutex);
+     primary_queue = amdgpu_userq_find(uq_mgr, primary_queue_id);
+     if (!primary_queue) {
+             dev_err(adev->dev, "Invalid gang primary queue id\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+     if ((primary_queue->queue_type != AMDGPU_HW_IP_GFX) &&
+         (primary_queue->queue_type != AMDGPU_HW_IP_COMPUTE)) {
+             dev_err(adev->dev, "Invalid gang primary queue type\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+     secondary_queue = amdgpu_userq_find(uq_mgr, secondary_queue_id);
+     if (!secondary_queue) {
+             dev_err(adev->dev, "Invalid gang secondary queue id\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+     if ((secondary_queue->queue_type != AMDGPU_HW_IP_GFX) &&
+         (secondary_queue->queue_type != AMDGPU_HW_IP_COMPUTE)) {
+             dev_err(adev->dev, "Invalid gang secondary queue type\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+
+     userq_funcs = adev->userq_funcs[primary_queue->queue_type];
+     r = userq_funcs->set_gang(uq_mgr, primary_queue, secondary_queue);
+     mutex_unlock(&uq_mgr->userq_mutex);
+
+     return r;
+}
+
 int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
                     struct drm_file *filp)

{

drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +7 -0
@@ -611,6 +656,13 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void
 *data,
              if (r)
                      DRM_ERROR("Failed to query usermode queue status\n");
              break;
+     case AMDGPU_USERQ_OP_CREATE_GANG:
+             if (args->in_cg.pad)
+                     return -EINVAL;
+             r = amdgpu_userq_create_gang(filp, args);
+             if (r)
+                     DRM_ERROR("Failed to create usermode queue gang\n");
+             break;
      default:
              DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n",
args->in.op);
              return -EINVAL;

--
2.49.0

#11 lore raw

Re: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangs

Quoted 6 lines · Prike

[Public]

Except for the patch4, 5, and the other patches set is Reviewed-by: Prike Liang <[email protected]>
Question:
1) I don't find the user queue gang submission code in the mesa, could you point out where is the flight mesa counterpart code for supporting user queue gang submission?

Not yet. I'm just proposing the UAPI here. I'd like to get feedback from the mesa tree to understand if this will meet their needs.

Quoted 1 line

2) Do we plan to support user queue gang priority set and change support in the further?

I guess that is another question for userspace. Do they want to be able to change a queue's priority at runtime or only at queue creation time?

Alex

Quoted 99 lines

Regards,
Prike

Quoted 94 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangs

Gangs are sets of userqs that schedule together. You specify the primary and secondary queues and the scheduler will make sure they always run at the same time.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 52
+++++++++++++++++++++++
 1 file changed, 52 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +45 -0
index e56fae10400db..95e1495d9a24c 100644
@@ -561,6 +561,51 @@ amdgpu_userq_query_status(struct drm_file *filp, union
 drm_amdgpu_userq *args)
      return 0;
 }
 
+static int
+amdgpu_userq_create_gang(struct drm_file *filp, union drm_amdgpu_userq
+*args) {
+     struct amdgpu_usermode_queue *primary_queue, *secondary_queue;
+     struct amdgpu_fpriv *fpriv = filp->driver_priv;
+     struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
+     const struct amdgpu_userq_funcs *userq_funcs;
+     struct amdgpu_device *adev = uq_mgr->adev;
+     int primary_queue_id = args->in_cg.primary_queue_id;
+     int secondary_queue_id = args->in_cg.secondary_queue_id;
+     int r;
+
+     mutex_lock(&uq_mgr->userq_mutex);
+     primary_queue = amdgpu_userq_find(uq_mgr, primary_queue_id);
+     if (!primary_queue) {
+             dev_err(adev->dev, "Invalid gang primary queue id\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+     if ((primary_queue->queue_type != AMDGPU_HW_IP_GFX) &&
+         (primary_queue->queue_type != AMDGPU_HW_IP_COMPUTE)) {
+             dev_err(adev->dev, "Invalid gang primary queue type\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+     secondary_queue = amdgpu_userq_find(uq_mgr, secondary_queue_id);
+     if (!secondary_queue) {
+             dev_err(adev->dev, "Invalid gang secondary queue id\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+     if ((secondary_queue->queue_type != AMDGPU_HW_IP_GFX) &&
+         (secondary_queue->queue_type != AMDGPU_HW_IP_COMPUTE)) {
+             dev_err(adev->dev, "Invalid gang secondary queue type\n");
+             mutex_unlock(&uq_mgr->userq_mutex);
+             return -EINVAL;
+     }
+
+     userq_funcs = adev->userq_funcs[primary_queue->queue_type];
+     r = userq_funcs->set_gang(uq_mgr, primary_queue, secondary_queue);
+     mutex_unlock(&uq_mgr->userq_mutex);
+
+     return r;
+}
+
 int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
                     struct drm_file *filp)

{

drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +7 -0
@@ -611,6 +656,13 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void
 *data,
              if (r)
                      DRM_ERROR("Failed to query usermode queue status\n");
              break;
+     case AMDGPU_USERQ_OP_CREATE_GANG:
+             if (args->in_cg.pad)
+                     return -EINVAL;
+             r = amdgpu_userq_create_gang(filp, args);
+             if (r)
+                     DRM_ERROR("Failed to create usermode queue gang\n");
+             break;
      default:
              DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n",
args->in.op);
              return -EINVAL;

--
2.49.0

Acked-by Sunil Khatri on v1 1/8
#12 lore raw
In reply to Alex Deucher

LGTM functionally,

But i would like @christian to look once as he is one of the original author of gang submission.

Regards,
Sunil khatri

Quoted 43 lines · Alex Deucher

Queues in a gang will schedule together.

---
  include/uapi/drm/amdgpu_drm.h | 11 +++++++++++
  1 file changed, 11 insertions(+)
include/uapi/drm/amdgpu_drm.h +11 -0
index 56f052a10ff38..83414563779fb 100644
@@ -329,6 +329,7 @@ union drm_amdgpu_ctx {
  #define AMDGPU_USERQ_OP_CREATE	1
  #define AMDGPU_USERQ_OP_FREE	2
  #define AMDGPU_USERQ_OP_QUERY_STATUS	3
+#define AMDGPU_USERQ_OP_CREATE_GANG	4
 
  /* queue priority levels */
  /* low < normal low < normal high < high */
@@ -417,6 +418,15 @@ struct drm_amdgpu_userq_in {
  	__u64 mqd_size;
  };
 
+struct drm_amdgpu_userq_in_create_gang {
+	/** AMDGPU_USERQ_OP_* */
+	__u32	op;
+	__u32	pad;
+	/** Queue ids passed for operation USERQ_OP_CREATE_GANG */
+	__u32	primary_queue_id;
+	__u32	secondary_queue_id;
+};
+
  /* The structure to carry output of userqueue ops */
  struct drm_amdgpu_userq_out {
  	/**
@@ -436,6 +446,7 @@ struct drm_amdgpu_userq_out_query_state {
 
  union drm_amdgpu_userq {
  	struct drm_amdgpu_userq_in in;
+	struct drm_amdgpu_userq_in_create_gang in_cg;
  	struct drm_amdgpu_userq_out out;
  	struct drm_amdgpu_userq_out_query_state out_qs;
  };
#13 lore raw

Small edit,
Series is Acked-by: Sunil Khatri <[email protected]>

Quoted 49 lines · Sunil

LGTM functionally,

But i would like @christian to look once as he is one of the original author of gang submission.

Regards,
Sunil khatri

Quoted 40 lines · Alex Deucher

Queues in a gang will schedule together.

---
  include/uapi/drm/amdgpu_drm.h | 11 +++++++++++

  1 file changed, 11 insertions(+)
include/uapi/drm/amdgpu_drm.h +10 -0
index 56f052a10ff38..83414563779fb 100644
@@ -329,6 +329,7 @@ union drm_amdgpu_ctx {
   #define AMDGPU_USERQ_OP_CREATE    1
   #define AMDGPU_USERQ_OP_FREE    2
   #define AMDGPU_USERQ_OP_QUERY_STATUS    3
+#define AMDGPU_USERQ_OP_CREATE_GANG    4
     /* queue priority levels */
   /* low < normal low < normal high < high */
@@ -417,6 +418,15 @@ struct drm_amdgpu_userq_in {
       __u64 mqd_size;
   };
   +struct drm_amdgpu_userq_in_create_gang {
+    /** AMDGPU_USERQ_OP_* */
+    __u32    op;
+    __u32    pad;
+    /** Queue ids passed for operation USERQ_OP_CREATE_GANG */
+    __u32    primary_queue_id;
+    __u32    secondary_queue_id;
+};
+
   /* The structure to carry output of userqueue ops */
   struct drm_amdgpu_userq_out {
       /**
@@ -436,6 +446,7 @@ struct drm_amdgpu_userq_out_query_state {
     union drm_amdgpu_userq {
       struct drm_amdgpu_userq_in in;
+    struct drm_amdgpu_userq_in_create_gang in_cg;
       struct drm_amdgpu_userq_out out;
       struct drm_amdgpu_userq_out_query_state out_qs;
   };