drm/amdgpu: add UAPI to create user queue gangs
Subsystems (3)
Alex Deucher
maintainerPATCH 1/8 drm/amdgpu: add UAPI to create user queue gangs
Queues in a gang will schedule together.
--- include/uapi/drm/amdgpu_drm.h | 11 +++++++++++ 1 file changed, 11 insertions(+)
include/uapi/drm/amdgpu_drm.h +11 -0
@@ -329,6 +329,7 @@ union drm_amdgpu_ctx {#define AMDGPU_USERQ_OP_CREATE 1#define AMDGPU_USERQ_OP_FREE 2#define AMDGPU_USERQ_OP_QUERY_STATUS 3 +#define AMDGPU_USERQ_OP_CREATE_GANG 4/* queue priority levels */ /* low < normal low < normal high < high */
@@ -417,6 +418,15 @@ struct drm_amdgpu_userq_in {__u64 mqd_size; }; +struct drm_amdgpu_userq_in_create_gang { +/** AMDGPU_USERQ_OP_* */ +__u32 op; +__u32 pad; +/** Queue ids passed for operation USERQ_OP_CREATE_GANG */ +__u32 primary_queue_id; +__u32 secondary_queue_id; +}; +/* The structure to carry output of userqueue ops */ struct drm_amdgpu_userq_out {/**
@@ -436,6 +446,7 @@ struct drm_amdgpu_userq_out_query_state {union drm_amdgpu_userq {struct drm_amdgpu_userq_in in; +struct drm_amdgpu_userq_in_create_gang in_cg;struct drm_amdgpu_userq_out out;struct drm_amdgpu_userq_out_query_state out_qs; };
Signature · 1 line
2.49.0
Alex Deucher
maintainerPATCH 2/8 drm/amdgpu/mes11: add API for gang submit
Add API for setting up a gang. Gangs schedule together.
--- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 2 +- drivers/gpu/drm/amd/include/mes_v11_api_def.h | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +1 -1
@@ -125,7 +125,7 @@ static const char *mes_v11_0_opcodes[] = {"UPDATE_ROOT_PAGE_TABLE" ,"AMD_LOG" ,"unused" , -"unused" , +"SET_GANG_SUBMIT" , "SET_HW_RSRC_1", };
drivers/gpu/drm/amd/include/mes_v11_api_def.h +18 -0
@@ -64,6 +64,7 @@ enum MES_SCH_API_OPCODE {MES_SCH_API_MISC =14 ,MES_SCH_API_UPDATE_ROOT_PAGE_TABLE =15 ,MES_SCH_API_AMD_LOG =16 , +MES_SCH_API_SET_GANG_SUBMIT =18 ,MES_SCH_API_SET_HW_RSRC_1 =19 , MES_SCH_API_MAX = 0xFF };
@@ -633,6 +634,13 @@ struct SET_SHADER_DEBUGGER {uint32_t trap_en; }; +struct SET_GANG_SUBMIT { +uint64_t gang_context_addr; +uint64_t slave_gang_context_addr; +uint32_t gang_context_array_index; +uint32_t slave_gang_context_array_index; +}; +enum MESAPI_MISC__CHANGE_CONFIG_OPTION {MESAPI_MISC__CHANGE_CONFIG_OPTION_LIMIT_SINGLE_PROCESS =0 ,MESAPI_MISC__CHANGE_CONFIG_OPTION_ENABLE_HWS_LOGGING_BUFFER =1 ,
@@ -703,5 +711,15 @@ union MESAPI_AMD_LOG { uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS]; }; +union MESAPI__SET_GANG_SUBMIT { +struct { +union MES_API_HEADER header; +struct MES_API_STATUS api_status; +struct SET_GANG_SUBMIT set_gang_submit; + }; + +uint32_t max_dwords_in_api[API_FRAME_SIZE_IN_DWORDS ]; +}; +#pragma pack(pop)#endif
Signature · 1 line
2.49.0
Alex Deucher
maintainerPATCH 3/8 drm/amdgpu/mes: add callback for gang submit
In reply to Alex DeucherAdd function pointer for gang submit setup.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 8 ++++++++ 1 file changed, 8 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +8 -0
@@ -279,6 +279,11 @@ struct mes_reset_queue_input {bool legacy_gfx; }; +struct mes_set_gang_submit_input { +uint64_t primary_gang_context_addr; +uint64_t secondary_gang_context_addr; +}; +enum mes_misc_opcode {MES_MISC_OP_WRITE_REG ,MES_MISC_OP_READ_REG ,
@@ -366,6 +371,9 @@ struct amdgpu_mes_funcs {int (*reset_hw_queue)(struct amdgpu_mes *mes,struct mes_reset_queue_input *input); + +int (*set_gang_submit)(struct amdgpu_mes *mes, +struct mes_set_gang_submit_input *input); };#define amdgpu_mes_kiq_hw_init (adev) (adev)->mes.kiq_hw_init((adev))
Signature · 1 line
2.49.0
Alex Deucher
maintainerPATCH 4/8 drm/amdgpu/mes11: add support for setting gang submit
In reply to Alex DeucherEnable a primary and secondary queue that schedule together.
--- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +22 -0
@@ -783,6 +783,27 @@ static int mes_v11_0_reset_hw_queue(struct amdgpu_mes *mes, offsetof(union MESAPI__RESET, api_status)); } +static int mes_v11_0_set_gang_submit (struct amdgpu_mes *mes, +struct mes_set_gang_submit_input *input) +{ +union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt; + +memset (&mes_gang_submit_pkt,0 ,sizeof (mes_gang_submit_pkt)); + + mes_gang_submit_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_gang_submit_pkt.header.opcode =MES_SCH_API_SET_GANG_SUBMIT ; + mes_gang_submit_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_gang_submit_pkt.set_gang_submit.gang_context_addr = + input->primary_gang_context_addr; + mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr = + input->secondary_gang_context_addr; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_gang_submit_pkt,sizeof (mes_gang_submit_pkt), + offsetof(union MESAPI__RESET , api_status)); +} +static const struct amdgpu_mes_funcs mes_v11_0_funcs = { .add_hw_queue = mes_v11_0_add_hw_queue, .remove_hw_queue = mes_v11_0_remove_hw_queue,
@@ -792,6 +813,7 @@ static const struct amdgpu_mes_funcs mes_v11_0_funcs = { .resume_gang = mes_v11_0_resume_gang, .misc_op = mes_v11_0_misc_op, .reset_hw_queue = mes_v11_0_reset_hw_queue, + .set_gang_submit = mes_v11_0_set_gang_submit, }; static int mes_v11_0_allocate_ucode_buffer(struct amdgpu_device *adev,
Signature · 1 line
2.49.0
RE: [PATCH 4/8] drm/amdgpu/mes11: add support for setting gang submit
[Public]
Quoted 34 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 4/8] drm/amdgpu/mes11: add support for setting gang submitEnable a primary and secondary queue that schedule together.
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +9
-0@@ -783,6 +783,27 @@ static int mes_v11_0_reset_hw_queue(structamdgpu_mes *mes, offsetof(union MESAPI__RESET, api_status)); } +static int mes_v11_0_set_gang_submit (struct amdgpu_mes *mes, +struct mes_set_gang_submit_input *input) { +union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt; + +memset (&mes_gang_submit_pkt,0 ,sizeof (mes_gang_submit_pkt)); + + mes_gang_submit_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_gang_submit_pkt.header.opcode =MES_SCH_API_SET_GANG_SUBMIT ; + mes_gang_submit_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ;
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c static int mes_v11_0_reset_hw_queue(struct · line 805
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +10
-0+ + mes_gang_submit_pkt.set_gang_submit.gang_context_addr = + input->primary_gang_context_addr; + mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr = + input->secondary_gang_context_addr; + +return mes_v11_0_submit_pkt_and_poll_completion (mes, + &mes_gang_submit_pkt,sizeof (mes_gang_submit_pkt), + offsetof(union MESAPI__RESET , api_status)); } +
Please use the MESAPI__SET_GANG_SUBMIT type to get the API status. With that fix, the patch is Reviewed-by: Prike Liang <[email protected]>
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +0
-0static const struct amdgpu_mes_funcs mes_v11_0_funcs = { .add_hw_queue = mes_v11_0_add_hw_queue, .remove_hw_queue = mes_v11_0_remove_hw_queue, @@ -792,6 +813,7@@ static const struct amdgpu_mes_funcs mes_v11_0_funcs = { .resume_gang = mes_v11_0_resume_gang, .misc_op = mes_v11_0_misc_op, .reset_hw_queue = mes_v11_0_reset_hw_queue,+ .set_gang_submit =
mes_v11_0_set_gang_submit,
};static int mes_v11_0_allocate_ucode_buffer(struct
amdgpu_device*adev,
--
2.49.0
Alex Deucher
maintainerPATCH 5/8 drm/amdgpu/mes12: add support for setting gang submit
In reply to Alex DeucherEnable a primary and secondary queue that schedule together.
--- drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +28 -0
@@ -879,6 +879,33 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes, offsetof(union MESAPI__RESET, api_status)); } +static int mes_v12_0_set_gang_submit (struct amdgpu_mes *mes, +struct mes_set_gang_submit_input *input) +{ +union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt; +int pipe; + +memset (&mes_gang_submit_pkt,0 ,sizeof (mes_gang_submit_pkt)); + + mes_gang_submit_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_gang_submit_pkt.header.opcode =MES_SCH_API_SET_GANG_SUBMIT ; + mes_gang_submit_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_gang_submit_pkt.set_gang_submit.gang_context_addr = + input->primary_gang_context_addr; + mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr = + input->secondary_gang_context_addr; + +if (mes->adev->enable_uni_mes) + pipe =AMDGPU_MES_KIQ_PIPE ; +else + pipe =AMDGPU_MES_SCHED_PIPE ; + +return mes_v12_0_submit_pkt_and_poll_completion (mes, pipe, + &mes_gang_submit_pkt,sizeof (mes_gang_submit_pkt), + offsetof(union MESAPI__RESET , api_status)); +} +static const struct amdgpu_mes_funcs mes_v12_0_funcs = { .add_hw_queue = mes_v12_0_add_hw_queue, .remove_hw_queue = mes_v12_0_remove_hw_queue,
@@ -888,6 +915,7 @@ static const struct amdgpu_mes_funcs mes_v12_0_funcs = { .resume_gang = mes_v12_0_resume_gang, .misc_op = mes_v12_0_misc_op, .reset_hw_queue = mes_v12_0_reset_hw_queue, + .set_gang_submit = mes_v12_0_set_gang_submit, }; static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,
Signature · 1 line
2.49.0
RE: [PATCH 5/8] drm/amdgpu/mes12: add support for setting gang submit
[Public]
Quoted 41 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 5/8] drm/amdgpu/mes12: add support for setting gang submitEnable a primary and secondary queue that schedule together.
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/mes_v12_0.c | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+)drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +15
-0@@ -879,6 +879,33 @@ static int mes_v12_0_reset_hw_queue(structamdgpu_mes *mes, offsetof(union MESAPI__RESET, api_status)); } +static int mes_v12_0_set_gang_submit (struct amdgpu_mes *mes, +struct mes_set_gang_submit_input *input) { +union MESAPI__SET_GANG_SUBMIT mes_gang_submit_pkt; +int pipe; + +memset (&mes_gang_submit_pkt,0 ,sizeof (mes_gang_submit_pkt)); + + mes_gang_submit_pkt.header.type =MES_API_TYPE_SCHEDULER ; + mes_gang_submit_pkt.header.opcode =MES_SCH_API_SET_GANG_SUBMIT ; + mes_gang_submit_pkt.header.dwsize =API_FRAME_SIZE_IN_DWORDS ; + + mes_gang_submit_pkt.set_gang_submit.gang_context_addr = + input->primary_gang_context_addr; + mes_gang_submit_pkt.set_gang_submit.slave_gang_context_addr = + input->secondary_gang_context_addr;
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c static int mes_v12_0_reset_hw_queue(struct · line 907
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +10
-0+ +if (mes->adev->enable_uni_mes) + pipe =AMDGPU_MES_KIQ_PIPE ; +else + pipe =AMDGPU_MES_SCHED_PIPE ; + +return mes_v12_0_submit_pkt_and_poll_completion (mes, pipe, + &mes_gang_submit_pkt,sizeof (mes_gang_submit_pkt), + offsetof(union MESAPI__RESET , api_status)); } +
This also requires using the MESAPI__SET_GANG_SUBMIT type to get the API status offset.
With this update, the patch is Reviewed-by: Prike Liang <[email protected]>
Quoted 13 lines
drivers/gpu/drm/amd/amdgpu/mes_v12_0.c +0
-0static const struct amdgpu_mes_funcs mes_v12_0_funcs = { .add_hw_queue = mes_v12_0_add_hw_queue, .remove_hw_queue = mes_v12_0_remove_hw_queue, @@ -888,6 +915,7@@ static const struct amdgpu_mes_funcs mes_v12_0_funcs = { .resume_gang = mes_v12_0_resume_gang, .misc_op = mes_v12_0_misc_op, .reset_hw_queue = mes_v12_0_reset_hw_queue,+ .set_gang_submit =
mes_v12_0_set_gang_submit,
};static int mes_v12_0_allocate_ucode_buffer(struct
amdgpu_device*adev,
--
2.49.0
Alex Deucher
maintainerPATCH 6/8 drm/amdgpu/userq: add callback for gang setup
In reply to Alex DeucherTwo queues can be ganged and will schedule together.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 3 +++ 1 file changed, 3 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h +3 -0
@@ -80,6 +80,9 @@ struct amdgpu_userq_funcs { struct amdgpu_usermode_queue *queue);int (*reset)(struct amdgpu_userq_mgr *uq_mgr,struct amdgpu_usermode_queue *queue); +int (*set_gang)(struct amdgpu_userq_mgr *uq_mgr, +struct amdgpu_usermode_queue *primary_queue, +struct amdgpu_usermode_queue *secondary_queue); };/* Usermode queues for gfx */
Signature · 1 line
2.49.0
Alex Deucher
maintainerPATCH 7/8 drm/amdgpu/userq/mes: implement set gang submit
In reply to Alex DeucherEnable the userq callback for setting up gang submission.
--- drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+)
drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +26 -0
@@ -367,10 +367,36 @@ static int mes_userq_reset(struct amdgpu_userq_mgr *uq_mgr,return r; } +static int mes_set_gang (struct amdgpu_userq_mgr *uq_mgr, +struct amdgpu_usermode_queue *primary_queue, +struct amdgpu_usermode_queue *secondary_queue) +{ +struct amdgpu_userq_obj *primary_ctx = &primary_queue->fw_obj; +struct amdgpu_userq_obj *secondary_ctx = &secondary_queue->fw_obj; +struct amdgpu_device *adev = uq_mgr->adev; +struct mes_set_gang_submit_input input; +int r; + +memset (&input,0x0 ,sizeof (struct mes_set_gang_submit_input )); + + input.primary_gang_context_addr = + primary_ctx->gpu_addr +AMDGPU_USERQ_PROC_CTX_SZ ; + input.secondary_gang_context_addr = + secondary_ctx->gpu_addr +AMDGPU_USERQ_PROC_CTX_SZ ; + +amdgpu_mes_lock (&adev->mes); + r = adev->mes.funcs->set_gang_submit (&adev->mes, &input); +amdgpu_mes_unlock (&adev->mes); +if (r) +dev_err (adev->dev,"Failed to set gang submit, err (%d)\n" , r); +return r; +} +const struct amdgpu_userq_funcs userq_mes_funcs = { .mqd_create = mes_userq_mqd_create, .mqd_destroy = mes_userq_mqd_destroy, .unmap = mes_userq_unmap, .map = mes_userq_map, .reset = mes_userq_reset, + .set_gang = mes_set_gang, };
Signature · 1 line
2.49.0
Alex Deucher
maintainerPATCH 8/8 drm/amdgpu/userq: enable support for creating gangs
In reply to Alex DeucherGangs are sets of userqs that schedule together. You specify the primary and secondary queues and the scheduler will make sure they always run at the same time.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +52 -0
@@ -561,6 +561,51 @@ amdgpu_userq_query_status(struct drm_file *filp, union drm_amdgpu_userq *args)return 0 ; } +static int +amdgpu_userq_create_gang (struct drm_file *filp,union drm_amdgpu_userq *args) +{ +struct amdgpu_usermode_queue *primary_queue, *secondary_queue; +struct amdgpu_fpriv *fpriv = filp->driver_priv; +struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; +const struct amdgpu_userq_funcs *userq_funcs; +struct amdgpu_device *adev = uq_mgr->adev; +int primary_queue_id = args->in_cg.primary_queue_id; +int secondary_queue_id = args->in_cg.secondary_queue_id; +int r; + +mutex_lock (&uq_mgr->userq_mutex); + primary_queue =amdgpu_userq_find (uq_mgr, primary_queue_id); +if (!primary_queue) { +dev_err (adev->dev,"Invalid gang primary queue id\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } +if ((primary_queue->queue_type !=AMDGPU_HW_IP_GFX ) && + (primary_queue->queue_type !=AMDGPU_HW_IP_COMPUTE )) { +dev_err (adev->dev,"Invalid gang primary queue type\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } + secondary_queue =amdgpu_userq_find (uq_mgr, secondary_queue_id); +if (!secondary_queue) { +dev_err (adev->dev,"Invalid gang secondary queue id\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } +if ((secondary_queue->queue_type !=AMDGPU_HW_IP_GFX ) && + (secondary_queue->queue_type !=AMDGPU_HW_IP_COMPUTE )) { +dev_err (adev->dev,"Invalid gang secondary queue type\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } + + userq_funcs = adev->userq_funcs[primary_queue->queue_type]; + r = userq_funcs->set_gang (uq_mgr, primary_queue, secondary_queue); +mutex_unlock (&uq_mgr->userq_mutex); + +return r; +} +int amdgpu_userq_ioctl (struct drm_device *dev,void *data,struct drm_file *filp) {
@@ -611,6 +656,13 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data,if (r)DRM_ERROR ("Failed to query usermode queue status\n" );break ; +case AMDGPU_USERQ_OP_CREATE_GANG : +if (args->in_cg.pad) +return -EINVAL ; + r =amdgpu_userq_create_gang (filp, args); +if (r) +DRM_ERROR ("Failed to create usermode queue gang\n" ); +break ;default :DRM_DEBUG_DRIVER ("Invalid user queue op specified: %d\n" , args->in.op);return -EINVAL ;
Signature · 1 line
2.49.0
RE: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangs
[Public]
Except for the patch4, 5, and the other patches set is Reviewed-by: Prike Liang <[email protected]>
Question:
1) I don't find the user queue gang submission code in the mesa, could you point out where is the flight mesa counterpart code for supporting user queue gang submission?
2) Do we plan to support user queue gang priority set and change support in the further?
Regards,
Prike
Quoted 94 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangsGangs are sets of userqs that schedule together. You specify the primary and secondary queues and the scheduler will make sure they always run at the same time.
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +45
-0@@ -561,6 +561,51 @@ amdgpu_userq_query_status(struct drm_file *filp, union drm_amdgpu_userq *args)return 0 ; } +static int +amdgpu_userq_create_gang (struct drm_file *filp,union drm_amdgpu_userq +*args) { +struct amdgpu_usermode_queue *primary_queue, *secondary_queue; +struct amdgpu_fpriv *fpriv = filp->driver_priv; +struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; +const struct amdgpu_userq_funcs *userq_funcs; +struct amdgpu_device *adev = uq_mgr->adev; +int primary_queue_id = args->in_cg.primary_queue_id; +int secondary_queue_id = args->in_cg.secondary_queue_id; +int r; + +mutex_lock (&uq_mgr->userq_mutex); + primary_queue =amdgpu_userq_find (uq_mgr, primary_queue_id); +if (!primary_queue) { +dev_err (adev->dev,"Invalid gang primary queue id\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } +if ((primary_queue->queue_type !=AMDGPU_HW_IP_GFX ) && + (primary_queue->queue_type !=AMDGPU_HW_IP_COMPUTE )) { +dev_err (adev->dev,"Invalid gang primary queue type\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } + secondary_queue =amdgpu_userq_find (uq_mgr, secondary_queue_id); +if (!secondary_queue) { +dev_err (adev->dev,"Invalid gang secondary queue id\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } +if ((secondary_queue->queue_type !=AMDGPU_HW_IP_GFX ) && + (secondary_queue->queue_type !=AMDGPU_HW_IP_COMPUTE )) { +dev_err (adev->dev,"Invalid gang secondary queue type\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } + + userq_funcs = adev->userq_funcs[primary_queue->queue_type]; + r = userq_funcs->set_gang (uq_mgr, primary_queue, secondary_queue); +mutex_unlock (&uq_mgr->userq_mutex); + +return r; +} +int amdgpu_userq_ioctl (struct drm_device *dev,void *data,struct drm_file *filp){
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +7
-0@@ -611,6 +656,13 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, if (r)DRM_ERROR ("Failed to query usermode queue status\n" );break ; +case AMDGPU_USERQ_OP_CREATE_GANG : +if (args->in_cg.pad) +return -EINVAL ; + r =amdgpu_userq_create_gang (filp, args); +if (r) +DRM_ERROR ("Failed to create usermode queue gang\n" ); +break ;default : DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n",args->in.op); return -EINVAL;--
2.49.0
Re: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangs
Quoted 6 lines · Prike
[Public]
Except for the patch4, 5, and the other patches set is Reviewed-by: Prike Liang <[email protected]>
Question:
1) I don't find the user queue gang submission code in the mesa, could you point out where is the flight mesa counterpart code for supporting user queue gang submission?
Not yet. I'm just proposing the UAPI here. I'd like to get feedback from the mesa tree to understand if this will meet their needs.
Quoted 1 line
2) Do we plan to support user queue gang priority set and change support in the further?
I guess that is another question for userspace. Do they want to be able to change a queue's priority at runtime or only at queue creation time?
Alex
Quoted 99 lines
Regards,
PrikeQuoted 94 lines
-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Alex Deucher
Sent: Saturday, April 26, 2025 2:41 AM
To: [email protected]
Cc: Deucher, Alexander <[email protected]>
Subject: [PATCH 8/8] drm/amdgpu/userq: enable support for creating gangsGangs are sets of userqs that schedule together. You specify the primary and secondary queues and the scheduler will make sure they always run at the same time.
Signed-off-by Alex Deucher <[email protected]>--- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 52 +++++++++++++++++++++++ 1 file changed, 52 insertions(+)drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +45
-0@@ -561,6 +561,51 @@ amdgpu_userq_query_status(struct drm_file *filp, union drm_amdgpu_userq *args)return 0 ; } +static int +amdgpu_userq_create_gang (struct drm_file *filp,union drm_amdgpu_userq +*args) { +struct amdgpu_usermode_queue *primary_queue, *secondary_queue; +struct amdgpu_fpriv *fpriv = filp->driver_priv; +struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; +const struct amdgpu_userq_funcs *userq_funcs; +struct amdgpu_device *adev = uq_mgr->adev; +int primary_queue_id = args->in_cg.primary_queue_id; +int secondary_queue_id = args->in_cg.secondary_queue_id; +int r; + +mutex_lock (&uq_mgr->userq_mutex); + primary_queue =amdgpu_userq_find (uq_mgr, primary_queue_id); +if (!primary_queue) { +dev_err (adev->dev,"Invalid gang primary queue id\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } +if ((primary_queue->queue_type !=AMDGPU_HW_IP_GFX ) && + (primary_queue->queue_type !=AMDGPU_HW_IP_COMPUTE )) { +dev_err (adev->dev,"Invalid gang primary queue type\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } + secondary_queue =amdgpu_userq_find (uq_mgr, secondary_queue_id); +if (!secondary_queue) { +dev_err (adev->dev,"Invalid gang secondary queue id\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } +if ((secondary_queue->queue_type !=AMDGPU_HW_IP_GFX ) && + (secondary_queue->queue_type !=AMDGPU_HW_IP_COMPUTE )) { +dev_err (adev->dev,"Invalid gang secondary queue type\n" ); +mutex_unlock (&uq_mgr->userq_mutex); +return -EINVAL ; + } + + userq_funcs = adev->userq_funcs[primary_queue->queue_type]; + r = userq_funcs->set_gang (uq_mgr, primary_queue, secondary_queue); +mutex_unlock (&uq_mgr->userq_mutex); + +return r; +} +int amdgpu_userq_ioctl (struct drm_device *dev,void *data,struct drm_file *filp){
drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +7
-0@@ -611,6 +656,13 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, if (r)DRM_ERROR ("Failed to query usermode queue status\n" );break ; +case AMDGPU_USERQ_OP_CREATE_GANG : +if (args->in_cg.pad) +return -EINVAL ; + r =amdgpu_userq_create_gang (filp, args); +if (r) +DRM_ERROR ("Failed to create usermode queue gang\n" ); +break ;default : DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n",args->in.op); return -EINVAL;--
2.49.0
LGTM functionally,
But i would like @christian to look once as he is one of the original author of gang submission.
Regards,
Sunil khatri
Quoted 43 lines · Alex Deucher
Queues in a gang will schedule together.
Signed-off-by Alex Deucher <[email protected]>--- include/uapi/drm/amdgpu_drm.h | 11 +++++++++++ 1 file changed, 11 insertions(+)include/uapi/drm/amdgpu_drm.h +11
-0@@ -329,6 +329,7 @@ union drm_amdgpu_ctx {#define AMDGPU_USERQ_OP_CREATE 1#define AMDGPU_USERQ_OP_FREE 2#define AMDGPU_USERQ_OP_QUERY_STATUS 3 +#define AMDGPU_USERQ_OP_CREATE_GANG 4/* queue priority levels */ /* low < normal low < normal high < high */ @@ -417,6 +418,15 @@ struct drm_amdgpu_userq_in {__u64 mqd_size; }; +struct drm_amdgpu_userq_in_create_gang { +/** AMDGPU_USERQ_OP_* */ +__u32 op; +__u32 pad; +/** Queue ids passed for operation USERQ_OP_CREATE_GANG */ +__u32 primary_queue_id; +__u32 secondary_queue_id; +}; +/* The structure to carry output of userqueue ops */ struct drm_amdgpu_userq_out {/** @@ -436,6 +446,7 @@ struct drm_amdgpu_userq_out_query_state {union drm_amdgpu_userq {struct drm_amdgpu_userq_in in; +struct drm_amdgpu_userq_in_create_gang in_cg;struct drm_amdgpu_userq_out out;struct drm_amdgpu_userq_out_query_state out_qs; };
Small edit,
Series is Acked-by: Sunil Khatri <[email protected]>
Quoted 49 lines · Sunil
LGTM functionally,
Acked-by Sunil Khatri <[email protected]>But i would like @christian to look once as he is one of the original author of gang submission.
Regards,
Sunil khatriQuoted 40 lines · Alex Deucher
Queues in a gang will schedule together.
Signed-off-by Alex Deucher <[email protected]>---
include/uapi/drm/amdgpu_drm.h| 11 +++++++++++1 file changed, 11 insertions(+)include/uapi/drm/amdgpu_drm.h +10
-0@@ -329,6 +329,7 @@ union drm_amdgpu_ctx { #define AMDGPU_USERQ_OP_CREATE 1 #define AMDGPU_USERQ_OP_FREE 2 #define AMDGPU_USERQ_OP_QUERY_STATUS 3 +#define AMDGPU_USERQ_OP_CREATE_GANG 4 /* queue priority levels */ /* low < normal low < normal high < high */@@ -417,6 +418,15 @@ struct drm_amdgpu_userq_in { __u64 mqd_size; }; +struct drm_amdgpu_userq_in_create_gang { + /** AMDGPU_USERQ_OP_* */ + __u32 op; + __u32 pad; + /** Queue ids passed for operation USERQ_OP_CREATE_GANG */ + __u32 primary_queue_id; + __u32 secondary_queue_id; +}; + /* The structure to carry output of userqueue ops */ struct drm_amdgpu_userq_out { /**@@ -436,6 +446,7 @@ struct drm_amdgpu_userq_out_query_state { union drm_amdgpu_userq { struct drm_amdgpu_userq_in in; + struct drm_amdgpu_userq_in_create_gang in_cg; struct drm_amdgpu_userq_out out; struct drm_amdgpu_userq_out_query_state out_qs; };