RDMA/efa: Support QP service level
Subsystems (2)
Michael Margolin
maintainercover RDMA/efa: Support QP service level
Align the device interface and add an option to pass QP service level from user to the device.
Michael Margolin (2):
RDMA/efa: Update device interface
RDMA/efa: Add option to set QP service level on create
.../infiniband/hw/efa/efa_admin_cmds_defs.h | 54 +++++++-- drivers/infiniband/hw/efa/efa_admin_defs.h | 4 +- drivers/infiniband/hw/efa/efa_com_cmd.c | 5 +- drivers/infiniband/hw/efa/efa_com_cmd.h | 3 +- drivers/infiniband/hw/efa/efa_io_defs.h | 106 ++++++++++++++++-- drivers/infiniband/hw/efa/efa_verbs.c | 6 +- include/uapi/rdma/efa-abi.h | 3 +- 7 files changed, 156 insertions(+), 25 deletions(-)
Signature · 1 line
2.40.1
Michael Margolin
maintainerPATCH 1/2 RDMA/efa: Update device interface
Update device interface header files.
--- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 54 +++++++-- drivers/infiniband/hw/efa/efa_admin_defs.h | 4 +- drivers/infiniband/hw/efa/efa_com_cmd.c | 4 +- drivers/infiniband/hw/efa/efa_com_cmd.h | 2 +- drivers/infiniband/hw/efa/efa_io_defs.h | 106 ++++++++++++++++-- drivers/infiniband/hw/efa/efa_verbs.c | 2 +- 6 files changed, 149 insertions(+), 23 deletions(-)
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +47 -7
@@ -30,7 +30,8 @@ enum efa_admin_aq_opcode {EFA_ADMIN_DEALLOC_UAR =17 ,EFA_ADMIN_CREATE_EQ =18 ,EFA_ADMIN_DESTROY_EQ =19 , - EFA_ADMIN_MAX_OPCODE = 19, +EFA_ADMIN_ALLOC_MR =20 , + EFA_ADMIN_MAX_OPCODE = 20, }; enum efa_admin_aq_feature_id {
@@ -150,8 +151,11 @@ struct efa_admin_create_qp_cmd {/* UAR number */ u16 uar; +/* Requested service level for the QP, 0 is the default SL */ +u8 sl; +/* MBZ */ -u16 reserved; +u8 reserved;/* MBZ */ u32 reserved2;
@@ -459,6 +463,41 @@ struct efa_admin_dereg_mr_resp {struct efa_admin_acq_common_desc acq_common_desc; }; +/* +* Allocation of MemoryRegion, required for QP working with Virtual +* Addresses in kernel verbs semantics, ready for fast registration use. +*/ +struct efa_admin_alloc_mr_cmd { +/* Common Admin Queue descriptor */ +struct efa_admin_aq_common_desc aq_common_desc; + +/* Protection Domain */ +u16 pd; + +/* MBZ */ +u16 reserved1; + +/* Maximum number of pages this MR supports. */ +u32 max_pages; +}; + +struct efa_admin_alloc_mr_resp { +/* Common Admin Queue completion descriptor */ +struct efa_admin_acq_common_desc acq_common_desc; + +/* +* L_Key, to be used in conjunction with local buffer references in +* SQ and RQ WQE, or with virtual RQ/CQ rings +*/ +u32 l_key; + +/* +* R_Key, to be used in RDMA messages to refer to remotely accessed +* memory region +*/ +u32 r_key; +}; +struct efa_admin_create_cq_cmd {struct efa_admin_aq_common_desc aq_common_desc;
@@ -483,8 +522,8 @@ struct efa_admin_create_cq_cmd {*/ u8 cq_caps_2; -/* completion queue depth in # of entries. must be power of 2 */ -u16 cq_depth; +/* Sub completion queue depth in # of entries. must be power of 2 */ +u16 sub_cq_depth;/* EQ number assigned to this cq */ u16 eqn;
@@ -519,8 +558,8 @@ struct efa_admin_create_cq_resp {u16 cq_idx; -/* actual cq depth in number of entries */ -u16 cq_actual_depth; +/* actual sub cq depth in number of entries */ +u16 sub_cq_actual_depth;/* CQ doorbell address, as offset to PCIe DB BAR */ u32 db_offset;
@@ -578,6 +617,8 @@ struct efa_admin_basic_stats {u64 rx_pkts;u64 rx_drops; + +u64 qkey_viol; };struct efa_admin_messages_stats {
@@ -1057,7 +1098,6 @@ struct efa_admin_host_info {/* create_eq_cmd */ #define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) -#define EFA_ADMIN_CREATE_EQ_CMD_VIRT_MASK BIT(6)#define EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS_MASK BIT(0)/* host_info */
drivers/infiniband/hw/efa/efa_admin_defs.h +2 -2
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_ADMIN_H_
@@ -96,7 +96,7 @@ struct efa_admin_acq_entry {struct efa_admin_aenq_common_desc {u16 group; -u16 syndrom; +u16 syndrome;/* * 0 : phase
drivers/infiniband/hw/efa/efa_com_cmd.c +2 -2
@@ -163,7 +163,7 @@ int efa_com_create_cq(struct efa_com_dev *edev,EFA_SET (&create_cmd.cq_caps_2,EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS , params->entry_size_in_bytes /4 ); - create_cmd.cq_depth = params->cq_depth; + create_cmd.sub_cq_depth = params->sub_cq_depth; create_cmd.num_sub_cqs = params->num_sub_cqs; create_cmd.uar = params->uarn;if (params->interrupt_mode_enabled) {
@@ -191,7 +191,7 @@ int efa_com_create_cq(struct efa_com_dev *edev, } result->cq_idx = cmd_completion.cq_idx; - result->actual_depth = params->cq_depth; + result->actual_depth = params->sub_cq_depth; result->db_off = cmd_completion.db_offset; result->db_valid =EFA_GET (&cmd_completion.flags,EFA_ADMIN_CREATE_CQ_RESP_DB_VALID );
drivers/infiniband/hw/efa/efa_com_cmd.h +1 -1
@@ -71,7 +71,7 @@ struct efa_com_create_cq_params {/* cq physical base address in OS memory */ dma_addr_t dma_addr;/* completion queue depth in # of entries */ -u16 cq_depth; +u16 sub_cq_depth;u16 num_sub_cqs;u16 uarn;u16 eqn;
drivers/infiniband/hw/efa/efa_io_defs.h +96 -10
187 lines not on this page · raw message
drivers/infiniband/hw/efa/efa_verbs.c +1 -1
@@ -1167,7 +1167,7 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } params.uarn = cq->ucontext->uarn; - params.cq_depth = entries; + params.sub_cq_depth = entries; params.dma_addr = cq->dma_addr; params.entry_size_in_bytes = cmd.cq_entry_size; params.num_sub_cqs = cmd.num_sub_cqs;
Signature · 1 line
2.40.1
Michael Margolin
maintainerPATCH 2/2 RDMA/efa: Add option to set QP service level on create
In reply to Michael MargolinUsing modify QP with AH attributes and IB_QP_AV flag set doesn't make much sense for connectionless QP types like SRD. Add SL parameter to EFA create QP user ABI and pass it to the device.
--- drivers/infiniband/hw/efa/efa_com_cmd.c | 1 + drivers/infiniband/hw/efa/efa_com_cmd.h | 1 + drivers/infiniband/hw/efa/efa_verbs.c | 4 +++- include/uapi/rdma/efa-abi.h | 3 ++- 4 files changed, 7 insertions(+), 2 deletions(-)
drivers/infiniband/hw/efa/efa_com_cmd.c +1 -0
@@ -31,6 +31,7 @@ int efa_com_create_qp(struct efa_com_dev *edev, create_qp_cmd.qp_alloc_size.recv_queue_depth = params->rq_depth; create_qp_cmd.uar = params->uarn; + create_qp_cmd.sl = params->sl;if (params->unsolicited_write_recv)EFA_SET (&create_qp_cmd.flags,EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV ,1 );
drivers/infiniband/hw/efa/efa_com_cmd.h +1 -0
@@ -27,6 +27,7 @@ struct efa_com_create_qp_params {u16 pd;u16 uarn;u8 qp_type; +u8 sl; u8 unsolicited_write_recv : 1; };
drivers/infiniband/hw/efa/efa_verbs.c +3 -1
@@ -676,7 +676,7 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, goto err_out; } -if (cmd.comp_mask || !is_reserved_cleared (cmd.reserved_90)) { +if (cmd.comp_mask || !is_reserved_cleared (cmd.reserved_98)) {ibdev_dbg (&dev->ibdev,"Incompatible ABI params, unknown fields in udata\n" ); err = -EINVAL ;
@@ -732,6 +732,8 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, create_qp_params.rq_base_addr = qp->rq_dma_addr; } + create_qp_params.sl = cmd.sl; +if (cmd.flags &EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV ) create_qp_params.unsolicited_write_recv = true;
include/uapi/rdma/efa-abi.h +2 -1
@@ -95,7 +95,8 @@ struct efa_ibv_create_qp {__u32 sq_ring_size;/* bytes */ __u32 driver_qp_type;__u16 flags; -__u8 reserved_90[6 ]; +__u8 sl; +__u8 reserved_98[5 ]; };struct efa_ibv_create_qp_resp {
Signature · 1 line
2.40.1
Quoted 6 lines · Michael Margolin
Align the device interface and add an option to pass QP service level from user to the device.
Michael Margolin (2):
RDMA/efa: Update device interface
RDMA/efa: Add option to set QP service level on create
Applied to for-next
Thanks,
Jason