Skip to contentSkip to conversation

RDMA/efa: Support QP service level

PATCH Applied Michael Margolin 2 participants 4 messages Oct 15, 2024 – Oct 21, 2024

Subsystems (2)
lore
Share & tools
lore
author maintainer
#1 lore raw

PATCH 1/2 RDMA/efa: Update device interface

In reply to Michael Margolin

Update device interface header files.

---
 .../infiniband/hw/efa/efa_admin_cmds_defs.h   |  54 +++++++--
 drivers/infiniband/hw/efa/efa_admin_defs.h    |   4 +-
 drivers/infiniband/hw/efa/efa_com_cmd.c       |   4 +-
 drivers/infiniband/hw/efa/efa_com_cmd.h       |   2 +-
 drivers/infiniband/hw/efa/efa_io_defs.h       | 106 ++++++++++++++++--
 drivers/infiniband/hw/efa/efa_verbs.c         |   2 +-
 6 files changed, 149 insertions(+), 23 deletions(-)
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +47 -7
index cd03a5429beb..88a9aee7e743 100644
@@ -30,7 +30,8 @@ enum efa_admin_aq_opcode {
 	EFA_ADMIN_DEALLOC_UAR                       = 17,
 	EFA_ADMIN_CREATE_EQ                         = 18,
 	EFA_ADMIN_DESTROY_EQ                        = 19,
-	EFA_ADMIN_MAX_OPCODE                        = 19,
+	EFA_ADMIN_ALLOC_MR                          = 20,
+	EFA_ADMIN_MAX_OPCODE                        = 20,
 };
 
 enum efa_admin_aq_feature_id {
@@ -150,8 +151,11 @@ struct efa_admin_create_qp_cmd {
 	/* UAR number */
 	u16 uar;
 
+	/* Requested service level for the QP, 0 is the default SL */
+	u8 sl;
+
 	/* MBZ */
-	u16 reserved;
+	u8 reserved;
 
 	/* MBZ */
 	u32 reserved2;
@@ -459,6 +463,41 @@ struct efa_admin_dereg_mr_resp {
 	struct efa_admin_acq_common_desc acq_common_desc;
 };
 
+/*
+ * Allocation of MemoryRegion, required for QP working with Virtual
+ * Addresses in kernel verbs semantics, ready for fast registration use.
+ */
+struct efa_admin_alloc_mr_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* Protection Domain */
+	u16 pd;
+
+	/* MBZ */
+	u16 reserved1;
+
+	/* Maximum number of pages this MR supports. */
+	u32 max_pages;
+};
+
+struct efa_admin_alloc_mr_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/*
+	 * L_Key, to be used in conjunction with local buffer references in
+	 * SQ and RQ WQE, or with virtual RQ/CQ rings
+	 */
+	u32 l_key;
+
+	/*
+	 * R_Key, to be used in RDMA messages to refer to remotely accessed
+	 * memory region
+	 */
+	u32 r_key;
+};
+
 struct efa_admin_create_cq_cmd {
 	struct efa_admin_aq_common_desc aq_common_desc;
 
@@ -483,8 +522,8 @@ struct efa_admin_create_cq_cmd {
 	 */
 	u8 cq_caps_2;
 
-	/* completion queue depth in # of entries. must be power of 2 */
-	u16 cq_depth;
+	/* Sub completion queue depth in # of entries. must be power of 2 */
+	u16 sub_cq_depth;
 
 	/* EQ number assigned to this cq */
 	u16 eqn;
@@ -519,8 +558,8 @@ struct efa_admin_create_cq_resp {
 
 	u16 cq_idx;
 
-	/* actual cq depth in number of entries */
-	u16 cq_actual_depth;
+	/* actual sub cq depth in number of entries */
+	u16 sub_cq_actual_depth;
 
 	/* CQ doorbell address, as offset to PCIe DB BAR */
 	u32 db_offset;
@@ -578,6 +617,8 @@ struct efa_admin_basic_stats {
 	u64 rx_pkts;
 
 	u64 rx_drops;
+
+	u64 qkey_viol;
 };
 
 struct efa_admin_messages_stats {
@@ -1057,7 +1098,6 @@ struct efa_admin_host_info {
 
 /* create_eq_cmd */
 #define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK       GENMASK(4, 0)
-#define EFA_ADMIN_CREATE_EQ_CMD_VIRT_MASK                   BIT(6)
 #define EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS_MASK      BIT(0)
 
 /* host_info */
drivers/infiniband/hw/efa/efa_admin_defs.h +2 -2
index 83f20c38a840..35700c93e639 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_ADMIN_H_
@@ -96,7 +96,7 @@ struct efa_admin_acq_entry {
 struct efa_admin_aenq_common_desc {
 	u16 group;
 
-	u16 syndrom;
+	u16 syndrome;
 
 	/*
 	 * 0 : phase
drivers/infiniband/hw/efa/efa_com_cmd.c +2 -2
index 5a774925cdea..206f377db27e 100644
@@ -163,7 +163,7 @@ int efa_com_create_cq(struct efa_com_dev *edev,
 	EFA_SET(&create_cmd.cq_caps_2,
 		EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS,
 		params->entry_size_in_bytes / 4);
-	create_cmd.cq_depth = params->cq_depth;
+	create_cmd.sub_cq_depth = params->sub_cq_depth;
 	create_cmd.num_sub_cqs = params->num_sub_cqs;
 	create_cmd.uar = params->uarn;
 	if (params->interrupt_mode_enabled) {
@@ -191,7 +191,7 @@ int efa_com_create_cq(struct efa_com_dev *edev,
 	}
 
 	result->cq_idx = cmd_completion.cq_idx;
-	result->actual_depth = params->cq_depth;
+	result->actual_depth = params->sub_cq_depth;
 	result->db_off = cmd_completion.db_offset;
 	result->db_valid = EFA_GET(&cmd_completion.flags,
 				   EFA_ADMIN_CREATE_CQ_RESP_DB_VALID);
drivers/infiniband/hw/efa/efa_com_cmd.h +1 -1
index 668d033f7477..2599f8e58cc4 100644
@@ -71,7 +71,7 @@ struct efa_com_create_cq_params {
 	/* cq physical base address in OS memory */
 	dma_addr_t dma_addr;
 	/* completion queue depth in # of entries */
-	u16 cq_depth;
+	u16 sub_cq_depth;
 	u16 num_sub_cqs;
 	u16 uarn;
 	u16 eqn;
drivers/infiniband/hw/efa/efa_io_defs.h +96 -10

187 lines not on this page · raw message

drivers/infiniband/hw/efa/efa_verbs.c +1 -1
index cc13415ff7e7..feb04cfdb8da 100644
@@ -1167,7 +1167,7 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	}
 
 	params.uarn = cq->ucontext->uarn;
-	params.cq_depth = entries;
+	params.sub_cq_depth = entries;
 	params.dma_addr = cq->dma_addr;
 	params.entry_size_in_bytes = cmd.cq_entry_size;
 	params.num_sub_cqs = cmd.num_sub_cqs;
Signature · 1 line
2.40.1
author maintainer
#2 lore raw

PATCH 2/2 RDMA/efa: Add option to set QP service level on create

In reply to Michael Margolin

Using modify QP with AH attributes and IB_QP_AV flag set doesn't make much sense for connectionless QP types like SRD. Add SL parameter to EFA create QP user ABI and pass it to the device.

---
 drivers/infiniband/hw/efa/efa_com_cmd.c | 1 +
 drivers/infiniband/hw/efa/efa_com_cmd.h | 1 +
 drivers/infiniband/hw/efa/efa_verbs.c   | 4 +++-
 include/uapi/rdma/efa-abi.h             | 3 ++-
 4 files changed, 7 insertions(+), 2 deletions(-)
drivers/infiniband/hw/efa/efa_com_cmd.c +1 -0
index 206f377db27e..9e04edb9dbda 100644
@@ -31,6 +31,7 @@ int efa_com_create_qp(struct efa_com_dev *edev,
 	create_qp_cmd.qp_alloc_size.recv_queue_depth =
 			params->rq_depth;
 	create_qp_cmd.uar = params->uarn;
+	create_qp_cmd.sl = params->sl;
 
 	if (params->unsolicited_write_recv)
 		EFA_SET(&create_qp_cmd.flags, EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV, 1);
drivers/infiniband/hw/efa/efa_com_cmd.h +1 -0
index 2599f8e58cc4..25f02c0d9698 100644
@@ -27,6 +27,7 @@ struct efa_com_create_qp_params {
 	u16 pd;
 	u16 uarn;
 	u8 qp_type;
+	u8 sl;
 	u8 unsolicited_write_recv : 1;
 };
 
drivers/infiniband/hw/efa/efa_verbs.c +3 -1
index feb04cfdb8da..ca3af866a5df 100644
@@ -676,7 +676,7 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 		goto err_out;
 	}
 
-	if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_90)) {
+	if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_98)) {
 		ibdev_dbg(&dev->ibdev,
 			  "Incompatible ABI params, unknown fields in udata\n");
 		err = -EINVAL;
@@ -732,6 +732,8 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 		create_qp_params.rq_base_addr = qp->rq_dma_addr;
 	}
 
+	create_qp_params.sl = cmd.sl;
+
 	if (cmd.flags & EFA_CREATE_QP_WITH_UNSOLICITED_WRITE_RECV)
 		create_qp_params.unsolicited_write_recv = true;
 
include/uapi/rdma/efa-abi.h +2 -1
index d689b8b34189..11b94b0b035b 100644
@@ -95,7 +95,8 @@ struct efa_ibv_create_qp {
 	__u32 sq_ring_size; /* bytes */
 	__u32 driver_qp_type;
 	__u16 flags;
-	__u8 reserved_90[6];
+	__u8 sl;
+	__u8 reserved_98[5];
 };
 
 struct efa_ibv_create_qp_resp {
Signature · 1 line
2.40.1

Michael Margolin

author maintainer
#0 lore raw

cover RDMA/efa: Support QP service level

Align the device interface and add an option to pass QP service level from user to the device.

Michael Margolin (2):
RDMA/efa: Update device interface
RDMA/efa: Add option to set QP service level on create

 .../infiniband/hw/efa/efa_admin_cmds_defs.h   |  54 +++++++--
 drivers/infiniband/hw/efa/efa_admin_defs.h    |   4 +-
 drivers/infiniband/hw/efa/efa_com_cmd.c       |   5 +-
 drivers/infiniband/hw/efa/efa_com_cmd.h       |   3 +-
 drivers/infiniband/hw/efa/efa_io_defs.h       | 106 ++++++++++++++++--
 drivers/infiniband/hw/efa/efa_verbs.c         |   6 +-
 include/uapi/rdma/efa-abi.h                   |   3 +-
 7 files changed, 156 insertions(+), 25 deletions(-)
Signature · 1 line
2.40.1
maintainer Applied
#3 lore raw
Quoted 6 lines · Michael Margolin

Align the device interface and add an option to pass QP service level from user to the device.

Michael Margolin (2):
RDMA/efa: Update device interface
RDMA/efa: Add option to set QP service level on create

Applied to for-next

Thanks,
Jason