Skip to contentSkip to conversation

RDMA/efa: Add EFA query MR support

PATCH Applied Michael Margolin 3 participants 10 messages Jan 4, 2024 – Jan 10, 2024

Subsystems (2)
lore
Share & tools
lore

Michael Margolin

author maintainer
#0 lore raw

PATCH v4 1/1 RDMA/efa: Add EFA query MR support

Add EFA driver uapi definitions and register a new query MR method that currently returns the physical interconnects the device is using to reach the MR. Update admin definitions and efa-abi accordingly.

---
 drivers/infiniband/hw/efa/efa.h               | 12 +++-
 .../infiniband/hw/efa/efa_admin_cmds_defs.h   | 33 ++++++++-
 drivers/infiniband/hw/efa/efa_com_cmd.c       | 11 ++-
 drivers/infiniband/hw/efa/efa_com_cmd.h       | 12 +++-
 drivers/infiniband/hw/efa/efa_main.c          |  7 +-
 drivers/infiniband/hw/efa/efa_verbs.c         | 71 ++++++++++++++++++-
 include/uapi/rdma/efa-abi.h                   | 21 +++++-
 7 files changed, 160 insertions(+), 7 deletions(-)
drivers/infiniband/hw/efa/efa.h +11 -1
index 7352a1f5d811..e2bdec32ae80 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -80,9 +80,19 @@ struct efa_pd {
 	u16 pdn;
 };
 
+struct efa_mr_interconnect_info {
+	u16 recv_ic_id;
+	u16 rdma_read_ic_id;
+	u16 rdma_recv_ic_id;
+	u8 recv_ic_id_valid : 1;
+	u8 rdma_read_ic_id_valid : 1;
+	u8 rdma_recv_ic_id_valid : 1;
+};
+
 struct efa_mr {
 	struct ib_mr ibmr;
 	struct ib_umem *umem;
+	struct efa_mr_interconnect_info ic_info;
 };
 
 struct efa_cq {
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +32 -1
index 9c65bd27bae0..7377c8a9f4d5 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_ADMIN_CMDS_H_
@@ -415,6 +415,32 @@ struct efa_admin_reg_mr_resp {
 	 * memory region
 	 */
 	u32 r_key;
+
+	/*
+	 * Mask indicating which fields have valid values
+	 * 0 : recv_ic_id
+	 * 1 : rdma_read_ic_id
+	 * 2 : rdma_recv_ic_id
+	 */
+	u8 validity;
+
+	/*
+	 * Physical interconnect used by the device to reach the MR for receive
+	 * operation
+	 */
+	u8 recv_ic_id;
+
+	/*
+	 * Physical interconnect used by the device to reach the MR for RDMA
+	 * read operation
+	 */
+	u8 rdma_read_ic_id;
+
+	/*
+	 * Physical interconnect used by the device to reach the MR for RDMA
+	 * write receive
+	 */
+	u8 rdma_recv_ic_id;
 };
 
 struct efa_admin_dereg_mr_cmd {
@@ -999,6 +1025,11 @@ struct efa_admin_host_info {
 #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK       BIT(1)
 #define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_MASK        BIT(2)
 
+/* reg_mr_resp */
+#define EFA_ADMIN_REG_MR_RESP_RECV_IC_ID_MASK               BIT(0)
+#define EFA_ADMIN_REG_MR_RESP_RDMA_READ_IC_ID_MASK          BIT(1)
+#define EFA_ADMIN_REG_MR_RESP_RDMA_RECV_IC_ID_MASK          BIT(2)
+
 /* create_cq_cmd */
 #define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)
 #define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK                   BIT(6)
drivers/infiniband/hw/efa/efa_com_cmd.c +10 -1
index 576811885d59..d3398c7b0bd0 100644
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include "efa_com.h"
@@ -270,6 +270,15 @@ int efa_com_register_mr(struct efa_com_dev *edev,
 
 	result->l_key = cmd_completion.l_key;
 	result->r_key = cmd_completion.r_key;
+	result->ic_info.recv_ic_id = cmd_completion.recv_ic_id;
+	result->ic_info.rdma_read_ic_id = cmd_completion.rdma_read_ic_id;
+	result->ic_info.rdma_recv_ic_id = cmd_completion.rdma_recv_ic_id;
+	result->ic_info.recv_ic_id_valid = EFA_GET(&cmd_completion.validity,
+						   EFA_ADMIN_REG_MR_RESP_RECV_IC_ID);
+	result->ic_info.rdma_read_ic_id_valid = EFA_GET(&cmd_completion.validity,
+							EFA_ADMIN_REG_MR_RESP_RDMA_READ_IC_ID);
+	result->ic_info.rdma_recv_ic_id_valid = EFA_GET(&cmd_completion.validity,
+							EFA_ADMIN_REG_MR_RESP_RDMA_RECV_IC_ID);
 
 	return 0;
 }
drivers/infiniband/hw/efa/efa_com_cmd.h +11 -1
index fc97f37bb39b..720a99ba0f7d 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_COM_CMD_H_
@@ -199,6 +199,15 @@ struct efa_com_reg_mr_params {
 	u8 indirect;
 };
 
+struct efa_com_mr_interconnect_info {
+	u16 recv_ic_id;
+	u16 rdma_read_ic_id;
+	u16 rdma_recv_ic_id;
+	u8 recv_ic_id_valid : 1;
+	u8 rdma_read_ic_id_valid : 1;
+	u8 rdma_recv_ic_id_valid : 1;
+};
+
 struct efa_com_reg_mr_result {
 	/*
 	 * To be used in conjunction with local buffers references in SQ and
@@ -210,6 +219,7 @@ struct efa_com_reg_mr_result {
 	 * accessed memory region
 	 */
 	u32 r_key;
+	struct efa_com_mr_interconnect_info ic_info;
 };
 
 struct efa_com_dereg_mr_params {
drivers/infiniband/hw/efa/efa_main.c +6 -1
index 15ee92081118..7b1910a86216 100644
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
 /*
- * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #include <linux/module.h>
@@ -9,6 +9,7 @@
 #include <linux/version.h>
 
 #include <rdma/ib_user_verbs.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "efa.h"
 
@@ -36,6 +37,8 @@ MODULE_DEVICE_TABLE(pci, efa_pci_tbl);
 	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \
 	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))
 
+extern const struct uapi_definition efa_uapi_defs[];
+
 /* This handler will called for unknown event group or unimplemented handlers */
 static void unimplemented_aenq_handler(void *data,
 				       struct efa_admin_aenq_entry *aenq_e)
@@ -432,6 +435,8 @@ static int efa_ib_device_add(struct efa_dev *dev)
 
 	ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
 
+	dev->ibdev.driver_def = efa_uapi_defs;
+
 	err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev);
 	if (err)
 		goto err_destroy_eqs;
drivers/infiniband/hw/efa/efa_verbs.c +70 -1

97 lines not on this page · raw message

include/uapi/rdma/efa-abi.h +20 -1
index d94c32f28804..701e2d567e41 100644
@@ -1,12 +1,13 @@
 /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
 /*
- * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef EFA_ABI_USER_H
 #define EFA_ABI_USER_H
 
 #include <linux/types.h>
+#include <rdma/ib_user_ioctl_cmds.h>
 
 /*
  * Increment this value if any changes that break userspace ABI
@@ -134,4 +135,22 @@ struct efa_ibv_ex_query_device_resp {
 	__u32 device_caps;
 };
 
+enum {
+	EFA_QUERY_MR_VALIDITY_RECV_IC_ID = 1 << 0,
+	EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID = 1 << 1,
+	EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID = 1 << 2,
+};
+
+enum efa_query_mr_attrs {
+	EFA_IB_ATTR_QUERY_MR_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
+	EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
+	EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
+	EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,
+	EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,
+};
+
+enum efa_mr_methods {
+	EFA_IB_METHOD_MR_QUERY = (1U << UVERBS_ID_NS_SHIFT),
+};
+
 #endif /* EFA_ABI_USER_H */
Signature · 1 line
2.40.1
#1 lore raw
Quoted 7 lines · Michael Margolin

Add EFA driver uapi definitions and register a new query MR method that currently returns the physical interconnects the device is using to reach the MR. Update admin definitions and efa-abi accordingly.

Quoted 9 lines
---
 drivers/infiniband/hw/efa/efa.h               | 12 +++-
 .../infiniband/hw/efa/efa_admin_cmds_defs.h   | 33 ++++++++-
 drivers/infiniband/hw/efa/efa_com_cmd.c       | 11 ++-
 drivers/infiniband/hw/efa/efa_com_cmd.h       | 12 +++-
 drivers/infiniband/hw/efa/efa_main.c          |  7 +-
 drivers/infiniband/hw/efa/efa_verbs.c         | 71 ++++++++++++++++++-
 include/uapi/rdma/efa-abi.h                   | 21 +++++-
 7 files changed, 160 insertions(+), 7 deletions(-)

It is already fourth version of this patch and not a single word about the changes. Please add changelog in your next submissions.

Applied this patch with the following change.

drivers/infiniband/hw/efa/efa_verbs.c +7 -7
index 8f4435706e4d..2f412db2edcd 100644
@@ -1748,7 +1748,7 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
 {
        struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE);
        struct efa_mr *mr = to_emr(ibmr);
-       u16 ic_id_validity;
+       u16 ic_id_validity = 0;
        int ret;
 
        ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
@@ -1766,12 +1766,12 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
        if (ret)
                return ret;
 
-       ic_id_validity = (mr->ic_info.recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RECV_IC_ID : 0) |
-                        (mr->ic_info.rdma_read_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID : 0) |
-                        (mr->ic_info.rdma_recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID : 0);
+       if (mr->ic_info.recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID;
+       if (mr->ic_info.rdma_read_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID;
+       if (mr->ic_info.rdma_recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID;
 
        return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
                              &ic_id_validity, sizeof(ic_id_validity));
author maintainer
#2 lore raw

Thanks Leon.

Quoted 59 lines · Leon Romanovsky

CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.

Quoted 7 lines · Michael Margolin

Add EFA driver uapi definitions and register a new query MR method that currently returns the physical interconnects the device is using to reach the MR. Update admin definitions and efa-abi accordingly.

Quoted 9 lines
---
 drivers/infiniband/hw/efa/efa.h               | 12 +++-
 .../infiniband/hw/efa/efa_admin_cmds_defs.h   | 33 ++++++++-
 drivers/infiniband/hw/efa/efa_com_cmd.c       | 11 ++-
 drivers/infiniband/hw/efa/efa_com_cmd.h       | 12 +++-
 drivers/infiniband/hw/efa/efa_main.c          |  7 +-
 drivers/infiniband/hw/efa/efa_verbs.c         | 71 ++++++++++++++++++-
 include/uapi/rdma/efa-abi.h                   | 21 +++++-
 7 files changed, 160 insertions(+), 7 deletions(-)

It is already fourth version of this patch and not a single word about the changes. Please add changelog in your next submissions.

Applied this patch with the following change.

drivers/infiniband/hw/efa/efa_verbs.c +7 -7
index 8f4435706e4d..2f412db2edcd 100644
@@ -1748,7 +1748,7 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
 {
        struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE);
        struct efa_mr *mr = to_emr(ibmr);
-       u16 ic_id_validity;
+       u16 ic_id_validity = 0;
        int ret;
 
        ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
@@ -1766,12 +1766,12 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
        if (ret)
                return ret;
 
-       ic_id_validity = (mr->ic_info.recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RECV_IC_ID : 0) |
-                        (mr->ic_info.rdma_read_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID : 0) |
-                        (mr->ic_info.rdma_recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID : 0);
+       if (mr->ic_info.recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID;
+       if (mr->ic_info.rdma_read_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID;
+       if (mr->ic_info.rdma_recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID;
 
        return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
                              &ic_id_validity, sizeof(ic_id_validity));
maintainer
#3 lore raw
In reply to Leon Romanovsky
Quoted 42 lines · Leon Romanovsky
Quoted 7 lines · Michael Margolin

Add EFA driver uapi definitions and register a new query MR method that currently returns the physical interconnects the device is using to reach the MR. Update admin definitions and efa-abi accordingly.

Quoted 9 lines
---
 drivers/infiniband/hw/efa/efa.h               | 12 +++-
 .../infiniband/hw/efa/efa_admin_cmds_defs.h   | 33 ++++++++-
 drivers/infiniband/hw/efa/efa_com_cmd.c       | 11 ++-
 drivers/infiniband/hw/efa/efa_com_cmd.h       | 12 +++-
 drivers/infiniband/hw/efa/efa_main.c          |  7 +-
 drivers/infiniband/hw/efa/efa_verbs.c         | 71 ++++++++++++++++++-
 include/uapi/rdma/efa-abi.h                   | 21 +++++-
 7 files changed, 160 insertions(+), 7 deletions(-)

It is already fourth version of this patch and not a single word about the changes. Please add changelog in your next submissions.

Applied this patch with the following change.

drivers/infiniband/hw/efa/efa_verbs.c +1 -3
index 8f4435706e4d..2f412db2edcd 100644
@@ -1748,7 +1748,7 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
 {
        struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE);
        struct efa_mr *mr = to_emr(ibmr);
-       u16 ic_id_validity;
+       u16 ic_id_validity = 0;
        int ret;
 
        ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
@@ -1766,12 +1766,12 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
        if (ret)
                return ret;
 
-       ic_id_validity = (mr->ic_info.recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RECV_IC_ID : 0) |

drivers/infiniband/hw/efa/efa_verbs.c static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att · line 1774

Quoted 10 lines
drivers/infiniband/hw/efa/efa_verbs.c +6 -4
-                        (mr->ic_info.rdma_read_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID : 0) |
-                        (mr->ic_info.rdma_recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID : 0);
+       if (mr->ic_info.recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID;
+       if (mr->ic_info.rdma_read_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID;
+       if (mr->ic_info.rdma_recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID;

I was saying in the rdma-core PR that this field shouldn't even exist..

Jason

#4 lore raw
Quoted 41 lines · Jason Gunthorpe
Quoted 40 lines · Leon Romanovsky
Quoted 7 lines · Michael Margolin

Add EFA driver uapi definitions and register a new query MR method that currently returns the physical interconnects the device is using to reach the MR. Update admin definitions and efa-abi accordingly.

Quoted 9 lines
---
 drivers/infiniband/hw/efa/efa.h               | 12 +++-
 .../infiniband/hw/efa/efa_admin_cmds_defs.h   | 33 ++++++++-
 drivers/infiniband/hw/efa/efa_com_cmd.c       | 11 ++-
 drivers/infiniband/hw/efa/efa_com_cmd.h       | 12 +++-
 drivers/infiniband/hw/efa/efa_main.c          |  7 +-
 drivers/infiniband/hw/efa/efa_verbs.c         | 71 ++++++++++++++++++-
 include/uapi/rdma/efa-abi.h                   | 21 +++++-
 7 files changed, 160 insertions(+), 7 deletions(-)

It is already fourth version of this patch and not a single word about the changes. Please add changelog in your next submissions.

Applied this patch with the following change.

drivers/infiniband/hw/efa/efa_verbs.c +1 -1
index 8f4435706e4d..2f412db2edcd 100644
@@ -1748,7 +1748,7 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
 {
        struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE);
        struct efa_mr *mr = to_emr(ibmr);
-       u16 ic_id_validity;
+       u16 ic_id_validity = 0;
        int ret;
 
        ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
@@ -1766,12 +1766,12 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
        if (ret)
                return ret;
 
Quoted 15 lines

drivers/infiniband/hw/efa/efa_verbs.c static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att · line 1774

Quoted 12 lines
drivers/infiniband/hw/efa/efa_verbs.c +6 -6
-       ic_id_validity = (mr->ic_info.recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RECV_IC_ID : 0) |
-                        (mr->ic_info.rdma_read_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID : 0) |
-                        (mr->ic_info.rdma_recv_ic_id_valid ?
-                         EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID : 0);
+       if (mr->ic_info.recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID;
+       if (mr->ic_info.rdma_read_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID;
+       if (mr->ic_info.rdma_recv_ic_id_valid)
+               ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID;

I was saying in the rdma-core PR that this field shouldn't even exist..

Something like that?

drivers/infiniband/hw/efa/efa_verbs.c +22 -29
index 2f412db2edcd..97619b29afe0 100644
@@ -1748,33 +1748,29 @@ static int UVERBS_HANDLER(EFA_IB_METHOD_MR_QUERY)(struct uverbs_attr_bundle *att
 {
 	struct ib_mr *ibmr = uverbs_attr_get_obj(attrs, EFA_IB_ATTR_QUERY_MR_HANDLE);
 	struct efa_mr *mr = to_emr(ibmr);
-	u16 ic_id_validity = 0;
 	int ret;
 
-	ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
-			     &mr->ic_info.recv_ic_id, sizeof(mr->ic_info.recv_ic_id));
-	if (ret)
-		return ret;
-
-	ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,
-			     &mr->ic_info.rdma_read_ic_id, sizeof(mr->ic_info.rdma_read_ic_id));
-	if (ret)
-		return ret;
-
-	ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,
-			     &mr->ic_info.rdma_recv_ic_id, sizeof(mr->ic_info.rdma_recv_ic_id));
-	if (ret)
-		return ret;
+	if (mr->ic_info.recv_ic_id_valid) {
+		ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
+				     &mr->ic_info.recv_ic_id, sizeof(mr->ic_info.recv_ic_id));
+		if (ret)
+			return ret;
+	}
 
-	if (mr->ic_info.recv_ic_id_valid)
-		ic_id_validity |= EFA_QUERY_MR_VALIDITY_RECV_IC_ID;
-	if (mr->ic_info.rdma_read_ic_id_valid)
-		ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_READ_IC_ID;
-	if (mr->ic_info.rdma_recv_ic_id_valid)
-		ic_id_validity |= EFA_QUERY_MR_VALIDITY_RDMA_RECV_IC_ID;
+	if (mr->ic_info.rdma_read_ic_id_valid) {
+		ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,
+				     &mr->ic_info.rdma_read_ic_id, sizeof(mr->ic_info.rdma_read_ic_id));
+		if (ret)
+			return ret;
+	}
 
-	return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
-			      &ic_id_validity, sizeof(ic_id_validity));
+	if (mr->ic_info.rdma_recv_ic_id_valid) {
+		ret = uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,
+				     &mr->ic_info.rdma_recv_ic_id, sizeof(mr->ic_info.rdma_recv_ic_id));
+		if (ret)
+			return ret;
+	}
+	return 0;
 }
 
 int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
@@ -2204,18 +2200,15 @@ DECLARE_UVERBS_NAMED_METHOD(EFA_IB_METHOD_MR_QUERY,
 					    UVERBS_OBJECT_MR,
 					    UVERBS_ACCESS_READ,
 					    UA_MANDATORY),
-			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
-						UVERBS_ATTR_TYPE(u16),
-						UA_MANDATORY),
 			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
 						UVERBS_ATTR_TYPE(u16),
-						UA_MANDATORY),
+						UA_OPTIONAL),
 			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,
 						UVERBS_ATTR_TYPE(u16),
-						UA_MANDATORY),
+						UA_OPTIONAL),
 			    UVERBS_ATTR_PTR_OUT(EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,
 						UVERBS_ATTR_TYPE(u16),
-						UA_MANDATORY));
+						UA_OPTIONAL));
 
 ADD_UVERBS_METHODS(efa_mr,
 		   UVERBS_OBJECT_MR,
include/uapi/rdma/efa-abi.h +0 -1
index 701e2d567e41..fbe5936e22cd 100644
@@ -143,7 +143,6 @@ enum {
 
 enum efa_query_mr_attrs {
 	EFA_IB_ATTR_QUERY_MR_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
-	EFA_IB_ATTR_QUERY_MR_RESP_IC_ID_VALIDITY,
 	EFA_IB_ATTR_QUERY_MR_RESP_RECV_IC_ID,
 	EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_IC_ID,
 	EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_IC_ID,
maintainer
#5 lore raw
Quoted 4 lines · Leon Romanovsky
Quoted 2 lines

I was saying in the rdma-core PR that this field shouldn't even exist..

Something like that?

Yeah, like that. However it is difficult to get the out valid uattr back in the rdma-core side.

This is best if the ID's can have well defined not-valid values such as 0 or -1.

Jason

#6 lore raw
Quoted 11 lines · Jason Gunthorpe
Quoted 4 lines · Leon Romanovsky
Quoted 2 lines

I was saying in the rdma-core PR that this field shouldn't even exist..

Something like that?

Yeah, like that. However it is difficult to get the out valid uattr back in the rdma-core side.

This is best if the ID's can have well defined not-valid values such as 0 or -1.

Michael tried something like that in previous versions by defining 0xffff as not valid.

I didn't like it because there's no promise from PCI core that it is invalid value.

Quoted 2 lines

Jason

#7 lore raw
Quoted 12 lines · Leon Romanovsky
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Leon Romanovsky
Quoted 2 lines

I was saying in the rdma-core PR that this field shouldn't even exist..

Something like that?

Yeah, like that. However it is difficult to get the out valid uattr back in the rdma-core side.

Quoted 6 lines
Quoted 2 lines

This is best if the ID's can have well defined not-valid values such as 0 or -1.

Michael tried something like that in previous versions by defining 0xffff as not valid.

I didn't like it because there's no promise from PCI core that it is invalid value.

Michael,
What do you think?

Thanks

Quoted 6 lines
Quoted 5 lines

Jason

author maintainer
#8 lore raw
Quoted 16 lines · Leon Romanovsky

CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.

Quoted 11 lines · Leon Romanovsky
Quoted 9 lines · Jason Gunthorpe
Quoted 3 lines · Leon Romanovsky
Quoted 2 lines

I was saying in the rdma-core PR that this field shouldn't even exist..

Something like that?

Yeah, like that. However it is difficult to get the out valid uattr back in the rdma-core side.

This is best if the ID's can have well defined not-valid values such as 0 or -1.

Quoted 7 lines
Quoted 3 lines

Michael tried something like that in previous versions by defining 0xffff as not valid.

I didn't like it because there's no promise from PCI core that it is invalid value.

Michael,
What do you think?

Thanks

In this case I prefer to keep the explicit validity for consistency with the device originated values.

In general I think it will be useful to have some convenient way to get attr validity from the ioctl mechanism in rdma-core.

Michael