RDMA/efa: Add EFA query MR support
Subsystems (2)
Michael Margolin
maintainerPATCH v2 1/1 RDMA/efa: Add EFA query MR support
Add EFA driver uapi definitions and register a new query MR method that currently returns the physical PCI buses' IDs the device is using to reach the MR. Update admin definitions and efa-abi accordingly.
--- drivers/infiniband/hw/efa/efa.h | 5 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 31 ++++++++ drivers/infiniband/hw/efa/efa_com_cmd.c | 6 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 4 + drivers/infiniband/hw/efa/efa_main.c | 5 ++ drivers/infiniband/hw/efa/efa_verbs.c | 77 +++++++++++++++++++ include/uapi/rdma/efa-abi.h | 19 +++++ 7 files changed, 146 insertions(+), 1 deletion(-)
drivers/infiniband/hw/efa/efa.h +4 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_
@@ -83,6 +83,9 @@ struct efa_pd {struct efa_mr {struct ib_mr ibmr;struct ib_umem *umem; +u16 recv_pci_bus_id; +u16 rdma_read_pci_bus_id; +u16 rdma_recv_pci_bus_id; };struct efa_cq {
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +31 -0
@@ -415,6 +415,32 @@ struct efa_admin_reg_mr_resp {* memory region */ u32 r_key; + +/* +* Mask indicating which fields have valid values +* 0 : recv_pci_bus_id +* 1 : rdma_read_pci_bus_id +* 2 : rdma_recv_pci_bus_id +*/ +u8 validity; + +/* +* Physical PCIe bus used by the device to reach the MR for receive +* operation +*/ +u8 recv_pci_bus_id; + +/* +* Physical PCIe bus used by the device to reach the MR for RDMA read +* operation +*/ +u8 rdma_read_pci_bus_id; + +/* +* Physical PCIe bus used by the device to reach the MR for RDMA write +* receive +*/ +u8 rdma_recv_pci_bus_id; };struct efa_admin_dereg_mr_cmd {
@@ -999,6 +1025,11 @@ struct efa_admin_host_info {#define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1)#define EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE_MASK BIT(2) +/* reg_mr_resp */ +#define EFA_ADMIN_REG_MR_RESP_RECV_PCI_BUS_ID_MASK BIT(0) +#define EFA_ADMIN_REG_MR_RESP_RDMA_READ_PCI_BUS_ID_MASK BIT(1) +#define EFA_ADMIN_REG_MR_RESP_RDMA_RECV_PCI_BUS_ID_MASK BIT(2) +/* create_cq_cmd */ #define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK BIT(6)
drivers/infiniband/hw/efa/efa_com_cmd.c +6 -0
@@ -270,6 +270,12 @@ int efa_com_register_mr(struct efa_com_dev *edev, result->l_key = cmd_completion.l_key; result->r_key = cmd_completion.r_key; + result->recv_pci_bus_id =EFA_GET (&cmd_completion.validity,EFA_ADMIN_REG_MR_RESP_RECV_PCI_BUS_ID ) ? + cmd_completion.recv_pci_bus_id :EFA_INVALID_PCI_BUS_ID ; + result->rdma_read_pci_bus_id =EFA_GET (&cmd_completion.validity,EFA_ADMIN_REG_MR_RESP_RDMA_READ_PCI_BUS_ID ) ? + cmd_completion.rdma_read_pci_bus_id :EFA_INVALID_PCI_BUS_ID ; + result->rdma_recv_pci_bus_id =EFA_GET (&cmd_completion.validity,EFA_ADMIN_REG_MR_RESP_RDMA_RECV_PCI_BUS_ID ) ? + cmd_completion.rdma_recv_pci_bus_id :EFA_INVALID_PCI_BUS_ID ;return 0 ; }
drivers/infiniband/hw/efa/efa_com_cmd.h +4 -0
@@ -9,6 +9,7 @@#include "efa_com.h" #define EFA_GID_SIZE 16 +#define EFA_INVALID_PCI_BUS_ID 0xffffstruct efa_com_create_qp_params {u64 rq_base_addr;
@@ -210,6 +211,9 @@ struct efa_com_reg_mr_result {* accessed memory region */ u32 r_key; +u16 recv_pci_bus_id; +u16 rdma_read_pci_bus_id; +u16 rdma_recv_pci_bus_id; };struct efa_com_dereg_mr_params {
drivers/infiniband/hw/efa/efa_main.c +5 -0
@@ -9,6 +9,7 @@#include <linux/version.h> #include <rdma/ib_user_verbs.h> +#include <rdma/uverbs_ioctl.h> #include "efa.h"
@@ -36,6 +37,8 @@ MODULE_DEVICE_TABLE(pci, efa_pci_tbl); (BIT (EFA_ADMIN_FATAL_ERROR ) |BIT (EFA_ADMIN_WARNING ) | \ BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE)) +extern const struct uapi_definition efa_uapi_defs[]; +/* This handler will called for unknown event group or unimplemented handlers */ static void unimplemented_aenq_handler (void *data,struct efa_admin_aenq_entry *aenq_e)
@@ -432,6 +435,8 @@ static int efa_ib_device_add(struct efa_dev *dev)ib_set_device_ops (&dev->ibdev, &efa_dev_ops); + dev->ibdev.driver_def = efa_uapi_defs; + err =ib_register_device (&dev->ibdev,"efa_%d" , &pdev->dev);if (err) goto err_destroy_eqs;
drivers/infiniband/hw/efa/efa_verbs.c +77 -0
98 lines not on this page · raw message
include/uapi/rdma/efa-abi.h +19 -0
@@ -7,6 +7,7 @@#define EFA_ABI_USER_H #include <linux/types.h> +#include <rdma/ib_user_ioctl_cmds.h> /* * Increment this value if any changes that break userspace ABI
@@ -134,4 +135,22 @@ struct efa_ibv_ex_query_device_resp { __u32 device_caps; }; +enum { +EFA_QUERY_MR_VALIDITY_RECV_PCI_BUS_ID =1 <<0 , +EFA_QUERY_MR_VALIDITY_RDMA_READ_PCI_BUS_ID =1 <<1 , +EFA_QUERY_MR_VALIDITY_RDMA_RECV_PCI_BUS_ID =1 <<2 , +}; + +enum efa_query_mr_attrs { +EFA_IB_ATTR_QUERY_MR_HANDLE = (1U <<UVERBS_ID_NS_SHIFT ), +EFA_IB_ATTR_QUERY_MR_RESP_PCI_BUS_ID_VALIDITY , +EFA_IB_ATTR_QUERY_MR_RESP_RECV_PCI_BUS_ID , +EFA_IB_ATTR_QUERY_MR_RESP_RDMA_READ_PCI_BUS_ID , +EFA_IB_ATTR_QUERY_MR_RESP_RDMA_RECV_PCI_BUS_ID , +}; + +enum efa_mr_methods { +EFA_IB_METHOD_MR_QUERY = (1U <<UVERBS_ID_NS_SHIFT ), +}; +#endif /* EFA_ABI_USER_H */
Signature · 1 line
2.40.1
Quoted 16 lines · Michael Margolin
Add EFA driver uapi definitions and register a new query MR method that currently returns the physical PCI buses' IDs the device is using to reach the MR. Update admin definitions and efa-abi accordingly.
Reviewed-by Anas Mousa <[email protected]>Reviewed-by Firas Jahjah <[email protected]>Signed-off-by Michael Margolin <[email protected]>--- drivers/infiniband/hw/efa/efa.h | 5 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 31 ++++++++ drivers/infiniband/hw/efa/efa_com_cmd.c | 6 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 4 + drivers/infiniband/hw/efa/efa_main.c | 5 ++ drivers/infiniband/hw/efa/efa_verbs.c | 77 +++++++++++++++++++ include/uapi/rdma/efa-abi.h | 19 +++++ 7 files changed, 146 insertions(+), 1 deletion(-)
<...>
Quoted 7 lines
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +7
-0+/* +* Mask indicating which fields have valid values +* 0 : recv_pci_bus_id +* 1 : rdma_read_pci_bus_id +* 2 : rdma_recv_pci_bus_id +*/ +u8 validity;
<...>
Quoted 2 lines
#define
EFA_GID_SIZE16
+#defineEFA_INVALID_PCI_BUS_ID0xffff
Is 0xffff value guaranteed by PCI subsystem to be invalid? Why don't you provide "validity" field to userspace instead?
Thanks
Margolin, Michael
maintainerQuoted 30 lines · Leon Romanovsky
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 16 lines · Michael Margolin
Add EFA driver uapi definitions and register a new query MR method that currently returns the physical PCI buses' IDs the device is using to reach the MR. Update admin definitions and efa-abi accordingly.
Reviewed-by Anas Mousa <[email protected]>Reviewed-by Firas Jahjah <[email protected]>Signed-off-by Michael Margolin <[email protected]>--- drivers/infiniband/hw/efa/efa.h | 5 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 31 ++++++++ drivers/infiniband/hw/efa/efa_com_cmd.c | 6 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 4 + drivers/infiniband/hw/efa/efa_main.c | 5 ++ drivers/infiniband/hw/efa/efa_verbs.c | 77 +++++++++++++++++++ include/uapi/rdma/efa-abi.h | 19 +++++ 7 files changed, 146 insertions(+), 1 deletion(-)<...>
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h struct efa_admin_reg_mr_resp { · line 425
Quoted 7 lines
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +7
-0+/* +* Mask indicating which fields have valid values +* 0 : recv_pci_bus_id +* 1 : rdma_read_pci_bus_id +* 2 : rdma_recv_pci_bus_id +*/ +u8 validity;
Quoted 6 lines
<...>
Quoted 2 lines
#define
EFA_GID_SIZE16
+#defineEFA_INVALID_PCI_BUS_ID0xffffIs
0xffffvalue guaranteed by PCI subsystem to be invalid? Why don't you provide "validity" field to userspace instead?
The 0xffff value in only used internally in the driver to indicate an invalid id and isn't exposed to userspace. For userspace there is a validity field as you suggested:
+ return uverbs_copy_to(attrs, EFA_IB_ATTR_QUERY_MR_RESP_PCI_BUS_ID_VALIDITY, + &pci_bus_id_validity, sizeof(pci_bus_id_validity));
Thanks
Quoted 39 lines · Michael
Quoted 30 lines · Leon Romanovsky
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 16 lines · Michael Margolin
Add EFA driver uapi definitions and register a new query MR method that currently returns the physical PCI buses' IDs the device is using to reach the MR. Update admin definitions and efa-abi accordingly.
Reviewed-by Anas Mousa <[email protected]>Reviewed-by Firas Jahjah <[email protected]>Signed-off-by Michael Margolin <[email protected]>--- drivers/infiniband/hw/efa/efa.h | 5 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 31 ++++++++ drivers/infiniband/hw/efa/efa_com_cmd.c | 6 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 4 + drivers/infiniband/hw/efa/efa_main.c | 5 ++ drivers/infiniband/hw/efa/efa_verbs.c | 77 +++++++++++++++++++ include/uapi/rdma/efa-abi.h | 19 +++++ 7 files changed, 146 insertions(+), 1 deletion(-)<...>
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h struct efa_admin_reg_mr_resp { · line 425
Quoted 7 lines
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +7
-0+/* +* Mask indicating which fields have valid values +* 0 : recv_pci_bus_id +* 1 : rdma_read_pci_bus_id +* 2 : rdma_recv_pci_bus_id +*/ +u8 validity;Quoted 6 lines
<...>
Quoted 2 lines
#define
EFA_GID_SIZE16
+#defineEFA_INVALID_PCI_BUS_ID0xffffIs
0xffffvalue guaranteed by PCI subsystem to be invalid? Why don't you provide "validity" field to userspace instead?
Quoted 8 lines
The
0xffffvalue in only used internally in the driver to indicate an invalid id and isn't exposed to userspace. For userspace there is a validity field as you suggested:+ return uverbs_copy_to(attrs,
EFA_IB_ATTR_QUERY_MR_RESP_PCI_BUS_ID_VALIDITY, +&pci_bus_id_validity,sizeof(pci_bus_id_validity));
So please rely on your EFA_GET(&cmd_completion.validity, EFA_ADMIN_XXX_PCI_BUS_ID) checks when you fill pci_bus_id_validity and not on 0xffff value which can be valid from PCI perspective.
Thanks
Quoted 4 lines
Thanks
Margolin, Michael
maintainerQuoted 50 lines · Leon Romanovsky
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 37 lines · Michael
Quoted 30 lines · Leon Romanovsky
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 16 lines · Michael Margolin
Add EFA driver uapi definitions and register a new query MR method that currently returns the physical PCI buses' IDs the device is using to reach the MR. Update admin definitions and efa-abi accordingly.
Reviewed-by Anas Mousa <[email protected]>Reviewed-by Firas Jahjah <[email protected]>Signed-off-by Michael Margolin <[email protected]>--- drivers/infiniband/hw/efa/efa.h | 5 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 31 ++++++++ drivers/infiniband/hw/efa/efa_com_cmd.c | 6 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 4 + drivers/infiniband/hw/efa/efa_main.c | 5 ++ drivers/infiniband/hw/efa/efa_verbs.c | 77 +++++++++++++++++++ include/uapi/rdma/efa-abi.h | 19 +++++ 7 files changed, 146 insertions(+), 1 deletion(-)<...>
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h struct efa_admin_reg_mr_resp { · line 425
Quoted 7 lines
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +7
-0+/* +* Mask indicating which fields have valid values +* 0 : recv_pci_bus_id +* 1 : rdma_read_pci_bus_id +* 2 : rdma_recv_pci_bus_id +*/ +u8 validity;Quoted 6 lines
<...>
Quoted 2 lines
#define
EFA_GID_SIZE16
+#defineEFA_INVALID_PCI_BUS_ID0xffffIs
0xffffvalue guaranteed by PCI subsystem to be invalid? Why don't you provide "validity" field to userspace instead?Quoted 8 lines
The
0xffffvalue in only used internally in the driver to indicate an invalid id and isn't exposed to userspace. For userspace there is a validity field as you suggested:+ return uverbs_copy_to(attrs,
EFA_IB_ATTR_QUERY_MR_RESP_PCI_BUS_ID_VALIDITY,
+&pci_bus_id_validity,sizeof(pci_bus_id_validity));
Quoted 5 lines
So please rely on your
EFA_GET(&cmd_completion.validity, EFA_ADMIN_XXX_PCI_BUS_ID)checks when you fillpci_bus_id_validityand not on0xffffvalue which can be valid from PCI perspective.Thanks
0xffff can't practically be a valid PCI id in this context, but anyway changed to use validity fields through all the code to be clear on that andto be more consistent. Sending v3 patch.
Thanks
Quoted 52 lines · Michael
Quoted 50 lines · Leon Romanovsky
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 45 lines · Michael
Quoted 30 lines · Leon Romanovsky
CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
Quoted 16 lines · Michael Margolin
Add EFA driver uapi definitions and register a new query MR method that currently returns the physical PCI buses' IDs the device is using to reach the MR. Update admin definitions and efa-abi accordingly.
Reviewed-by Anas Mousa <[email protected]>Reviewed-by Firas Jahjah <[email protected]>Signed-off-by Michael Margolin <[email protected]>--- drivers/infiniband/hw/efa/efa.h | 5 +- .../infiniband/hw/efa/efa_admin_cmds_defs.h | 31 ++++++++ drivers/infiniband/hw/efa/efa_com_cmd.c | 6 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 4 + drivers/infiniband/hw/efa/efa_main.c | 5 ++ drivers/infiniband/hw/efa/efa_verbs.c | 77 +++++++++++++++++++ include/uapi/rdma/efa-abi.h | 19 +++++ 7 files changed, 146 insertions(+), 1 deletion(-)<...>
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h struct efa_admin_reg_mr_resp { · line 425
Quoted 7 lines
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +7
-0+/* +* Mask indicating which fields have valid values +* 0 : recv_pci_bus_id +* 1 : rdma_read_pci_bus_id +* 2 : rdma_recv_pci_bus_id +*/ +u8 validity;Quoted 6 lines
<...>
Quoted 2 lines
#define
EFA_GID_SIZE16
+#defineEFA_INVALID_PCI_BUS_ID0xffffIs
0xffffvalue guaranteed by PCI subsystem to be invalid? Why don't you provide "validity" field to userspace instead?The
0xffffvalue in only used internally in the driver to indicate an invalid id and isn't exposed to userspace. For userspace there is a validity field as you suggested:+ return uverbs_copy_to(attrs,
EFA_IB_ATTR_QUERY_MR_RESP_PCI_BUS_ID_VALIDITY,
+&pci_bus_id_validity,sizeof(pci_bus_id_validity));
Quoted 7 lines
Quoted 5 lines
So please rely on your
EFA_GET(&cmd_completion.validity, EFA_ADMIN_XXX_PCI_BUS_ID)checks when you fillpci_bus_id_validityand not on0xffffvalue which can be valid from PCI perspective.Thanks
0xffffcan't practically be a valid PCI id in this context,
As long as PCI subsystem doesn't declare 0xffff as invalid, we can't assume it too.
Thanks