RDMA-CM unique device identifier
cover RDMA-CM unique device identifier
From: Leon Romanovsky <[email protected]>
Hi,
This series extends librdmacm to unique identify the IBV device connected to the CMA device.
https://github.com/linux-rdma/rdma-core/pull/749
Thanks
Leon Romanovsky (4):
Update kernel headers
libibverbs: Fix description of ibv_get_device_guid man page
libibverbs: Get stable IB device index
librdmacm: Rely on IB device index if available
debian/libibverbs1.symbols | 2 + kernel-headers/rdma/rdma_user_cm.h | 2 + libibverbs/CMakeLists.txt | 2 +- libibverbs/device.c | 7 ++++ libibverbs/libibverbs.map.in | 5 +++ libibverbs/man/CMakeLists.txt | 1 + libibverbs/man/ibv_get_device_guid.3.md | 4 +- libibverbs/man/ibv_get_device_index.3.md | 40 ++++++++++++++++++ libibverbs/man/ibv_get_device_list.3.md | 1 + libibverbs/verbs.h | 9 ++++ librdmacm/cma.c | 53 +++++++++++++++++++----- librdmacm/rdma_cma_abi.h | 2 + 12 files changed, 114 insertions(+), 14 deletions(-) create mode 100644 libibverbs/man/ibv_get_device_index.3.md
--
2.26.2
PATCH 1/4 Update kernel headers
From: Leon Romanovsky <[email protected]>
To commit ?? ("RDMA/ucma: Return stable IB device index as identifier")
--- kernel-headers/rdma/rdma_user_cm.h | 2 ++ 1 file changed, 2 insertions(+)
kernel-headers/rdma/rdma_user_cm.h +2 -0
@@ -164,6 +164,7 @@ struct rdma_ucm_query_route_resp {__u32 num_paths;__u8 port_num;__u8 reserved[3 ]; +__u32 ibdev_index; };struct rdma_ucm_query_addr_resp {
@@ -175,6 +176,7 @@ struct rdma_ucm_query_addr_resp {__u16 dst_size;struct __kernel_sockaddr_storage src_addr;struct __kernel_sockaddr_storage dst_addr; +__u32 ibdev_index; };struct rdma_ucm_query_path_resp {
--
2.26.2
PATCH 2/4 libibverbs: Fix description of ibv_get_device_guid man page
In reply to Leon RomanovskyFrom: Leon Romanovsky <[email protected]>
There is a copy/paste error in the description ofibv_get_device_guid(), fix it.
--- libibverbs/man/ibv_get_device_guid.3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
libibverbs/man/ibv_get_device_guid.3.md +1 -1
@@ -22,7 +22,7 @@ uint64_t ibv_get_device_guid(struct ibv_device *device); # DESCRIPTION -**ibv_get_device_name()** returns the Global Unique IDentifier (GUID) of the +**ibv_get_device_guid()** returns the Global Unique IDentifier (GUID) of the RDMA device *device*. # RETURN VALUE
--
2.26.2
PATCH 3/4 libibverbs: Get stable IB device index
In reply to Leon RomanovskyFrom: Leon Romanovsky <[email protected]>
The kernel which supports query over netlink interface returns stable IB device index as part of it. This index much better identifier than sysfs name that can be renamed and/or disappear. Up till now, this index was used internally by libibverbs, but the next patch will rely on this API in the librdmacm code.
The -1 as returned value means that kernel doesn't support indexes.
--- debian/libibverbs1.symbols | 2 ++ libibverbs/CMakeLists.txt | 2 +- libibverbs/device.c | 7 +++++ libibverbs/libibverbs.map.in | 5 +++ libibverbs/man/CMakeLists.txt | 1 + libibverbs/man/ibv_get_device_guid.3.md | 2 +- libibverbs/man/ibv_get_device_index.3.md | 40 ++++++++++++++++++++++++ libibverbs/man/ibv_get_device_list.3.md | 1 + libibverbs/verbs.h | 9 ++++++ 9 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 libibverbs/man/ibv_get_device_index.3.md
debian/libibverbs1.symbols +2 -0
@@ -6,6 +6,7 @@ libibverbs.so.1 libibverbs1 #MINVER# IBVERBS_1.6@IBVERBS_1.6 24 IBVERBS_1.7@IBVERBS_1.7 25 IBVERBS_1.8@IBVERBS_1.8 28 + IBVERBS_1.9@IBVERBS_1.9 30 (symver)IBVERBS_PRIVATE_25 25 ibv_ack_async_event@IBVERBS_1.0 1.1.6 ibv_ack_async_event@IBVERBS_1.1 1.1.6
@@ -58,6 +59,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_get_cq_event@IBVERBS_1.1 1.1.6 ibv_get_device_guid@IBVERBS_1.0 1.1.6 ibv_get_device_guid@IBVERBS_1.1 1.1.6 + ibv_get_device_index@IBVERBS_1.9 30 ibv_get_device_list@IBVERBS_1.0 1.1.6 ibv_get_device_list@IBVERBS_1.1 1.1.6 ibv_get_device_name@IBVERBS_1.0 1.1.6
libibverbs/CMakeLists.txt +1 -1
@@ -21,7 +21,7 @@ configure_file("libibverbs.map.in" rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map" # See Documentation/versioning.md - 1 1.8.${PACKAGE_VERSION} + 1 1.9.${PACKAGE_VERSION} all_providers.c cmd.c cmd_ah.c
libibverbs/device.c +7 -0
@@ -150,6 +150,13 @@ LATEST_SYMVER_FUNC(ibv_get_device_guid, 1_1, "IBVERBS_1.1",return htobe64 (guid); } +int ibv_get_device_index (struct ibv_device *device) +{ +struct verbs_sysfs_dev *sysfs_dev =verbs_get_device (device)->sysfs; + +return sysfs_dev->ibdev_idx; +} +int ibv_get_fw_ver (char *value,size_t len,struct verbs_sysfs_dev *sysfs_dev) {/*
libibverbs/libibverbs.map.in +5 -0
@@ -126,6 +126,11 @@ IBVERBS_1.8 { ibv_reg_mr_iova2; } IBVERBS_1.7; +IBVERBS_1.9 { + global: + ibv_get_device_index; +} IBVERBS_1.8; + /* If any symbols in this stanza change ABI then the entire staza gets a new symbol version. See the top level CMakeLists.txt for this setting. */
libibverbs/man/CMakeLists.txt +1 -0
@@ -32,6 +32,7 @@ rdma_man_pages( ibv_get_async_event.3 ibv_get_cq_event.3 ibv_get_device_guid.3.md + ibv_get_device_index.3.md ibv_get_device_list.3.md ibv_get_device_name.3.md ibv_get_pkey_index.3.md
libibverbs/man/ibv_get_device_guid.3.md +1 -1
@@ -31,7 +31,7 @@ RDMA device *device*. order. # SEE ALSO - +**ibv_get_device_index**(3), **ibv_get_device_list**(3), **ibv_get_device_name**(3), **ibv_open_device**(3)
libibverbs/man/ibv_get_device_index.3.md new file +40 -0
@@ -0,0 +1,40 @@ +--- +date: ' 2020-04-22' +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: IBV_GET_DEVICE_INDEX +--- + +# NAME + +ibv_get_device_index - get an RDMA device index + +# SYNOPSIS + +```c +#include <infiniband/verbs.h> + +int ibv_get_device_index(struct ibv_device *device); +``` + +# DESCRIPTION + +**ibv_get_device_index()** returns stable IB device index as it is assigned by the kernel. + +# RETURN VALUE + +**ibv_get_device_index()** returns an index, or -1 if the kernel doesn't support device indexes. + +# SEE ALSO + +**ibv_get_device_name**(3), +**ibv_get_device_guid**(3), +**ibv_get_device_list**(3), +**ibv_open_device**(3) + +# AUTHOR + +Leon Romanovsky <[email protected]>
libibverbs/man/ibv_get_device_list.3.md +1 -0
@@ -88,6 +88,7 @@ recommended. **ibv_fork_init**(3), **ibv_get_device_guid**(3), **ibv_get_device_name**(3), +**ibv_get_device_index**(3), **ibv_open_device**(3) # AUTHOR
libibverbs/verbs.h +9 -0
@@ -2202,6 +2202,15 @@ void ibv_free_device_list(struct ibv_device **list);*/ const char *ibv_get_device_name (struct ibv_device *device); +/** +* ibv_get_device_index - Return kernel device index +* +* Available for the kernel with support of IB device query +* over netlink interface. For the unsupported kernels, the +* relevant -1 will be returned. +*/ +int ibv_get_device_index (struct ibv_device *device); +/** * ibv_get_device_guid - Return device's node GUID */
--
2.26.2
PATCH 4/4 librdmacm: Rely on IB device index if available
In reply to Leon RomanovskyFrom: Leon Romanovsky <[email protected]>
For the kernels that support query over netlink, rely on the device index returned and not on node_guid which doesn't identify IB device reliably.
--- librdmacm/cma.c | 53 +++++++++++++++++++++++++++++++--------- librdmacm/rdma_cma_abi.h | 2 ++ 2 files changed, 44 insertions(+), 11 deletions(-)
librdmacm/cma.c +42 -11
@@ -74,6 +74,8 @@ do { \ (req)->response = (uintptr_t ) (resp); \ } while (0) +#define UCMA_INVALID_IB_INDEX -1 +struct cma_port {uint8_t link_layer; };
@@ -89,6 +91,7 @@ struct cma_device {int max_qpsize;uint8_t max_initiator_depth;uint8_t max_responder_resources; +int ibv_idx; };struct cma_id_private {
@@ -292,8 +295,10 @@ int ucma_init(void) goto err2; } -for (i =0 ; dev_list[i]; i++) +for (i =0 ; dev_list[i]; i++) { cma_dev_array[i].guid =ibv_get_device_guid (dev_list[i]); + cma_dev_array[i].ibv_idx =ibv_get_device_index (dev_list[i]); + } cma_dev_cnt = dev_cnt;ucma_set_af_ib_support ();
@@ -309,20 +314,31 @@ err1:return ret; } -static struct ibv_context *ucma_open_device (__be64 guid) +static bool match (struct cma_device *cma_dev,__be64 guid,uint32_t idx) +{ +if (idx ==UCMA_INVALID_IB_INDEX ) +return cma_dev->guid == guid; + +return cma_dev->ibv_idx == idx && cma_dev->guid == guid; +} + +static struct ibv_context *ucma_open_device (struct cma_device *cma_dev) {struct ibv_device **dev_list;struct ibv_context *verbs =NULL ;int i; dev_list =ibv_get_device_list (NULL ); -if (!dev_list) { +if (!dev_list)return NULL ; - }for (i =0 ; dev_list[i]; i++) { -if (ibv_get_device_guid (dev_list[i]) == guid) { - verbs =ibv_open_device (dev_list[i]); +struct ibv_device *dev = dev_list[i]; +uint32_t idx =ibv_get_device_index (dev); +__be64 guid =ibv_get_device_guid (dev); + +if (match (cma_dev, guid, idx)) { + verbs =ibv_open_device (dev);break ; } }
@@ -340,7 +356,7 @@ static int ucma_init_device(struct cma_device *cma_dev)if (cma_dev->verbs)return 0 ; - cma_dev->verbs =ucma_open_device (cma_dev->guid); + cma_dev->verbs =ucma_open_device (cma_dev);if (!cma_dev->verbs)return ERR (ENODEV );
@@ -452,14 +468,15 @@ void rdma_destroy_event_channel(struct rdma_event_channel *channel)free (channel); } -static int ucma_get_device (struct cma_id_private *id_priv,__be64 guid) +static int ucma_get_device (struct cma_id_private *id_priv,__be64 guid, +uint32_t idx) {struct cma_device *cma_dev;int i, ret;for (i =0 ; i < cma_dev_cnt; i++) { cma_dev = &cma_dev_array[i]; -if (cma_dev->guid == guid) +if (match (cma_dev, guid, idx)) goto match; }
@@ -701,6 +718,12 @@ static int ucma_query_addr(struct rdma_cm_id *id) cmd.id = id_priv->handle; cmd.option =UCMA_QUERY_ADDR ; +/* +* If kernel doesn't support ibdev_index, this field will +* be left as is by the kernel. +*/ + resp.ibdev_index =UCMA_INVALID_IB_INDEX ; + ret =write (id->channel->fd, &cmd,sizeof cmd);if (ret !=sizeof cmd)return (ret >=0 ) ?ERR (ENODATA ) :-1 ;
@@ -711,7 +734,8 @@ static int ucma_query_addr(struct rdma_cm_id *id)memcpy (&id->route.addr.dst_addr, &resp.dst_addr, resp.dst_size);if (!id_priv->cma_dev && resp.node_guid) { - ret =ucma_get_device (id_priv, resp.node_guid); + ret =ucma_get_device (id_priv, resp.node_guid, + resp.ibdev_index);if (ret)return ret; id->port_num = resp.port_num;
@@ -826,6 +850,12 @@ static int ucma_query_route(struct rdma_cm_id *id) id_priv = container_of(id, struct cma_id_private, id); cmd.id = id_priv->handle; +/* +* If kernel doesn't support ibdev_index, this field will +* be left as is by the kernel. +*/ + resp.ibdev_index =UCMA_INVALID_IB_INDEX ; + ret =write (id->channel->fd, &cmd,sizeof cmd);if (ret !=sizeof cmd)return (ret >=0 ) ?ERR (ENODATA ) :-1 ;
@@ -855,7 +885,8 @@ static int ucma_query_route(struct rdma_cm_id *id) sizeof resp.dst_addr);if (!id_priv->cma_dev && resp.node_guid) { - ret =ucma_get_device (id_priv, resp.node_guid); + ret =ucma_get_device (id_priv, resp.node_guid, + resp.ibdev_index);if (ret)return ret; id_priv->id.port_num = resp.port_num;
librdmacm/rdma_cma_abi.h +2 -0
@@ -180,6 +180,7 @@ struct ucma_abi_query_route_resp {__u32 num_paths;__u8 port_num;__u8 reserved[3 ]; +__u32 ibdev_index; };struct ucma_abi_query_addr_resp {
@@ -191,6 +192,7 @@ struct ucma_abi_query_addr_resp {__u16 dst_size;struct sockaddr_storage src_addr;struct sockaddr_storage dst_addr; +__u32 ibdev_index; };struct ucma_abi_query_path_resp {
--
2.26.2