Skip to contentSkip to conversation

libibverbs: display gid type in ibv_devinfo

PATCH Superseded Devesh Sharma 4 participants 7 messages Feb 4, 2020 – Feb 6, 2020 InfiniBand

v6 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

PATCH v6 1/1 libibverbs: display gid type in ibv_devinfo

It becomes difficult to make out from the output of ibv_devinfo if a particular gid index is RoCE v2 or not.

Adding a string to the output of ibv_devinfo -v to display the gid type at the end of gid.

The output would look something like below:

$ ibv_devinfo -v -d bnxt_re2
hca_id: bnxt_re2
 transport:             InfiniBand (0)
 fw_ver:                216.0.220.0
 node_guid:             b226:28ff:fed3:b0f0
 sys_image_guid:        b226:28ff:fed3:b0f0
  .
  .
  .
  .
	phys_state:     LINK_UP (5)
	GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
	GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
	GID[  1]:       fe80::b226:28ff:fed3:b0f0
	GID[  2]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, IB/RoCE v1
	GID[  3]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, RoCE v2
	GID[  3]:       ::ffff:192.170.1.101
$
$
$
---
 libibverbs/examples/devinfo.c | 60 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 12 deletions(-)
libibverbs/examples/devinfo.c +48 -12
index bf53eac..83acc22 100644
@@ -40,6 +40,7 @@
 #include <getopt.h>
 #include <endian.h>
 #include <inttypes.h>
+#include <arpa/inet.h>
 
 #include <infiniband/verbs.h>
 #include <infiniband/driver.h>
@@ -162,12 +163,49 @@ static const char *vl_str(uint8_t vl_num)
 	}
 }
 
-static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tbl_len)
+#define DEVINFO_INVALID_GID_TYPE	2
+static const char *gid_type_str(enum ibv_gid_type type)
 {
+	switch (type) {
+	case IBV_GID_TYPE_IB_ROCE_V1: return "IB/RoCE v1";
+	case IBV_GID_TYPE_ROCE_V2: return "RoCE v2";
+	default: return "Invalid gid type";
+	}
+}
+
+static void print_formated_gid(union ibv_gid *gid, int i,
+			       enum ibv_gid_type type, int ll)
+{
+	char gid_str[INET6_ADDRSTRLEN] = {};
+	char str[20] = {};
+
+	if (ll == IBV_LINK_LAYER_ETHERNET) {
+		sprintf(str, ", %s", gid_type_str(type));
+		printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x%s\n",
+		       i, gid->raw[0], gid->raw[1], gid->raw[2],
+		       gid->raw[3], gid->raw[4], gid->raw[5], gid->raw[6],
+		       gid->raw[7], gid->raw[8], gid->raw[9], gid->raw[10],
+		       gid->raw[11], gid->raw[12], gid->raw[13], gid->raw[14],
+		       gid->raw[15], str);
+	}
+
+	if (type == IBV_GID_TYPE_ROCE_V2) {
+		inet_ntop(AF_INET6, gid->raw, gid_str, sizeof(gid_str));
+		printf("\t\t\tGID[%3d]:\t\t%s\n", i, gid_str);
+	}
+}
+
+static int print_all_port_gids(struct ibv_context *ctx,
+			       struct ibv_port_attr *port_attr,
+			       uint8_t port_num)
+{
+	enum ibv_gid_type type;
 	union ibv_gid gid;
+	int tbl_len;
 	int rc = 0;
 	int i;
 
+	tbl_len = port_attr->gid_tbl_len;
 	for (i = 0; i < tbl_len; i++) {
 		rc = ibv_query_gid(ctx, port_num, i, &gid);
 		if (rc) {
@@ -175,17 +213,15 @@ static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tb
 			       port_num, i);
 			return rc;
 		}
+
+		rc = ibv_query_gid_type(ctx, port_num, i, &type);
+		if (rc) {
+			rc = 0;
+			type = DEVINFO_INVALID_GID_TYPE;
+		}
 		if (!null_gid(&gid))
-			printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n",
-			       i,
-			       gid.raw[ 0], gid.raw[ 1],
-			       gid.raw[ 2], gid.raw[ 3],
-			       gid.raw[ 4], gid.raw[ 5],
-			       gid.raw[ 6], gid.raw[ 7],
-			       gid.raw[ 8], gid.raw[ 9],
-			       gid.raw[10], gid.raw[11],
-			       gid.raw[12], gid.raw[13],
-			       gid.raw[14], gid.raw[15]);
+			print_formated_gid(&gid, i, type,
+					   port_attr->link_layer);
 	}
 	return rc;
 }
@@ -614,7 +650,7 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
 				printf("\t\t\tphys_state:\t\t%s (%d)\n",
 				       port_phy_state_str(port_attr.phys_state), port_attr.phys_state);
 
-			if (print_all_port_gids(ctx, port, port_attr.gid_tbl_len))
+			if (print_all_port_gids(ctx, &port_attr, port))
 				goto cleanup;
 		}
 		printf("\n");
Signature · 1 line
1.8.3.1
#1 lore raw
Quoted 28 lines · Devesh Sharma

It becomes difficult to make out from the output of ibv_devinfo if a particular gid index is RoCE v2 or not.

Adding a string to the output of ibv_devinfo -v to display the gid type at the end of gid.

The output would look something like below:

$ ibv_devinfo -v -d bnxt_re2
hca_id: bnxt_re2
 transport:             InfiniBand (0)
 fw_ver:                216.0.220.0
 node_guid:             b226:28ff:fed3:b0f0
 sys_image_guid:        b226:28ff:fed3:b0f0
  .
  .
  .
  .
	phys_state:     LINK_UP (5)
	GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
	GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
	GID[  1]:       fe80::b226:28ff:fed3:b0f0
	GID[  2]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, IB/RoCE v1
	GID[  3]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, RoCE v2
	GID[  3]:       ::ffff:192.170.1.101
$
$
$
Quoted 2 lines

At least with my ROB tag, you was too fast :)

Quoted 108 lines
---
 libibverbs/examples/devinfo.c | 60 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 12 deletions(-)
libibverbs/examples/devinfo.c +48 -13
index bf53eac..83acc22 100644
@@ -40,6 +40,7 @@
 #include <getopt.h>
 #include <endian.h>
 #include <inttypes.h>
+#include <arpa/inet.h>
 
 #include <infiniband/verbs.h>
 #include <infiniband/driver.h>
@@ -162,12 +163,49 @@ static const char *vl_str(uint8_t vl_num)
 	}
 }
 
-static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tbl_len)
+#define DEVINFO_INVALID_GID_TYPE	2
+static const char *gid_type_str(enum ibv_gid_type type)
 {
+	switch (type) {
+	case IBV_GID_TYPE_IB_ROCE_V1: return "IB/RoCE v1";
+	case IBV_GID_TYPE_ROCE_V2: return "RoCE v2";
+	default: return "Invalid gid type";
+	}
+}
+
+static void print_formated_gid(union ibv_gid *gid, int i,
+			       enum ibv_gid_type type, int ll)
+{
+	char gid_str[INET6_ADDRSTRLEN] = {};
+	char str[20] = {};
+
+	if (ll == IBV_LINK_LAYER_ETHERNET) {
+		sprintf(str, ", %s", gid_type_str(type));
+		printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x%s\n",
+		       i, gid->raw[0], gid->raw[1], gid->raw[2],
+		       gid->raw[3], gid->raw[4], gid->raw[5], gid->raw[6],
+		       gid->raw[7], gid->raw[8], gid->raw[9], gid->raw[10],
+		       gid->raw[11], gid->raw[12], gid->raw[13], gid->raw[14],
+		       gid->raw[15], str);
+	}
+
+	if (type == IBV_GID_TYPE_ROCE_V2) {
+		inet_ntop(AF_INET6, gid->raw, gid_str, sizeof(gid_str));
+		printf("\t\t\tGID[%3d]:\t\t%s\n", i, gid_str);
+	}
+}
+
+static int print_all_port_gids(struct ibv_context *ctx,
+			       struct ibv_port_attr *port_attr,
+			       uint8_t port_num)
+{
+	enum ibv_gid_type type;
 	union ibv_gid gid;
+	int tbl_len;
 	int rc = 0;
 	int i;
 
+	tbl_len = port_attr->gid_tbl_len;
 	for (i = 0; i < tbl_len; i++) {
 		rc = ibv_query_gid(ctx, port_num, i, &gid);
 		if (rc) {
@@ -175,17 +213,15 @@ static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tb
 			       port_num, i);
 			return rc;
 		}
+
+		rc = ibv_query_gid_type(ctx, port_num, i, &type);
+		if (rc) {
+			rc = 0;
+			type = DEVINFO_INVALID_GID_TYPE;
+		}
 		if (!null_gid(&gid))
-			printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n",
-			       i,
-			       gid.raw[ 0], gid.raw[ 1],
-			       gid.raw[ 2], gid.raw[ 3],
-			       gid.raw[ 4], gid.raw[ 5],
-			       gid.raw[ 6], gid.raw[ 7],
-			       gid.raw[ 8], gid.raw[ 9],
-			       gid.raw[10], gid.raw[11],
-			       gid.raw[12], gid.raw[13],
-			       gid.raw[14], gid.raw[15]);
+			print_formated_gid(&gid, i, type,
+					   port_attr->link_layer);
 	}
 	return rc;
 }
@@ -614,7 +650,7 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
 				printf("\t\t\tphys_state:\t\t%s (%d)\n",
 				       port_phy_state_str(port_attr.phys_state), port_attr.phys_state);
 
-			if (print_all_port_gids(ctx, port, port_attr.gid_tbl_len))
+			if (print_all_port_gids(ctx, &port_attr, port))
 				goto cleanup;
 		}
 		printf("\n");
--

1.8.3.1

#2 lore raw
In reply to Devesh Sharma

Hi Devesh,

Quoted 6 lines

From: [email protected] <linux-rdma- [email protected]> On Behalf Of Devesh Sharma

	phys_state:     LINK_UP (5)
	GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
	GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
	GID[  1]:       fe80::b226:28ff:fed3:b0f0

Showing two entries as individual raw like this is surely confusing to user.
Either all content should be in single raw or as Leon said just single different format for RoCEv2 is fine.

Quoted 3 lines
	GID[  2]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, IB/RoCE v1
	GID[  3]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, RoCE v2
	GID[  3]:       ::ffff:192.170.1.101
#3 lore raw
Quoted 11 lines · Parav Pandit

Hi Devesh,

Quoted 6 lines

From: [email protected] <linux-rdma- [email protected]> On Behalf Of Devesh Sharma

      phys_state:     LINK_UP (5)
      GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
      GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
      GID[  1]:       fe80::b226:28ff:fed3:b0f0

Showing two entries as individual raw like this is surely confusing to user.
Either all content should be in single raw or as Leon said just single different format for RoCEv2 is fine.

Yes, I liked the single display in new format, I would wait for Jason to agree/disagree and then send a rev.

Quoted 4 lines
Quoted 4 lines
      GID[  2]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, IB/RoCE v1
      GID[  3]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, RoCE v2
      GID[  3]:       ::ffff:192.170.1.101
#4 lore raw
Quoted 10 lines · Devesh Sharma
Quoted 9 lines · Parav Pandit

Hi Devesh,

Quoted 6 lines

From: [email protected] <linux-rdma- [email protected]> On Behalf Of Devesh Sharma

      phys_state:     LINK_UP (5)
      GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
      GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
      GID[  1]:       fe80::b226:28ff:fed3:b0f0
Quoted 4 lines
Quoted 2 lines

Showing two entries as individual raw like this is surely confusing to user.
Either all content should be in single raw or as Leon said just single different format for RoCEv2 is fine.

Yes, I liked the single display in new format, I would wait for Jason to agree/disagree and then send a rev.

Well, I wasn't thinking to display the GID[xx] prefix for the 2nd entry

It seems like other people want to just show one, that seems OK too

Jason

#5 lore raw
Quoted 14 lines · Jason Gunthorpe
Quoted 12 lines · Devesh Sharma
Quoted 11 lines · Parav Pandit

Hi Devesh,

Quoted 6 lines

From: [email protected] <linux-rdma- [email protected]> On Behalf Of Devesh Sharma

      phys_state:     LINK_UP (5)
      GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
      GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
      GID[  1]:       fe80::b226:28ff:fed3:b0f0

Showing two entries as individual raw like this is surely confusing to user.
Either all content should be in single raw or as Leon said just single different format for RoCEv2 is fine.

Quoted 7 lines
Quoted 2 lines

Yes, I liked the single display in new format, I would wait for Jason to agree/disagree and then send a rev.

Well, I wasn't thinking to display the GID[xx] prefix for the 2nd entry

It seems like other people want to just show one, that seems OK too

Thanks for confirming, sending down V7 shortly please review.

Quoted 2 lines

Jason

#6 lore raw
In reply to Devesh Sharma
Quoted 70 lines · Devesh Sharma

It becomes difficult to make out from the output of ibv_devinfo if a particular gid index is RoCE v2 or not.

Adding a string to the output of ibv_devinfo -v to display the gid type at the end of gid.

The output would look something like below:

$ ibv_devinfo -v -d bnxt_re2
hca_id: bnxt_re2
 transport:             InfiniBand (0)
 fw_ver:                216.0.220.0
 node_guid:             b226:28ff:fed3:b0f0
 sys_image_guid:        b226:28ff:fed3:b0f0
  .
  .
  .
  .
        phys_state:     LINK_UP (5)
        GID[  0]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, IB/RoCE v1
        GID[  1]:       fe80:0000:0000:0000:b226:28ff:fed3:b0f0, RoCE v2
        GID[  1]:       fe80::b226:28ff:fed3:b0f0
        GID[  2]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, IB/RoCE v1
        GID[  3]:       0000:0000:0000:0000:0000:ffff:c0aa:0165, RoCE v2
        GID[  3]:       ::ffff:192.170.1.101
$
$
$
---
 libibverbs/examples/devinfo.c | 60 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 12 deletions(-)
libibverbs/examples/devinfo.c +15 -1
index bf53eac..83acc22 100644
@@ -40,6 +40,7 @@
 #include <getopt.h>
 #include <endian.h>
 #include <inttypes.h>
+#include <arpa/inet.h>
 
 #include <infiniband/verbs.h>
 #include <infiniband/driver.h>
@@ -162,12 +163,49 @@ static const char *vl_str(uint8_t vl_num)
        }
 }
 
-static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tbl_len)
+#define DEVINFO_INVALID_GID_TYPE       2
+static const char *gid_type_str(enum ibv_gid_type type)
 {
+       switch (type) {
+       case IBV_GID_TYPE_IB_ROCE_V1: return "IB/RoCE v1";
+       case IBV_GID_TYPE_ROCE_V2: return "RoCE v2";
+       default: return "Invalid gid type";
+       }
+}
+
+static void print_formated_gid(union ibv_gid *gid, int i,
+                              enum ibv_gid_type type, int ll)
+{
+       char gid_str[INET6_ADDRSTRLEN] = {};
+       char str[20] = {};

libibverbs/examples/devinfo.c static const char *vl_str(uint8_t vl_num) · line 190

Quoted 10 lines
libibverbs/examples/devinfo.c +10 -0
+
+       if (ll == IBV_LINK_LAYER_ETHERNET) {
+               sprintf(str, ", %s", gid_type_str(type));
+               printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x%s\n",
+                      i, gid->raw[0], gid->raw[1], gid->raw[2],
+                      gid->raw[3], gid->raw[4], gid->raw[5], gid->raw[6],
+                      gid->raw[7], gid->raw[8], gid->raw[9], gid->raw[10],
+                      gid->raw[11], gid->raw[12], gid->raw[13], gid->raw[14],
+                      gid->raw[15], str);
+       }

Opps!! this needs a rev in anycase, for IB devices gid won't be printed at all, my bad!

Quoted 59 lines
libibverbs/examples/devinfo.c +23 -12
+
+       if (type == IBV_GID_TYPE_ROCE_V2) {
+               inet_ntop(AF_INET6, gid->raw, gid_str, sizeof(gid_str));
+               printf("\t\t\tGID[%3d]:\t\t%s\n", i, gid_str);
+       }
+}
+
+static int print_all_port_gids(struct ibv_context *ctx,
+                              struct ibv_port_attr *port_attr,
+                              uint8_t port_num)
+{
+       enum ibv_gid_type type;
        union ibv_gid gid;
+       int tbl_len;
        int rc = 0;
        int i;
 
+       tbl_len = port_attr->gid_tbl_len;
        for (i = 0; i < tbl_len; i++) {
                rc = ibv_query_gid(ctx, port_num, i, &gid);
                if (rc) {
@@ -175,17 +213,15 @@ static int print_all_port_gids(struct ibv_context *ctx, uint8_t port_num, int tb
                               port_num, i);
                        return rc;
                }
+
+               rc = ibv_query_gid_type(ctx, port_num, i, &type);
+               if (rc) {
+                       rc = 0;
+                       type = DEVINFO_INVALID_GID_TYPE;
+               }
                if (!null_gid(&gid))
-                       printf("\t\t\tGID[%3d]:\t\t%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n",
-                              i,
-                              gid.raw[ 0], gid.raw[ 1],
-                              gid.raw[ 2], gid.raw[ 3],
-                              gid.raw[ 4], gid.raw[ 5],
-                              gid.raw[ 6], gid.raw[ 7],
-                              gid.raw[ 8], gid.raw[ 9],
-                              gid.raw[10], gid.raw[11],
-                              gid.raw[12], gid.raw[13],
-                              gid.raw[14], gid.raw[15]);
+                       print_formated_gid(&gid, i, type,
+                                          port_attr->link_layer);
        }
        return rc;
 }
@@ -614,7 +650,7 @@ static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port)
                                printf("\t\t\tphys_state:\t\t%s (%d)\n",
                                       port_phy_state_str(port_attr.phys_state), port_attr.phys_state);
 
-                       if (print_all_port_gids(ctx, port, port_attr.gid_tbl_len))
+                       if (print_all_port_gids(ctx, &port_attr, port))
                                goto cleanup;
                }
                printf("\n");
--

1.8.3.1