RDMA/vmw_pvrdma: Add RoCEv2 support

From: Bryan Tan <[email protected]>

The driver version is bumped for compatibility purposes. Also, send correct GID type during register to device. Added compatibility check macros for the device.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Bryan Tan <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
 4 files changed, 51 insertions(+), 22 deletions(-)

From: Bryan Tan <[email protected]>

The driver version is bumped for compatibility purposes. Also, send correct GID type during register to device. Added compatibility check macros for the device.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Bryan Tan <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
 4 files changed, 51 insertions(+), 22 deletions(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma.h +2 -0

index 8e2f0a1..663a0c3 100644
@@ -194,6 +194,7 @@ struct pvrdma_dev {
 	void *resp_slot;
 	unsigned long flags;
 	struct list_head device_link;
+	unsigned int dsr_version;
 
 	/* Locking and interrupt information. */
 	spinlock_t cmd_lock; /* Command lock. */

@@ -444,6 +445,7 @@ void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst,
 			    const struct pvrdma_ah_attr *src);
 void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 			    const struct rdma_ah_attr *src);
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type);
 
 int pvrdma_uar_table_init(struct pvrdma_dev *dev);
 void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +27 -1

index 09078cc..3a308ff 100644
@@ -50,7 +50,15 @@
 
 #include "pvrdma_verbs.h"
 
-#define PVRDMA_VERSION			17
+/*
+ * PVRDMA version macros. Some new features require updates to PVRDMA_VERSION.
+ * These macros allow us to check for different features if necessary.
+ */
+
+#define PVRDMA_ROCEV1_VERSION		17
+#define PVRDMA_ROCEV2_VERSION		18
+#define PVRDMA_VERSION			PVRDMA_ROCEV2_VERSION
+
 #define PVRDMA_BOARD_ID			1
 #define PVRDMA_REV_ID			1
 

@@ -123,6 +131,24 @@
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
 
+/*
+ * Version checks. This checks whether each version supports specific
+ * capabilities from the device.
+ */
+
+#define PVRDMA_IS_VERSION17(_dev)					\
+	(_dev->dsr_version == PVRDMA_ROCEV1_VERSION &&			\
+	 _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+
+#define PVRDMA_IS_VERSION18(_dev)					\
+	(_dev->dsr_version >= PVRDMA_ROCEV2_VERSION &&			\
+	 (_dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1 ||  \
+	  _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2))	\
+
+#define PVRDMA_SUPPORTED(_dev)						\
+	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
+	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
+
 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */

drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +15 -21

index 5b00156..6ce709a 100644
@@ -128,10 +128,14 @@ static int pvrdma_init_device(struct pvrdma_dev *dev)
 static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
 				 struct ib_port_immutable *immutable)
 {
+	struct pvrdma_dev *dev = to_vdev(ibdev);
 	struct ib_port_attr attr;
 	int err;
 
-	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE;
+	else if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 
 	err = ib_query_port(ibdev, port_num, &attr);
 	if (err)

@@ -569,6 +573,7 @@ static void pvrdma_free_slots(struct pvrdma_dev *dev)
 
 static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 				   const union ib_gid *gid,
+				   u8 gid_type,
 				   int index)
 {
 	int ret;

@@ -586,7 +591,7 @@ static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
 	cmd_bind->vlan = 0xfff;
 	cmd_bind->index = index;
-	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+	cmd_bind->gid_type = gid_type;
 
 	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
 	if (ret < 0) {

@@ -607,7 +612,9 @@ static int pvrdma_add_gid(struct ib_device *ibdev,
 {
 	struct pvrdma_dev *dev = to_vdev(ibdev);
 
-	return pvrdma_add_gid_at_index(dev, gid, index);
+	return pvrdma_add_gid_at_index(dev, gid,
+				       ib_gid_type_to_pvrdma(attr->gid_type),
+				       index);
 }
 
 static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)

@@ -722,7 +729,6 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	int ret;
 	unsigned long start;
 	unsigned long len;
-	unsigned int version;
 	dma_addr_t slot_dma = 0;
 
 	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));

@@ -819,13 +825,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 		goto err_unmap_regs;
 	}
 
-	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
+	dev->dsr_version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
 	dev_info(&pdev->dev, "device version %d, driver version %d\n",
-		 version, PVRDMA_VERSION);
-	if (version < PVRDMA_VERSION) {
-		dev_err(&pdev->dev, "incompatible device version\n");
-		goto err_uar_unmap;
-	}
+		 dev->dsr_version, PVRDMA_VERSION);
 
 	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
 				      &dev->dsrbase, GFP_KERNEL);

@@ -896,17 +898,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	/* Make sure the write is complete before reading status. */
 	mb();
 
-	/* Currently, the driver only supports RoCE mode. */
-	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
-		dev_err(&pdev->dev, "unsupported transport %d\n",
-			dev->dsr->caps.mode);
-		ret = -EFAULT;
-		goto err_free_cq_ring;
-	}
-
-	/* Currently, the driver only supports RoCE V1. */
-	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
-		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
+	/* The driver supports RoCE V1 and V2. */
+	if (!PVRDMA_SUPPORTED(dev)) {
+		dev_err(&pdev->dev, "driver needs RoCE v1 or v2 support\n");
 		ret = -EFAULT;
 		goto err_free_cq_ring;
 	}

drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c +7 -0

index ec6a4ca..fb0c5c0 100644
@@ -303,3 +303,10 @@ void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 	dst->port_num = rdma_ah_get_port_num(src);
 	memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac));
 }
+
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type)
+{
+	return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
+		PVRDMA_GID_TYPE_FLAG_ROCE_V2 :
+		PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+}

Signature · 1 line

2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +0 -0

index 3a308ff..df0a6b5 100644

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +1 -1

 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */

@@ -251,7 +258,7 @@ struct pvrdma_device_caps {
 	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
 	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
 	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
-	u8  reserved[4];
+	u32 max_fast_reg_page_list_len;
 };
 
 struct pvrdma_ring_page_info {

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +9 -1

index 2851704..48776f5 100644
@@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
 	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
 	props->max_sge = dev->dsr->caps.max_sge;
+	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
+					   dev->dsr->caps.max_sge_rd);
 	props->max_cq = dev->dsr->caps.max_cq;
 	props->max_cqe = dev->dsr->caps.max_cqe;
 	props->max_mr = dev->dsr->caps.max_mr;

@@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
 		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
+		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
+				PVRDMA_MAX_FAST_REG_PAGES,
+				dev->dsr->caps.max_fast_reg_page_list_len);
 	}
 
+	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
+				   IB_DEVICE_RC_RNR_NAK_GEN;
+
 	return 0;
 }
 

@@ -143,6 +151,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
 	props->gid_tbl_len = resp->attrs.gid_tbl_len;
 	props->port_cap_flags =
 		pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
+	props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
 	props->max_msg_sz = resp->attrs.max_msg_sz;
 	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
 	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
--

2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 20 lines · Adit Ranadive

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +0 -0

index 3a308ff..df0a6b5 100644
@@ -149,6 +149,13 @@
 	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
 	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h line 158

Quoted 7 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +7 -0

+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+

The current macro implementation will require you to go and update all the places once you will move to new version. In simple case, everything is supported and you will change your check of PVRDMA_IS_VERSION18 to something PVRDMA_IS_VERSIONXXX and it will work.

In more complex case, when one of the features is supported in one version but isn't supported in another you will need to add "if(specific_feature)" magic into it.

Quoted 21 lines · Leon Romanovsky

Quoted 20 lines · Adit Ranadive

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +0 -0

index 3a308ff..df0a6b5 100644
@@ -149,6 +149,13 @@
 	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
 	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 

Quoted 12 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h line 158

Quoted 7 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +7 -0

+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+

The current macro implementation will require you to go and update all the places once you will move to new version. In simple case, everything is supported and you will change your check of PVRDMA_IS_VERSION18 to something PVRDMA_IS_VERSIONXXX and it will work.

Thanks for taking a look. That's the intention I wrote the macro with so it should pick up the value based on the current device version. If you are concerned that we will change device caps without changing the PVRDMA version, that's something we avoid for compatibility reasons.

Quoted 4 lines

In more complex case, when one of the features is supported in one version but isn't supported in another you will need to add "if(specific_feature)" magic into it.

This will not happen based on how our version model currently works.

Quoted 35 lines · Adit Ranadive

Quoted 21 lines · Leon Romanovsky

Quoted 20 lines · Adit Ranadive

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +0 -0

index 3a308ff..df0a6b5 100644
@@ -149,6 +149,13 @@
 	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
 	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 

Quoted 12 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h line 158

Quoted 7 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +7 -0

+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+

The current macro implementation will require you to go and update all the places once you will move to new version. In simple case, everything is supported and you will change your check of PVRDMA_IS_VERSION18 to something PVRDMA_IS_VERSIONXXX and it will work.

Quoted 4 lines

Thanks for taking a look. That's the intention I wrote the macro with so it should pick up the value based on the current device version. If you are concerned that we will change device caps without changing the PVRDMA version, that's something we avoid for compatibility reasons.

I have hard time to understand how will you extend this macro once new feature is introduced.

Let's for an example conduct the following experiment:
1. IB/core gets new exciting ROCE_V3
2. You are adding support to it and increasing the version to be VERSION19.
3. You now have code for ROCE_V2 and ROCE_V3.

So the question is: how will your VRDMA_GET_CAP() look in such case? RECE_V2 is supported in VERSION18 and above, while ROCE_V3 is supported in VERSION19 only.

And if we add more than one new feature, how will it look?

Thanks

Quoted 44 lines · Leon Romanovsky

Quoted 35 lines · Adit Ranadive

Quoted 21 lines · Leon Romanovsky

Quoted 20 lines · Adit Ranadive

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +0 -0

index 3a308ff..df0a6b5 100644
@@ -149,6 +149,13 @@
 	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
 	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 

Quoted 12 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h line 158

Quoted 7 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +7 -0

+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+

The current macro implementation will require you to go and update all the places once you will move to new version. In simple case, everything is supported and you will change your check of PVRDMA_IS_VERSION18 to something PVRDMA_IS_VERSIONXXX and it will work.

Quoted 4 lines

Thanks for taking a look. That's the intention I wrote the macro with so it should pick up the value based on the current device version. If you are concerned that we will change device caps without changing the PVRDMA version, that's something we avoid for compatibility reasons.

I have hard time to understand how will you extend this macro once new feature is introduced.

Quoted 8 lines

Let's for an example conduct the following experiment:
1. IB/core gets new exciting ROCE_V3
2. You are adding support to it and increasing the version to be VERSION19.
3. You now have code for ROCE_V2 and ROCE_V3.

So the question is: how will your VRDMA_GET_CAP() look in such case? RECE_V2 is supported in VERSION18 and above, while ROCE_V3 is supported in VERSION19 only.

The values for the capabilities come from our DSR (device shared region) itself. This would have nothing to do with the exact RoCE version but is based on the device version itself. So, if we add a VERSION19, we can change the macro to something like PVRDMA_IS_VERSION_NEXT_GEN, it will just pick up the new values defined in the DSR rather than based on RoCE versions. If we have removed support for that capability for VERSION19, that should simply be 0.

To make this more clear, I can put the RoCE v1/v2 type checks, each in own macro. Drop them from the PVRDMA_IS_VERSION checks. Rename the PVRDMA_IS_VERSION18 to PVRDMA_IS_NEXTVERSION so we will always pick up the cap values correctly for future versions and we are not dependent on the RoCE version itself.

Quoted 4 lines

And if we add more than one new feature, how will it look?

Thanks

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +8 -1

index 3a308ff..df0a6b5 100644
@@ -149,6 +149,13 @@
 	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
 	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 
+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+
 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */

@@ -251,7 +258,7 @@ struct pvrdma_device_caps {
 	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
 	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
 	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
-	u8  reserved[4];
+	u32 max_fast_reg_page_list_len;
 };
 
 struct pvrdma_ring_page_info {

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +0 -0

index 2851704..48776f5 100644

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +0 -0

 	props->max_cq = dev->dsr->caps.max_cq;
 	props->max_cqe = dev->dsr->caps.max_cqe;
 	props->max_mr = dev->dsr->caps.max_mr;

@@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +0 -0

 	props->max_msg_sz = resp->attrs.max_msg_sz;
 	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
 	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;

Signature · 6 lines

2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 2 lines · Adit Ranadive

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len and the IP_BASED_GIDS port cap flag.

And IB_PORT_CM_SUP

Quoted 42 lines

Reviewed-by Jorgen Hansen <[email protected]>

Reviewed-by Bryan Tan <[email protected]>

Reviewed-by Aditya Sarwade <[email protected]>

Signed-off-by Adit Ranadive <[email protected]>

---
  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
  2 files changed, 17 insertions(+), 1 deletion(-)

drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h +8 -1

index 3a308ff..df0a6b5 100644
@@ -149,6 +149,13 @@
  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 
+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+
  enum pvrdma_pci_resource {
  	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
  	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */

@@ -251,7 +258,7 @@ struct pvrdma_device_caps {
  	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
  	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
  	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
-	u8  reserved[4];
+	u32 max_fast_reg_page_list_len;
  };
 
  struct pvrdma_ring_page_info {

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +0 -0

index 2851704..48776f5 100644

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c int pvrdma_query_device(struct ib_device *ibdev, · line 87

Quoted 5 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +2 -0

@@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
  	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
  	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
  	props->max_sge = dev->dsr->caps.max_sge;
+	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
+					   dev->dsr->caps.max_sge_rd);

I tend to agree with Leon on the other thread, i think that the following code is much more readable:

	props->max_sge_rd = dev->dsr->caps.max_sge;
	if (PVRDMA_IS_VERSION18(dev))
		props->max_sge_rd = dev->dsr->caps.max_sge_rd;

And then in the future add something like this:

	if (PVRDMA_IS_VERSION19(dev))
		props->max_sge_rd = dev->dsr->caps.max_sge_19;

But it is totally up to you.

Quoted 13 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +3 -0

  	props->max_cq = dev->dsr->caps.max_cq;
  	props->max_cqe = dev->dsr->caps.max_cqe;
  	props->max_mr = dev->dsr->caps.max_mr;

@@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
  	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
  	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
  		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
+		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
+				PVRDMA_MAX_FAST_REG_PAGES,
+				dev->dsr->caps.max_fast_reg_page_list_len);
  	}
 

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c int pvrdma_query_device(struct ib_device *ibdev, · line 113

Quoted 3 lines

drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +3 -0

+	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
+				   IB_DEVICE_RC_RNR_NAK_GEN;
+

Is it left over from version 17 or a new HW support?

Hi Doug,

These are a set of patches to add RoCEv2 support for the VMware PVRDMA
driver as well as exposing some device caps. Please add these to your 4.14 queue.

Thanks,
Adit
---
Changes v0 -> v1:
- Consolidated the version macros.
- Added macros to check for compatibility.
- Squashed a previous patch #3 into patch #2.
- Removed an unused function.
- Updated tag lines correctly.
---
Adit Ranadive (1):
RDMA/vmw_pvrdma: Update device query parameters and port caps

Bryan Tan (1):
RDMA/vmw_pvrdma: Add RoCEv2 support

 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 37
+++++++++++++++++++++--
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 +++++++++--
-----------
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   |  9 ++++++
 5 files changed, 68 insertions(+), 23 deletions(-)