Skip to contentSkip to conversation

RDMA/hns: Support UD for HIP09

PATCH Applied Weihang Li 3 participants 15 messages Nov 16, 2020 – Nov 27, 2020

Subsystems (2)
lore
Share & tools
lore

Weihang Li

author
#0 lore raw

v2 cover RDMA/hns: Support UD for HIP09

This series does cleanups on UD related code at first, including removing dead code, adding necessary check and some refactors. Then the UD feature is enabled on HIP09.

Changes since v1:
- Don't allow HIP08's user to create AH and UD QP from userspace and add .create_user_ah in #6.
- Drop #4 from the v1 series which needs more discussion about the reserved sl.

Weihang Li (7):
RDMA/hns: Only record vlan info for HIP08
RDMA/hns: Fix missing fields in address vector
RDMA/hns: Avoid setting loopback indicator when smac is same as dmac RDMA/hns: Remove the portn field in UD SQ WQE
RDMA/hns: Simplify process of filling UD SQ WQE
RDMA/hns: Add UD support for HIP09
RDMA/hns: Add support for UD inline

 drivers/infiniband/hw/hns/hns_roce_ah.c     |  62 ++++-----
 drivers/infiniband/hw/hns/hns_roce_device.h |   4 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 196 ++++++++++++++++++++--------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  44 +++----
 drivers/infiniband/hw/hns/hns_roce_main.c   |   1 +
 drivers/infiniband/hw/hns/hns_roce_qp.c     |  45 +++++--
 6 files changed, 229 insertions(+), 123 deletions(-)
Signature · 1 line
2.8.1
author
#1 lore raw

PATCH v2 1/7 RDMA/hns: Only record vlan info for HIP08

Information about vlan is stored in GMV(GID/MAC/VLAN) table for HIP09, so there is no need to copy it to address vector.

---
 drivers/infiniband/hw/hns/hns_roce_ah.c     | 51 +++++++++++++----------------
 drivers/infiniband/hw/hns/hns_roce_device.h |  2 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 13 +++++---
 3 files changed, 33 insertions(+), 33 deletions(-)
drivers/infiniband/hw/hns/hns_roce_ah.c +23 -28
index 75b06db..3be80d4 100644
@@ -31,13 +31,13 @@
  */
 
 #include <linux/platform_device.h>
+#include <linux/pci.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
 #include "hns_roce_device.h"
 
-#define HNS_ROCE_PORT_NUM_SHIFT		24
-#define HNS_ROCE_VLAN_SL_BIT_MASK	7
-#define HNS_ROCE_VLAN_SL_SHIFT		13
+#define VLAN_SL_MASK 7
+#define VLAN_SL_SHIFT 13
 
 static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr)
 {
@@ -58,37 +58,16 @@ static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr)
 int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 		       struct ib_udata *udata)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device);
-	const struct ib_gid_attr *gid_attr;
-	struct device *dev = hr_dev->dev;
-	struct hns_roce_ah *ah = to_hr_ah(ibah);
 	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
-	u16 vlan_id = 0xffff;
-	bool vlan_en = false;
-	int ret;
-
-	gid_attr = ah_attr->grh.sgid_attr;
-	ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
-	if (ret)
-		return ret;
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device);
+	struct hns_roce_ah *ah = to_hr_ah(ibah);
+	int ret = 0;
 
-	/* Get mac address */
 	memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
 
-	if (vlan_id < VLAN_N_VID) {
-		vlan_en = true;
-		vlan_id |= (rdma_ah_get_sl(ah_attr) &
-			     HNS_ROCE_VLAN_SL_BIT_MASK) <<
-			     HNS_ROCE_VLAN_SL_SHIFT;
-	}
-
 	ah->av.port = rdma_ah_get_port_num(ah_attr);
 	ah->av.gid_index = grh->sgid_index;
-	ah->av.vlan_id = vlan_id;
-	ah->av.vlan_en = vlan_en;
-	dev_dbg(dev, "gid_index = 0x%x,vlan_id = 0x%x\n", ah->av.gid_index,
-		ah->av.vlan_id);
 
 	if (rdma_ah_get_static_rate(ah_attr))
 		ah->av.stat_rate = IB_RATE_10_GBPS;
@@ -98,7 +77,23 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 	ah->av.flowlabel = grh->flow_label;
 	ah->av.udp_sport = get_ah_udp_sport(ah_attr);
 
-	return 0;
+	/* HIP08 needs to record vlan info in Address Vector */
+	if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
+		ah->av.vlan_en = 0;
+
+		ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr,
+					      &ah->av.vlan_id, NULL);
+		if (ret)
+			return ret;
+
+		if (ah->av.vlan_id < VLAN_N_VID) {
+			ah->av.vlan_en = 1;
+			ah->av.vlan_id |= (rdma_ah_get_sl(ah_attr) & VLAN_SL_MASK) <<
+					  VLAN_SL_SHIFT;
+		}
+	}
+
+	return ret;
 }
 
 int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -1
index 1d99022..9a032d0 100644
@@ -549,7 +549,7 @@ struct hns_roce_av {
 	u8 dgid[HNS_ROCE_GID_SIZE];
 	u8 mac[ETH_ALEN];
 	u16 vlan_id;
-	bool vlan_en;
+	u8 vlan_en;
 };
 
 struct hns_roce_ah {
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +9 -4
index 4d697e4..3895a2a 100644
@@ -492,8 +492,6 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
 		       V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
 
-	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
-		       V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
 	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
 		       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
 	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
@@ -505,11 +503,18 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M,
 		       V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port);
 
-	roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
-		     ah->av.vlan_en ? 1 : 0);
 	roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
 		       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);
 
+	if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
+		roce_set_bit(ud_sq_wqe->byte_40,
+			     V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
+			     ah->av.vlan_en);
+		roce_set_field(ud_sq_wqe->byte_36,
+			       V2_UD_SEND_WQE_BYTE_36_VLAN_M,
+			       V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
+	}
+
 	memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2);
 
 	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
Signature · 1 line
2.8.1
author
#2 lore raw

PATCH v2 2/7 RDMA/hns: Fix missing fields in address vector

In reply to Weihang Li

Traffic class and hop limit in address vector is not assigned from GRH, but it will be filled into UD SQ WQE. So the hardware will get a wrong value.

Fixes 82e620d9c3a0 ("RDMA/hns: Modify the data structure of hns_roce_av")
---
 drivers/infiniband/hw/hns/hns_roce_ah.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)
drivers/infiniband/hw/hns/hns_roce_ah.c +6 -4
index 3be80d4..d65ff6a 100644
@@ -64,18 +64,20 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 	struct hns_roce_ah *ah = to_hr_ah(ibah);
 	int ret = 0;
 
-	memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
-
 	ah->av.port = rdma_ah_get_port_num(ah_attr);
 	ah->av.gid_index = grh->sgid_index;
 
 	if (rdma_ah_get_static_rate(ah_attr))
 		ah->av.stat_rate = IB_RATE_10_GBPS;
 
-	memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE);
-	ah->av.sl = rdma_ah_get_sl(ah_attr);
+	ah->av.hop_limit = grh->hop_limit;
 	ah->av.flowlabel = grh->flow_label;
 	ah->av.udp_sport = get_ah_udp_sport(ah_attr);
+	ah->av.sl = rdma_ah_get_sl(ah_attr);
+	ah->av.tclass = grh->traffic_class;
+
+	memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE);
+	memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
 
 	/* HIP08 needs to record vlan info in Address Vector */
 	if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
Signature · 1 line
2.8.1
author
#3 lore raw

PATCH v2 3/7 RDMA/hns: Avoid setting loopback indicator when smac is same as dmac

In reply to Weihang Li

The loopback flag will be set to 1 by the hardware when the source mac address is same as the destination mac address. So the driver don't need to compare them.

Fixes d6a3627e311c ("RDMA/hns: Optimize wqe buffer set flow for post send")
---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 9 ---------
 1 file changed, 9 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +0 -9
index 3895a2a..bd09df4 100644
@@ -433,8 +433,6 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	unsigned int curr_idx = *sge_idx;
 	int valid_num_sge;
 	u32 msg_len = 0;
-	bool loopback;
-	u8 *smac;
 	int ret;
 
 	valid_num_sge = calc_wr_sge_num(wr, &msg_len);
@@ -457,13 +455,6 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
 		       V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]);
 
-	/* MAC loopback */
-	smac = (u8 *)hr_dev->dev_addr[qp->port];
-	loopback = ether_addr_equal_unaligned(ah->av.mac, smac) ? 1 : 0;
-
-	roce_set_bit(ud_sq_wqe->byte_40,
-		     V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
-
 	ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
 
 	/* Set sig attr */
Signature · 1 line
2.8.1
author
#4 lore raw

PATCH v2 4/7 RDMA/hns: Remove the portn field in UD SQ WQE

In reply to Weihang Li

This field in UD WQE in not used by hardware.

Fixes 7bdee4158b37 ("RDMA/hns: Fill sq wqe context of ud type in hip08")
---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 --
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 3 ---
 2 files changed, 5 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +0 -2
index bd09df4..f4358d2 100644
@@ -491,8 +491,6 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 		       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
 	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
 		       V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
-	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_PORTN_M,
-		       V2_UD_SEND_WQE_BYTE_40_PORTN_S, qp->port);
 
 	roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
 		       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +0 -3
index 1409d05..1466888 100644
@@ -1121,9 +1121,6 @@ struct hns_roce_v2_ud_send_wqe {
 #define	V2_UD_SEND_WQE_BYTE_40_SL_S 20
 #define V2_UD_SEND_WQE_BYTE_40_SL_M GENMASK(23, 20)
 
-#define	V2_UD_SEND_WQE_BYTE_40_PORTN_S 24
-#define V2_UD_SEND_WQE_BYTE_40_PORTN_M GENMASK(26, 24)
-
 #define V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S 30
 
 #define	V2_UD_SEND_WQE_BYTE_40_LBI_S 31
Signature · 1 line
2.8.1
author
#5 lore raw

PATCH v2 5/7 RDMA/hns: Simplify process of filling UD SQ WQE

In reply to Weihang Li

There are some codes can be simplified or encapsulated in set_ud_wqe() to make them easier to be understand.

---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 83 ++++++++++++++----------------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 29 ++---------
 2 files changed, 43 insertions(+), 69 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +40 -43
index f4358d2..273d968 100644
@@ -422,16 +422,49 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
 	return 0;
 }
 
+static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
+		      struct hns_roce_ah *ah)
+{
+	struct ib_device *ib_dev = ah->ibah.device;
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
+
+	roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
+		       V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
+
+	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
+		       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
+	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
+		       V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
+	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
+		       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
+	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
+		       V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
+
+	ud_sq_wqe->sgid_index = ah->av.gid_index;
+
+	memcpy(ud_sq_wqe->dmac, ah->av.mac, ETH_ALEN);
+	memcpy(ud_sq_wqe->dgid, ah->av.dgid, GID_LEN_V2);
+
+	if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
+		return 0;
+
+	roce_set_bit(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
+		     ah->av.vlan_en);
+	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_VLAN_M,
+		       V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
+
+	return 0;
+}
+
 static inline int set_ud_wqe(struct hns_roce_qp *qp,
 			     const struct ib_send_wr *wr,
 			     void *wqe, unsigned int *sge_idx,
 			     unsigned int owner_bit)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
 	unsigned int curr_idx = *sge_idx;
-	int valid_num_sge;
+	unsigned int valid_num_sge;
 	u32 msg_len = 0;
 	int ret;
 
@@ -442,28 +475,13 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (WARN_ON(ret))
 		return ret;
 
-	roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
-		       V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
-	roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
-		       V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
-	roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
-		       V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
-	roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
-		       V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
-	roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
-		       V2_UD_SEND_WQE_BYTE_48_DMAC_4_S, ah->av.mac[4]);
-	roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
-		       V2_UD_SEND_WQE_BYTE_48_DMAC_5_S, ah->av.mac[5]);
-
 	ud_sq_wqe->msg_len = cpu_to_le32(msg_len);
 
-	/* Set sig attr */
 	roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_CQE_S,
-		     (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
+		     !!(wr->send_flags & IB_SEND_SIGNALED));
 
-	/* Set se attr */
 	roce_set_bit(ud_sq_wqe->byte_4, V2_UD_SEND_WQE_BYTE_4_SE_S,
-		     (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
+		     !!(wr->send_flags & IB_SEND_SOLICITED));
 
 	roce_set_field(ud_sq_wqe->byte_16, V2_UD_SEND_WQE_BYTE_16_PD_M,
 		       V2_UD_SEND_WQE_BYTE_16_PD_S, to_hr_pd(qp->ibqp.pd)->pdn);
@@ -476,35 +494,14 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 		       V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 		       curr_idx & (qp->sge.sge_cnt - 1));
 
-	roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
-		       V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
 	ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
 			  qp->qkey : ud_wr(wr)->remote_qkey);
 	roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
 		       V2_UD_SEND_WQE_BYTE_32_DQPN_S, ud_wr(wr)->remote_qpn);
 
-	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
-		       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S, ah->av.hop_limit);
-	roce_set_field(ud_sq_wqe->byte_36, V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
-		       V2_UD_SEND_WQE_BYTE_36_TCLASS_S, ah->av.tclass);
-	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
-		       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, ah->av.flowlabel);
-	roce_set_field(ud_sq_wqe->byte_40, V2_UD_SEND_WQE_BYTE_40_SL_M,
-		       V2_UD_SEND_WQE_BYTE_40_SL_S, ah->av.sl);
-
-	roce_set_field(ud_sq_wqe->byte_48, V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
-		       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S, ah->av.gid_index);
-
-	if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08) {
-		roce_set_bit(ud_sq_wqe->byte_40,
-			     V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
-			     ah->av.vlan_en);
-		roce_set_field(ud_sq_wqe->byte_36,
-			       V2_UD_SEND_WQE_BYTE_36_VLAN_M,
-			       V2_UD_SEND_WQE_BYTE_36_VLAN_S, ah->av.vlan_id);
-	}
-
-	memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN_V2);
+	ret = fill_ud_av(ud_sq_wqe, ah);
+	if (ret)
+		return ret;
 
 	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +3 -26
index 1466888..c068517 100644
@@ -1077,8 +1077,9 @@ struct hns_roce_v2_ud_send_wqe {
 	__le32	byte_32;
 	__le32	byte_36;
 	__le32	byte_40;
-	__le32	dmac;
-	__le32	byte_48;
+	u8	dmac[ETH_ALEN];
+	u8	sgid_index;
+	u8	smac_index;
 	u8	dgid[GID_LEN_V2];
 
 };
@@ -1125,30 +1126,6 @@ struct hns_roce_v2_ud_send_wqe {
 
 #define	V2_UD_SEND_WQE_BYTE_40_LBI_S 31
 
-#define	V2_UD_SEND_WQE_DMAC_0_S 0
-#define V2_UD_SEND_WQE_DMAC_0_M GENMASK(7, 0)
-
-#define	V2_UD_SEND_WQE_DMAC_1_S 8
-#define V2_UD_SEND_WQE_DMAC_1_M GENMASK(15, 8)
-
-#define	V2_UD_SEND_WQE_DMAC_2_S 16
-#define V2_UD_SEND_WQE_DMAC_2_M GENMASK(23, 16)
-
-#define	V2_UD_SEND_WQE_DMAC_3_S 24
-#define V2_UD_SEND_WQE_DMAC_3_M GENMASK(31, 24)
-
-#define	V2_UD_SEND_WQE_BYTE_48_DMAC_4_S 0
-#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_M GENMASK(7, 0)
-
-#define	V2_UD_SEND_WQE_BYTE_48_DMAC_5_S 8
-#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_M GENMASK(15, 8)
-
-#define	V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S 16
-#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M GENMASK(23, 16)
-
-#define	V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_S 24
-#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_M GENMASK(31, 24)
-
 struct hns_roce_v2_rc_send_wqe {
 	__le32		byte_4;
 	__le32		msg_len;
Signature · 1 line
2.8.1
author
#6 lore raw

PATCH v2 6/7 RDMA/hns: Add UD support for HIP09

In reply to Weihang Li

HIP09 supports service type of Unreliable Datagram, add necessary process to enable this feature.

---
 drivers/infiniband/hw/hns/hns_roce_ah.c    |  3 +++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c |  6 +++--
 drivers/infiniband/hw/hns/hns_roce_main.c  |  1 +
 drivers/infiniband/hw/hns/hns_roce_qp.c    | 39 ++++++++++++++++++++++--------
 4 files changed, 37 insertions(+), 12 deletions(-)
drivers/infiniband/hw/hns/hns_roce_ah.c +3 -0
index d65ff6a..b09ef33 100644
@@ -64,6 +64,9 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 	struct hns_roce_ah *ah = to_hr_ah(ibah);
 	int ret = 0;
 
+	if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 && udata)
+		return -EOPNOTSUPP;
+
 	ah->av.port = rdma_ah_get_port_num(ah_attr);
 	ah->av.gid_index = grh->sgid_index;
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +4 -2
index 273d968..57ff223 100644
@@ -693,7 +693,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 		       ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
 
 		/* Corresponding to the QP type, wqe process separately */
-		if (ibqp->qp_type == IB_QPT_GSI)
+		if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
 			ret = set_ud_wqe(qp, wr, wqe, &sge_idx, owner_bit);
 		else if (ibqp->qp_type == IB_QPT_RC)
 			ret = set_rc_wqe(qp, wr, wqe, &sge_idx, owner_bit);
@@ -5151,7 +5151,9 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 	unsigned long flags;
 	int ret = 0;
 
-	if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
+	if ((hr_qp->ibqp.qp_type == IB_QPT_RC ||
+	     hr_qp->ibqp.qp_type == IB_QPT_UD) &&
+	   hr_qp->state != IB_QPS_RESET) {
 		/* Modify qp to reset before destroying qp */
 		ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
 					    hr_qp->state, IB_QPS_RESET);
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index 97fdc55..f01590d 100644
@@ -424,6 +424,7 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.alloc_pd = hns_roce_alloc_pd,
 	.alloc_ucontext = hns_roce_alloc_ucontext,
 	.create_ah = hns_roce_create_ah,
+	.create_user_ah = hns_roce_create_ah,
 	.create_cq = hns_roce_create_cq,
 	.create_qp = hns_roce_create_qp,
 	.dealloc_pd = hns_roce_dealloc_pd,
drivers/infiniband/hw/hns/hns_roce_qp.c +29 -10
index e288946..5e505a3 100644
@@ -998,6 +998,30 @@ void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 	kfree(hr_qp);
 }
 
+static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,
+			 bool is_user)
+{
+	switch (type) {
+	case IB_QPT_UD:
+		if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 &&
+		    is_user)
+			goto out;
+		fallthrough;
+	case IB_QPT_RC:
+	case IB_QPT_GSI:
+		break;
+	default:
+		goto out;
+	}
+
+	return 0;
+
+out:
+	ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type);
+
+	return -EOPNOTSUPP;
+}
+
 struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
 				 struct ib_qp_init_attr *init_attr,
 				 struct ib_udata *udata)
@@ -1007,15 +1031,9 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
 	struct hns_roce_qp *hr_qp;
 	int ret;
 
-	switch (init_attr->qp_type) {
-	case IB_QPT_RC:
-	case IB_QPT_GSI:
-		break;
-	default:
-		ibdev_err(ibdev, "not support QP type %d\n",
-			  init_attr->qp_type);
-		return ERR_PTR(-EOPNOTSUPP);
-	}
+	ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata);
+	if (ret)
+		return ERR_PTR(ret);
 
 	hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
 	if (!hr_qp)
@@ -1030,10 +1048,11 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
 	if (ret) {
 		ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n",
 			  init_attr->qp_type, ret);
-		ibdev_err(ibdev, "Create GSI QP failed!\n");
+
 		kfree(hr_qp);
 		return ERR_PTR(ret);
 	}
+
 	return &hr_qp->ibqp;
 }
 
Signature · 1 line
2.8.1
author
#7 lore raw

PATCH v2 7/7 RDMA/hns: Add support for UD inline

In reply to Weihang Li

HIP09 supports UD inline up to size of 1024 Bytes. When data size is smaller than 8 bytes, they will be stored in sqwqe. Otherwise, the data will be filled into extended sges.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |   2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 111 ++++++++++++++++++++++++++--
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  14 ++++
 drivers/infiniband/hw/hns/hns_roce_qp.c     |   6 ++
 4 files changed, 126 insertions(+), 7 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
index 9a032d0..f54739b 100644
@@ -133,6 +133,7 @@ enum hns_roce_qp_caps {
 	HNS_ROCE_QP_CAP_RQ_RECORD_DB = BIT(0),
 	HNS_ROCE_QP_CAP_SQ_RECORD_DB = BIT(1),
 	HNS_ROCE_QP_CAP_OWNER_DB = BIT(2),
+	HNS_ROCE_QP_CAP_UD_SQ_INL = BIT(3),
 };
 
 enum hns_roce_cq_flags {
@@ -222,6 +223,7 @@ enum {
 	HNS_ROCE_CAP_FLAG_FRMR                  = BIT(8),
 	HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL		= BIT(9),
 	HNS_ROCE_CAP_FLAG_ATOMIC		= BIT(10),
+	HNS_ROCE_CAP_FLAG_UD_SQ_INL		= BIT(13),
 	HNS_ROCE_CAP_FLAG_SDI_MODE		= BIT(14),
 };
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +104 -7

147 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +14 -0
index c068517..1c1a773 100644
@@ -61,6 +61,8 @@
 #define HNS_ROCE_V2_MAX_SQ_SGE_NUM		64
 #define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM		0x200000
 #define HNS_ROCE_V2_MAX_SQ_INLINE		0x20
+#define HNS_ROCE_V2_MAX_SQ_INL_EXT		0x400
+#define HNS_ROCE_V2_MAX_UD_INL_INN_SZ		8
 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ		32
 #define HNS_ROCE_V2_UAR_NUM			256
 #define HNS_ROCE_V2_PHY_UAR_NUM			1
@@ -1126,6 +1128,18 @@ struct hns_roce_v2_ud_send_wqe {
 
 #define	V2_UD_SEND_WQE_BYTE_40_LBI_S 31
 
+#define V2_UD_SEND_WQE_BYTE_4_INL_S 12
+#define V2_UD_SEND_WQE_BYTE_20_INL_TYPE_S 31
+
+#define V2_UD_SEND_WQE_BYTE_8_INL_DATA_15_0_S 16
+#define V2_UD_SEND_WQE_BYTE_8_INL_DATA_15_0_M GENMASK(31, 16)
+#define V2_UD_SEND_WQE_BYTE_16_INL_DATA_23_16_S 24
+#define V2_UD_SEND_WQE_BYTE_16_INL_DATA_23_16_M GENMASK(31, 24)
+#define V2_UD_SEND_WQE_BYTE_20_INL_DATA_47_24_S 0
+#define V2_UD_SEND_WQE_BYTE_20_INL_DATA_47_24_M GENMASK(23, 0)
+#define V2_UD_SEND_WQE_BYTE_24_INL_DATA_63_48_S 0
+#define V2_UD_SEND_WQE_BYTE_24_INL_DATA_63_48_M GENMASK(15, 0)
+
 struct hns_roce_v2_rc_send_wqe {
 	__le32		byte_4;
 	__le32		msg_len;
drivers/infiniband/hw/hns/hns_roce_qp.c +6 -0
index 5e505a3..1210061 100644
@@ -862,6 +862,9 @@ static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
 		return ret;
 	}
 
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_UD_SQ_INL)
+		hr_qp->en_flags |= HNS_ROCE_QP_CAP_UD_SQ_INL;
+
 	if (udata) {
 		if (ib_copy_from_udata(ucmd, udata, sizeof(*ucmd))) {
 			ibdev_err(ibdev, "Failed to copy QP ucmd\n");
@@ -946,6 +949,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 	}
 
 	if (udata) {
+		if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_UD_SQ_INL)
+			resp.cap_flags |= HNS_ROCE_QP_CAP_UD_SQ_INL;
+
 		ret = ib_copy_to_udata(udata, &resp,
 				       min(udata->outlen, sizeof(resp)));
 		if (ret) {
Signature · 1 line
2.8.1
#8 lore raw

Re: [PATCH v2 for-next 7/7] RDMA/hns: Add support for UD inline

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static inline int set_ud_wqe(struct hns_roce_qp *qp, · line 587

Quoted 9 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +4 -1
@@ -503,7 +581,23 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (ret)
 		return ret;
 
-	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		ret = set_ud_inl(qp, wr, ud_sq_wqe, &curr_idx);
+		if (ret)
+			return ret;

Why are you implementing this in the kernel? No kernel ULP sets this flag?

Jason

author
#9 lore raw

Re: [PATCH v2 for-next 7/7] RDMA/hns: Add support for UD inline

Quoted 14 lines · Jason Gunthorpe

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static inline int set_ud_wqe(struct hns_roce_qp *qp, · line 587

Quoted 9 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +4 -1
@@ -503,7 +581,23 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (ret)
 		return ret;
 
-	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		ret = set_ud_inl(qp, wr, ud_sq_wqe, &curr_idx);
+		if (ret)
+			return ret;

Why are you implementing this in the kernel? No kernel ULP sets this flag?

Quoted 2 lines

Jason

Hi Jason,

Sorry, I don't understand. Some kernel users may set IB_SEND_INLINE when using UD, some may not, we just check this flag to decide how to fill the data into UD SQ WQE here.

Thanks
Weihang

maintainer
#10 lore raw

Re: [PATCH v2 for-next 7/7] RDMA/hns: Add support for UD inline

Quoted 15 lines · liweihang
Quoted 5 lines · Jason Gunthorpe
Quoted 4 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +0 -0
@@ -503,7 +581,23 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (ret)
 		return ret;
 
Quoted 8 lines

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static inline int set_ud_wqe(struct hns_roce_qp *qp, · line 587

Quoted 5 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +4 -1
-	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		ret = set_ud_inl(qp, wr, ud_sq_wqe, &curr_idx);
+		if (ret)
+			return ret;

Why are you implementing this in the kernel? No kernel ULP sets this flag?

Quoted 3 lines

Sorry, I don't understand. Some kernel users may set IB_SEND_INLINE when using UD, some may not, we just check this flag to decide how to fill the data into UD SQ WQE here.

I mean if you 'git grep IB_SEND_INLINE' nothing uses it.

This is all dead code.

How did you test it?

Jason

author
#11 lore raw

Re: [PATCH v2 for-next 7/7] RDMA/hns: Add support for UD inline

Quoted 20 lines · Jason Gunthorpe
Quoted 15 lines · liweihang
Quoted 5 lines · Jason Gunthorpe
Quoted 4 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +0 -0
@@ -503,7 +581,23 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (ret)
 		return ret;
 
Quoted 8 lines

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static inline int set_ud_wqe(struct hns_roce_qp *qp, · line 587

Quoted 5 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +4 -1
-	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		ret = set_ud_inl(qp, wr, ud_sq_wqe, &curr_idx);
+		if (ret)
+			return ret;

Why are you implementing this in the kernel? No kernel ULP sets this flag?

Quoted 3 lines

Sorry, I don't understand. Some kernel users may set IB_SEND_INLINE when using UD, some may not, we just check this flag to decide how to fill the data into UD SQ WQE here.

Quoted 8 lines

I mean if you 'git grep IB_SEND_INLINE' nothing uses it.

This is all dead code.

How did you test it?

Jason

Hi Jason,

We have tested it with our own tools. After running 'git grep IB_SEND_INLINE', I found following drivers refer to this flag:

bnxt_re/cxgb4/i40iw/mlx5/ocrdma/qedr/rxe/siw

So I'm still a bit confused. Do you mean that no kernel ULP uses UD inline or the IB_SEND_INLINE flag? Should current related codes be removed?

I also found a very old discussion about removing this flag, but there was no futher information:

https://lists.openfabrics.org/pipermail/general/2007-August/039826.html

I will appreciate it if you can give us more detailed suggestions.

Thanks
Weihang

#12 lore raw

Re: [PATCH v2 for-next 7/7] RDMA/hns: Add support for UD inline

Quoted 32 lines · liweihang
Quoted 20 lines · Jason Gunthorpe
Quoted 15 lines · liweihang
Quoted 5 lines · Jason Gunthorpe
Quoted 4 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +0 -0
@@ -503,7 +581,23 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (ret)
 		return ret;
 
Quoted 8 lines

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static inline int set_ud_wqe(struct hns_roce_qp *qp, · line 587

Quoted 5 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +4 -1
-	set_extend_sge(qp, wr, &curr_idx, valid_num_sge);
+	if (wr->send_flags & IB_SEND_INLINE) {
+		ret = set_ud_inl(qp, wr, ud_sq_wqe, &curr_idx);
+		if (ret)
+			return ret;

Why are you implementing this in the kernel? No kernel ULP sets this flag?

Quoted 3 lines

Sorry, I don't understand. Some kernel users may set IB_SEND_INLINE when using UD, some may not, we just check this flag to decide how to fill the data into UD SQ WQE here.

Quoted 8 lines

I mean if you 'git grep IB_SEND_INLINE' nothing uses it.

This is all dead code.

How did you test it?

Jason

Hi Jason,

Quoted 7 lines

We have tested it with our own tools. After running 'git grep IB_SEND_INLINE', I found following drivers refer to this flag:

bnxt_re/cxgb4/i40iw/mlx5/ocrdma/qedr/rxe/siw

So I'm still a bit confused. Do you mean that no kernel ULP uses UD inline or the IB_SEND_INLINE flag?

Yes

Quoted 1 line

Should current related codes be removed?

Quite possibly, though rxe and siw might be using it as part of the uAPI

Quoted 4 lines

I also found a very old discussion about removing this flag, but there was no futher information:

https://lists.openfabrics.org/pipermail/general/2007-August/039826.html

Right, it has been unused for a very long time, I doubt it will ever get used

Jason

maintainer Applied
#13 lore raw
In reply to Weihang Li
Quoted 11 lines · Weihang Li

This series does cleanups on UD related code at first, including removing dead code, adding necessary check and some refactors. Then the UD feature is enabled on HIP09.

Changes since v1:
- Don't allow HIP08's user to create AH and UD QP from userspace and add .create_user_ah in #6.
- Drop #4 from the v1 series which needs more discussion about the reserved sl.

Quoted 8 lines

Weihang Li (7):
RDMA/hns: Only record vlan info for HIP08
RDMA/hns: Fix missing fields in address vector
RDMA/hns: Avoid setting loopback indicator when smac is same as dmac RDMA/hns: Remove the portn field in UD SQ WQE
RDMA/hns: Simplify process of filling UD SQ WQE
RDMA/hns: Add UD support for HIP09
RDMA/hns: Add support for UD inline

Applied to for-next, thanks

Jason

maintainer
#14 lore raw
Quoted 20 lines · Jason Gunthorpe
Quoted 11 lines · Weihang Li

This series does cleanups on UD related code at first, including removing dead code, adding necessary check and some refactors. Then the UD feature is enabled on HIP09.

Changes since v1:
- Don't allow HIP08's user to create AH and UD QP from userspace and add .create_user_ah in #6.
- Drop #4 from the v1 series which needs more discussion about the reserved sl.

Quoted 7 lines

Weihang Li (7):
RDMA/hns: Only record vlan info for HIP08
RDMA/hns: Fix missing fields in address vector
RDMA/hns: Avoid setting loopback indicator when smac is same as dmac RDMA/hns: Remove the portn field in UD SQ WQE
RDMA/hns: Simplify process of filling UD SQ WQE
RDMA/hns: Add UD support for HIP09

Quoted 3 lines

Applied to for-next, thanks

Quoted 1 line

RDMA/hns: Add support for UD inline

But I didn't take this patch since it seems to be dead code

Jason