Skip to contentSkip to conversation

RDMA/hns: Refactor codes about memory

PATCH Superseded Weihang Li 2 participants 5 messages Jan 21, 2021

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Weihang Li

author
#0 lore raw

cover RDMA/hns: Refactor codes about memory

This series refactors memory related codes, including MR, MTR(Memory Translate Region) and MPT(Memory Protection Table).

Lang Cheng (2):
RDMA/hns: Optimize the MR registration process RDMA/hns: Use new interface to set MPT related fields

Xi Wang (1):
RDMA/hns: Refactor the MTR creation flow

 drivers/infiniband/hw/hns/hns_roce_common.h |  22 ++
 drivers/infiniband/hw/hns/hns_roce_cq.c     |   1 -
 drivers/infiniband/hw/hns/hns_roce_device.h |   4 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |  77 +++---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  39 +++
 drivers/infiniband/hw/hns/hns_roce_mr.c     | 404 +++++++++++-----------------
 drivers/infiniband/hw/hns/hns_roce_qp.c     |   1 -
 drivers/infiniband/hw/hns/hns_roce_srq.c    |   2 -
 8 files changed, 258 insertions(+), 292 deletions(-)
Signature · 1 line
2.8.1
author
#1 lore raw

PATCH 1/3 RDMA/hns: Refactor the MTR creation flow

From: Xi Wang <[email protected]>

Split the hns_roce_mtr_create() into serval small functions, remove unused member in 'struct hns_roce_buf_attr' and delete unnecessary MTR page count check flow to make the MTR creation related codes clearer.

---
 drivers/infiniband/hw/hns/hns_roce_cq.c     |   1 -
 drivers/infiniband/hw/hns/hns_roce_device.h |   1 -
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |   1 -
 drivers/infiniband/hw/hns/hns_roce_mr.c     | 291 ++++++++++++----------------
 drivers/infiniband/hw/hns/hns_roce_qp.c     |   1 -
 drivers/infiniband/hw/hns/hns_roce_srq.c    |   2 -
 6 files changed, 125 insertions(+), 172 deletions(-)
drivers/infiniband/hw/hns/hns_roce_cq.c +0 -1
index ffb7f7e..74fc494 100644
@@ -206,7 +206,6 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
 	buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size;
 	buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num;
 	buf_attr.region_count = 1;
-	buf_attr.fixed_page = true;
 
 	ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
 				  hr_dev->caps.cqe_ba_pg_sz + HNS_HW_PAGE_SHIFT,
drivers/infiniband/hw/hns/hns_roce_device.h +0 -1
index c46b330..ffed82d 100644
@@ -335,7 +335,6 @@ struct hns_roce_buf_attr {
 	} region[HNS_ROCE_MAX_BT_REGION];
 	unsigned int region_count; /* valid region count */
 	unsigned int page_shift;  /* buffer page shift */
-	bool fixed_page; /* decide page shift is fixed-size or maximum size */
 	unsigned int user_access; /* umem access flag */
 	bool mtt_only; /* only alloc buffer-required MTT memory */
 };
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +0 -1
index 4c06889..110354b 100644
@@ -5948,7 +5948,6 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 	buf_attr.region[0].size = eq->entries * eq->eqe_size;
 	buf_attr.region[0].hopnum = eq->hop_num;
 	buf_attr.region_count = 1;
-	buf_attr.fixed_page = true;
 
 	err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
 				  hr_dev->caps.eqe_ba_pg_sz +
drivers/infiniband/hw/hns/hns_roce_mr.c +125 -166

393 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_qp.c +0 -1
index d8e2fe5..9988ca9 100644
@@ -599,7 +599,6 @@ static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
 		return -EINVAL;
 
 	buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
-	buf_attr->fixed_page = true;
 	buf_attr->region_count = idx;
 
 	return 0;
drivers/infiniband/hw/hns/hns_roce_srq.c +0 -2
index c4ae57e..9403828 100644
@@ -194,7 +194,6 @@ static int alloc_srq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 							 srq->wqe_shift);
 	buf_attr.region[0].hopnum = hr_dev->caps.srqwqe_hop_num;
 	buf_attr.region_count = 1;
-	buf_attr.fixed_page = true;
 
 	err = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
 				  hr_dev->caps.srqwqe_ba_pg_sz +
@@ -226,7 +225,6 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
 					srq->idx_que.entry_shift);
 	buf_attr.region[0].hopnum = hr_dev->caps.idx_hop_num;
 	buf_attr.region_count = 1;
-	buf_attr.fixed_page = true;
 
 	err = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
 				  hr_dev->caps.idx_ba_pg_sz + HNS_HW_PAGE_SHIFT,
Signature · 1 line
2.8.1
author
#2 lore raw

PATCH 2/3 RDMA/hns: Optimize the MR registration process

In reply to Weihang Li

From: Lang Cheng <[email protected]>

When creating or re-registering an MR, storing the PDN, access flag and IOVA information ASAP can simplify the number of parameters passed into the subsequent process.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |   3 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |  22 ++----
 drivers/infiniband/hw/hns/hns_roce_mr.c     | 113 ++++++++++------------------
 3 files changed, 48 insertions(+), 90 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -2
index ffed82d..9f0b73b 100644
@@ -914,8 +914,7 @@ struct hns_roce_hw {
 	int (*write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,
 			  struct hns_roce_mr *mr, unsigned long mtpt_idx);
 	int (*rereg_write_mtpt)(struct hns_roce_dev *hr_dev,
-				struct hns_roce_mr *mr, int flags, u32 pdn,
-				int mr_access_flags, u64 iova, u64 size,
+				struct hns_roce_mr *mr, int flags,
 				void *mb_buf);
 	int (*frmr_write_mtpt)(struct hns_roce_dev *hr_dev, void *mb_buf,
 			       struct hns_roce_mr *mr);
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +8 -14
index 110354b..16ef631 100644
@@ -2927,20 +2927,17 @@ static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
 
 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
 					struct hns_roce_mr *mr, int flags,
-					u32 pdn, int mr_access_flags, u64 iova,
-					u64 size, void *mb_buf)
+					void *mb_buf)
 {
 	struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
+	u32 mr_access_flags = mr->access;
 	int ret = 0;
 
 	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
 		       V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
 
-	if (flags & IB_MR_REREG_PD) {
-		roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
-			       V2_MPT_BYTE_4_PD_S, pdn);
-		mr->pd = pdn;
-	}
+	roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
+		       V2_MPT_BYTE_4_PD_S, mr->pd);
 
 	if (flags & IB_MR_REREG_ACCESS) {
 		roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
@@ -2958,13 +2955,10 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
 	}
 
 	if (flags & IB_MR_REREG_TRANS) {
-		mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
-		mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
-		mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
-		mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
-
-		mr->iova = iova;
-		mr->size = size;
+		mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
+		mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
+		mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
+		mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
 
 		ret = set_mtpt_pbl(hr_dev, mpt_entry, mr);
 	}
drivers/infiniband/hw/hns/hns_roce_mr.c +39 -74

225 lines not on this page · raw message

Signature · 1 line
2.8.1
#3 lore raw

Re: [PATCH for-next 2/3] RDMA/hns: Optimize the MR registration process

Thank you for the patch! Perhaps something to improve:

Hi Weihang,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on rdma/for-next]
[also build test WARNING on v5.11-rc4 next-20210121]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Weihang-Li/RDMA-hns-Refactor-codes-about-memory/20210121-215817
base:   https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git for-next
config: sparc-allyesconfig (attached as .config)
compiler: sparc64-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/9ff47c3f8e6b778282c3bc9d4fb65ce835bdd83c
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Weihang-Li/RDMA-hns-Refactor-codes-about-memory/20210121-215817 git checkout 9ff47c3f8e6b778282c3bc9d4fb65ce835bdd83c
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=sparc

If you fix the issue, kindly add following tag as appropriate

All warnings (new ones prefixed by >>):

>> drivers/infiniband/hw/hns/hns_roce_mr.c:140:6: warning: no previous prototype for 'hns_roce_mr_free' [-Wmissing-prototypes]

140 | void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)
| ^~~~~~~~~~~~~~~~

vim +/hns_roce_mr_free +140 drivers/infiniband/hw/hns/hns_roce_mr.c

139

 > 140	void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)

141 {
142 struct ib_device *ibdev = &hr_dev->ib_dev;
143 int ret;
144
145 if (mr->enabled) {
146 ret = hns_roce_hw_destroy_mpt(hr_dev, NULL,
147 key_to_hw_index(mr->key) &
148 (hr_dev->caps.num_mtpts - 1));
149 if (ret)
150 ibdev_warn(ibdev, "failed to destroy mpt, ret = %d.\n",
151 ret);
152 }
153
154 free_mr_pbl(hr_dev, mr);
155 free_mr_key(hr_dev, mr);
156 }
157

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

author
#4 lore raw

PATCH 3/3 RDMA/hns: Use new interface to set MPT related fields

In reply to Weihang Li

From: Lang Cheng <[email protected]>

Achieve hr_reg_write() to simply the codes to fill fields.

---
 drivers/infiniband/hw/hns/hns_roce_common.h | 22 ++++++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 54 +++++++++++++----------------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  | 39 +++++++++++++++++++++
 3 files changed, 85 insertions(+), 30 deletions(-)
drivers/infiniband/hw/hns/hns_roce_common.h +22 -0
index 5afee04..3ca6e88 100644
@@ -32,6 +32,7 @@
 
 #ifndef _HNS_ROCE_COMMON_H
 #define _HNS_ROCE_COMMON_H
+#include <linux/bitfield.h>
 
 #define roce_write(dev, reg, val)	writel((val), (dev)->reg_base + (reg))
 #define roce_read(dev, reg)		readl((dev)->reg_base + (reg))
@@ -65,6 +66,27 @@
 
 #define hr_reg_enable(ptr, field) _hr_reg_enable(ptr, field)
 
+#define _hr_reg_clear(ptr, field_type, field_h, field_l)                       \
+	({                                                                     \
+		const field_type *_ptr = ptr;                                  \
+		*((__le32 *)_ptr + (field_h) / 32) &=                          \
+			cpu_to_le32(                                           \
+				~GENMASK((field_h) % 32, (field_l) % 32)) +    \
+			BUILD_BUG_ON_ZERO(((field_h) / 32) !=                  \
+					  ((field_l) / 32));                   \
+	})
+
+#define hr_reg_clear(ptr, field) _hr_reg_clear(ptr, field)
+
+#define _hr_reg_write(ptr, field_type, field_h, field_l, val)                  \
+	({                                                                     \
+		_hr_reg_clear(ptr, field_type, field_h, field_l);              \
+		*((__le32 *)ptr + (field_h) / 32) |= cpu_to_le32(FIELD_PREP(   \
+			GENMASK((field_h) % 32, (field_l) % 32), val));        \
+	})
+
+#define hr_reg_write(ptr, field, val) _hr_reg_write(ptr, field, val)
+
 #define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
 #define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +24 -30

69 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +39 -0
index bdaccf8..69bc072 100644
@@ -993,6 +993,45 @@ struct hns_roce_v2_mpt_entry {
 	__le32	byte_64_buf_pa1;
 };
 
+#define MPT_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_mpt_entry, h, l)
+
+#define MPT_ST MPT_FIELD_LOC(1, 0)
+#define MPT_PBL_HOP_NUM MPT_FIELD_LOC(3, 2)
+#define MPT_PBL_BA_PG_SZ MPT_FIELD_LOC(7, 4)
+#define MPT_PD MPT_FIELD_LOC(31, 8)
+#define MPT_RA_EN MPT_FIELD_LOC(32, 32)
+#define MPT_R_INV_EN MPT_FIELD_LOC(33, 33)
+#define MPT_L_INV_EN MPT_FIELD_LOC(34, 34)
+#define MPT_BIND_EN MPT_FIELD_LOC(35, 35)
+#define MPT_ATOMIC_EN MPT_FIELD_LOC(36, 36)
+#define MPT_RR_EN MPT_FIELD_LOC(37, 37)
+#define MPT_RW_EN MPT_FIELD_LOC(38, 38)
+#define MPT_LW_EN MPT_FIELD_LOC(39, 39)
+#define MPT_MW_CNT MPT_FIELD_LOC(63, 40)
+#define MPT_FRE MPT_FIELD_LOC(64, 64)
+#define MPT_PA MPT_FIELD_LOC(65, 65)
+#define MPT_ZBVA MPT_FIELD_LOC(66, 66)
+#define MPT_SHARE MPT_FIELD_LOC(67, 67)
+#define MPT_MR_MW MPT_FIELD_LOC(68, 68)
+#define MPT_BPD MPT_FIELD_LOC(69, 69)
+#define MPT_BQP MPT_FIELD_LOC(70, 70)
+#define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71)
+#define MPT_MW_BIND_QPN MPT_FIELD_LOC(95, 72)
+#define MPT_BOUND_LKEY MPT_FIELD_LOC(127, 96)
+#define MPT_LEN MPT_FIELD_LOC(191, 128)
+#define MPT_LKEY MPT_FIELD_LOC(223, 192)
+#define MPT_VA MPT_FIELD_LOC(287, 224)
+#define MPT_PBL_SIZE MPT_FIELD_LOC(319, 288)
+#define MPT_PBL_BA MPT_FIELD_LOC(380, 320)
+#define MPT_BLK_MODE MPT_FIELD_LOC(381, 381)
+#define MPT_RSV0 MPT_FIELD_LOC(383, 382)
+#define MPT_PA0 MPT_FIELD_LOC(441, 384)
+#define MPT_BOUND_VA MPT_FIELD_LOC(447, 442)
+#define MPT_PA1 MPT_FIELD_LOC(505, 448)
+#define MPT_PERSIST_EN MPT_FIELD_LOC(506, 506)
+#define MPT_RSV2 MPT_FIELD_LOC(507, 507)
+#define MPT_PBL_BUF_PG_SZ MPT_FIELD_LOC(511, 508)
+
 #define V2_MPT_BYTE_4_MPT_ST_S 0
 #define V2_MPT_BYTE_4_MPT_ST_M GENMASK(1, 0)
 
Signature · 1 line
2.8.1