RDMA/erdma: Add hierachical MTT support
Subsystems (2)
Cheng Xu
maintainercover RDMA/erdma: Add hierachical MTT support
Currently, erdma only supports 0-level or 1-level MTT, which limits the maximum length of MR. It fails to meet the requirements in some scenarios. Therefore, we implement hierachical MTT to support 2-level and 3-level MTT, so that users can register MRs large to 64G to erdma devices.
- #1 ~ #2 make some preparations, such as renaming certain variables and refactoring the storage structure of MTT.
- #3 implements the hierachical MTT.
Cheng Xu (3):
RDMA/erdma: Renaming variable names and field names of struct
erdma_mem
RDMA/erdma: Refactor the storage structure of MTT entries RDMA/erdma: Implement hierachical MTT
drivers/infiniband/hw/erdma/erdma_hw.h | 18 +- drivers/infiniband/hw/erdma/erdma_qp.c | 2 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 430 ++++++++++++++++------ drivers/infiniband/hw/erdma/erdma_verbs.h | 36 +- 4 files changed, 357 insertions(+), 129 deletions(-)
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH 1/3 RDMA/erdma: Renaming variable names and field names of struct erdma_mem
Currently, variable names and field names of struct erdma_mem contain 'mtt', which is not accurate. Renaming them with 'xxx_mem' or 'mem'.
--- drivers/infiniband/hw/erdma/erdma_verbs.c | 66 +++++++++++------------ drivers/infiniband/hw/erdma/erdma_verbs.h | 8 +-- 2 files changed, 37 insertions(+), 37 deletions(-)
drivers/infiniband/hw/erdma/erdma_verbs.c +33 -33
@@ -67,30 +67,30 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) user_qp = &qp->user_qp; req.sq_cqn_mtt_cfg =FIELD_PREP (ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK , -ilog2 (user_qp->sq_mtt.page_size) -ERDMA_HW_PAGE_SHIFT ); +ilog2 (user_qp->sq_mem.page_size) -ERDMA_HW_PAGE_SHIFT ); req.sq_cqn_mtt_cfg |=FIELD_PREP (ERDMA_CMD_CREATE_QP_CQN_MASK , qp->scq->cqn); req.rq_cqn_mtt_cfg =FIELD_PREP (ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK , -ilog2 (user_qp->rq_mtt.page_size) -ERDMA_HW_PAGE_SHIFT ); +ilog2 (user_qp->rq_mem.page_size) -ERDMA_HW_PAGE_SHIFT ); req.rq_cqn_mtt_cfg |=FIELD_PREP (ERDMA_CMD_CREATE_QP_CQN_MASK , qp->rcq->cqn); - req.sq_mtt_cfg = user_qp->sq_mtt.page_offset; + req.sq_mtt_cfg = user_qp->sq_mem.page_offset; req.sq_mtt_cfg |=FIELD_PREP (ERDMA_CMD_CREATE_QP_MTT_CNT_MASK , - user_qp->sq_mtt.mtt_nents) | + user_qp->sq_mem.mtt_nents) |FIELD_PREP (ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK , - user_qp->sq_mtt.mtt_type); + user_qp->sq_mem.mtt_type); - req.rq_mtt_cfg = user_qp->rq_mtt.page_offset; + req.rq_mtt_cfg = user_qp->rq_mem.page_offset; req.rq_mtt_cfg |=FIELD_PREP (ERDMA_CMD_CREATE_QP_MTT_CNT_MASK , - user_qp->rq_mtt.mtt_nents) | + user_qp->rq_mem.mtt_nents) |FIELD_PREP (ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK , - user_qp->rq_mtt.mtt_type); + user_qp->rq_mem.mtt_type); - req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0 ]; - req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0 ]; + req.sq_buf_addr = user_qp->sq_mem.mtt_entry[0 ]; + req.rq_buf_addr = user_qp->rq_mem.mtt_entry[0 ]; req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr;
@@ -161,7 +161,7 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) {struct erdma_dev *dev =to_edev (cq->ibcq.device);struct erdma_cmdq_create_cq_req req; -struct erdma_mem *mtt; +struct erdma_mem *mem;u32 page_size; erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
@@ -186,23 +186,23 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) req.cq_db_info_addr = cq->kern_cq.qbuf_dma_addr + (cq->depth <<CQE_SHIFT ); }else { - mtt = &cq->user_cq.qbuf_mtt; + mem = &cq->user_cq.qbuf_mem; req.cfg0 |=FIELD_PREP (ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK , -ilog2 (mtt->page_size) -ERDMA_HW_PAGE_SHIFT ); -if (mtt->mtt_nents ==1 ) { - req.qbuf_addr_l =lower_32_bits (*(u64 *)mtt->mtt_buf); - req.qbuf_addr_h =upper_32_bits (*(u64 *)mtt->mtt_buf); +ilog2 (mem->page_size) -ERDMA_HW_PAGE_SHIFT ); +if (mem->mtt_nents ==1 ) { + req.qbuf_addr_l =lower_32_bits (*(u64 *)mem->mtt_buf); + req.qbuf_addr_h =upper_32_bits (*(u64 *)mem->mtt_buf); }else { - req.qbuf_addr_l =lower_32_bits (mtt->mtt_entry[0 ]); - req.qbuf_addr_h =upper_32_bits (mtt->mtt_entry[0 ]); + req.qbuf_addr_l =lower_32_bits (mem->mtt_entry[0 ]); + req.qbuf_addr_h =upper_32_bits (mem->mtt_entry[0 ]); } req.cfg1 |=FIELD_PREP (ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK , - mtt->mtt_nents); + mem->mtt_nents); req.cfg1 |=FIELD_PREP (ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK , - mtt->mtt_type); + mem->mtt_type); - req.first_page_offset = mtt->page_offset; + req.first_page_offset = mem->page_offset; req.cq_db_info_addr = cq->user_cq.db_info_dma_addr;if (uctx->ext_db.enable) {
@@ -660,7 +660,7 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, qp->attrs.rq_size * RQE_SIZE))return -EINVAL ; - ret =get_mtt_entries (qp->dev, &qp->user_qp.sq_mtt, va, + ret =get_mtt_entries (qp->dev, &qp->user_qp.sq_mem, va, qp->attrs.sq_size <<SQEBB_SHIFT ,0 , va, (SZ_1M -SZ_4K ),1 );if (ret)
@@ -669,7 +669,7 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, rq_offset =ALIGN (qp->attrs.sq_size <<SQEBB_SHIFT ,ERDMA_HW_PAGE_SIZE ); qp->user_qp.rq_offset = rq_offset; - ret =get_mtt_entries (qp->dev, &qp->user_qp.rq_mtt, va + rq_offset, + ret =get_mtt_entries (qp->dev, &qp->user_qp.rq_mem, va + rq_offset, qp->attrs.rq_size <<RQE_SHIFT ,0 , va + rq_offset, (SZ_1M -SZ_4K ),1 );if (ret)
@@ -687,18 +687,18 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,return 0 ; put_rq_mtt: -put_mtt_entries (qp->dev, &qp->user_qp.rq_mtt); +put_mtt_entries (qp->dev, &qp->user_qp.rq_mem); put_sq_mtt: -put_mtt_entries (qp->dev, &qp->user_qp.sq_mtt); +put_mtt_entries (qp->dev, &qp->user_qp.sq_mem);return ret; }static void free_user_qp (struct erdma_qp *qp,struct erdma_ucontext *uctx) { -put_mtt_entries (qp->dev, &qp->user_qp.sq_mtt); -put_mtt_entries (qp->dev, &qp->user_qp.rq_mtt); +put_mtt_entries (qp->dev, &qp->user_qp.sq_mem); +put_mtt_entries (qp->dev, &qp->user_qp.rq_mem);erdma_unmap_user_dbrecords (uctx, &qp->user_qp.user_dbr_page); }
@@ -1041,7 +1041,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); }else {erdma_unmap_user_dbrecords (ctx, &cq->user_cq.user_dbr_page); -put_mtt_entries (dev, &cq->user_cq.qbuf_mtt); +put_mtt_entries (dev, &cq->user_cq.qbuf_mem); }xa_erase (&dev->cq_xa, cq->cqn);
@@ -1089,8 +1089,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)WARPPED_BUFSIZE (qp->attrs.sq_size <<SQEBB_SHIFT ), qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); }else { -put_mtt_entries (dev, &qp->user_qp.sq_mtt); -put_mtt_entries (dev, &qp->user_qp.rq_mtt); +put_mtt_entries (dev, &qp->user_qp.sq_mem); +put_mtt_entries (dev, &qp->user_qp.rq_mem);erdma_unmap_user_dbrecords (ctx, &qp->user_qp.user_dbr_page); }
@@ -1379,7 +1379,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,int ret;struct erdma_dev *dev =to_edev (cq->ibcq.device); - ret =get_mtt_entries (dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va, + ret =get_mtt_entries (dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, ureq->qbuf_len,0 , ureq->qbuf_va,SZ_64M -SZ_4K ,1 );if (ret)
@@ -1389,7 +1389,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, &cq->user_cq.user_dbr_page, &cq->user_cq.db_info_dma_addr);if (ret) -put_mtt_entries (dev, &cq->user_cq.qbuf_mtt); +put_mtt_entries (dev, &cq->user_cq.qbuf_mem);return ret; }
@@ -1473,7 +1473,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, err_free_res:if (!rdma_is_kernel_res (&ibcq->res)) {erdma_unmap_user_dbrecords (ctx, &cq->user_cq.user_dbr_page); -put_mtt_entries (dev, &cq->user_cq.qbuf_mtt); +put_mtt_entries (dev, &cq->user_cq.qbuf_mem); }else { dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(depth << CQE_SHIFT),
drivers/infiniband/hw/erdma/erdma_verbs.h +4 -4
@@ -65,7 +65,7 @@ struct erdma_pd {* MemoryRegion definition. */ #define ERDMA_MAX_INLINE_MTT_ENTRIES 4 -#define MTT_SIZE (mtt_cnt) (mtt_cnt << 3)/* per mtt takes 8 Bytes. */ +#define MTT_SIZE (mtt_cnt) (mtt_cnt << 3)/* per mtt entry takes 8 Bytes. */ #define ERDMA_MR_MAX_MTT_CNT 524288#define ERDMA_MTT_ENTRY_SIZE 8
@@ -121,8 +121,8 @@ struct erdma_user_dbrecords_page { };struct erdma_uqp { -struct erdma_mem sq_mtt; -struct erdma_mem rq_mtt; +struct erdma_mem sq_mem; +struct erdma_mem rq_mem;dma_addr_t sq_db_info_dma_addr;dma_addr_t rq_db_info_dma_addr;
@@ -234,7 +234,7 @@ struct erdma_kcq_info { };struct erdma_ucq_info { -struct erdma_mem qbuf_mtt; +struct erdma_mem qbuf_mem;struct erdma_user_dbrecords_page *user_dbr_page;dma_addr_t db_info_dma_addr; };
Signature · 1 line
2.31.1
Cheng Xu
maintainerPATCH 2/3 RDMA/erdma: Refactor the storage structure of MTT entries
In reply to Cheng XuCurrently our MTT only support inline mtt entries (0 level MTT) and indirect MTT entries (1 level mtt), which will limit the maximum length of MRs. In order to implement a multi-level MTT, we refactor the structure of MTT first.
--- drivers/infiniband/hw/erdma/erdma_hw.h | 4 +- drivers/infiniband/hw/erdma/erdma_qp.c | 2 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 214 +++++++++++++--------- drivers/infiniband/hw/erdma/erdma_verbs.h | 26 ++- 4 files changed, 152 insertions(+), 94 deletions(-)
drivers/infiniband/hw/erdma/erdma_hw.h +2 -2
@@ -228,7 +228,7 @@ struct erdma_cmdq_ext_db_req {/* create_cq cfg1 */ #define ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK GENMASK(31, 16) -#define ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK BIT(15) +#define ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK BIT(15)#define ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK BIT(11)#define ERDMA_CMD_CREATE_CQ_EQN_MASK GENMASK(9, 0)
@@ -258,7 +258,7 @@ struct erdma_cmdq_create_cq_req {/* regmr cfg2 */ #define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27) -#define ERDMA_CMD_REGMR_MTT_TYPE_MASK GENMASK(21, 20) +#define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)#define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)struct erdma_cmdq_reg_mr_req {
drivers/infiniband/hw/erdma/erdma_qp.c +1 -1
@@ -410,7 +410,7 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,/* Copy SGLs to SQE content to accelerate */ memcpy (get_queue_entry (qp->kern_qp.sq_buf, idx +1 , qp->attrs.sq_size,SQEBB_SHIFT ), - mr->mem.mtt_buf,MTT_SIZE (mr->mem.mtt_nents)); + mr->mem.mtt->buf,MTT_SIZE (mr->mem.mtt_nents)); wqe_size =sizeof (struct erdma_reg_mr_sqe ) +MTT_SIZE (mr->mem.mtt_nents); }else {
drivers/infiniband/hw/erdma/erdma_verbs.c +128 -86
312 lines not on this page · raw message
drivers/infiniband/hw/erdma/erdma_verbs.h +21 -5
@@ -65,7 +65,7 @@ struct erdma_pd {* MemoryRegion definition. */ #define ERDMA_MAX_INLINE_MTT_ENTRIES 4 -#define MTT_SIZE (mtt_cnt) (mtt_cnt << 3)/* per mtt entry takes 8 Bytes. */ +#define MTT_SIZE (mtt_cnt) ((mtt_cnt) << 3)/* per mtt entry takes 8 Bytes. */ #define ERDMA_MR_MAX_MTT_CNT 524288#define ERDMA_MTT_ENTRY_SIZE 8
@@ -90,10 +90,28 @@ static inline u8 to_erdma_access_flags(int access) (access &IB_ACCESS_REMOTE_ATOMIC ?ERDMA_MR_ACC_RA :0 ); } +/* Hierarchical storage structure for MTT entries */ +struct erdma_mtt { +u64 *buf; +size_t size; + +bool continuous; +union { +dma_addr_t buf_dma; +struct { +struct scatterlist *sglist; +u32 nsg; +u32 level; + }; + }; + +struct erdma_mtt *low_level; +}; +struct erdma_mem {struct ib_umem *umem; -void *mtt_buf; -u32 mtt_type; +struct erdma_mtt *mtt; +u32 page_size;u32 page_offset;u32 page_cnt;
@@ -101,8 +119,6 @@ struct erdma_mem {u64 va;u64 len; - -u64 mtt_entry[ERDMA_MAX_INLINE_MTT_ENTRIES ]; };struct erdma_mr {
Signature · 1 line
2.31.1
Re: [PATCH for-next 2/3] RDMA/erdma: Refactor the storage structure of MTT entries
Quoted 12 lines · Cheng Xu
Currently our MTT only support inline mtt entries (0 level MTT) and indirect MTT entries (1 level mtt), which will limit the maximum length of MRs. In order to implement a multi-level MTT, we refactor the structure of MTT first.
Signed-off-by Cheng Xu <[email protected]>--- drivers/infiniband/hw/erdma/erdma_hw.h | 4 +- drivers/infiniband/hw/erdma/erdma_qp.c | 2 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 214 +++++++++++++--------- drivers/infiniband/hw/erdma/erdma_verbs.h | 26 ++- 4 files changed, 152 insertions(+), 94 deletions(-)
<...>
Quoted 6 lines
drivers/infiniband/hw/erdma/erdma_verbs.h +6
-0+/* Hierarchical storage structure for MTT entries */ +struct erdma_mtt { +u64 *buf; +size_t size; + +bool continuous;
drivers/infiniband/hw/erdma/erdma_verbs.h static inline u8 to_erdma_access_flags(int access) · line 108
Quoted 10 lines
drivers/infiniband/hw/erdma/erdma_verbs.h +10
-0+union { +dma_addr_t buf_dma; +struct { +struct scatterlist *sglist; +u32 nsg; +u32 level; + }; + }; + +struct erdma_mtt *low_level;
This variable is used in third patch only, but please don't resubmit yet.
Thanks
Cheng Xu
maintainerPATCH 3/3 RDMA/erdma: Implement hierachical MTT
In reply to Cheng XuHierarchical MTT allows large MR registration without the need of continous physical address. This commit add the support of hierachical MTT support for erdma.
--- drivers/infiniband/hw/erdma/erdma_hw.h | 14 +- drivers/infiniband/hw/erdma/erdma_verbs.c | 200 +++++++++++++++++++--- drivers/infiniband/hw/erdma/erdma_verbs.h | 4 +- 3 files changed, 194 insertions(+), 24 deletions(-)
drivers/infiniband/hw/erdma/erdma_hw.h +12 -2
@@ -248,6 +248,7 @@ struct erdma_cmdq_create_cq_req {/* regmr/deregmr cfg0 */ #define ERDMA_CMD_MR_VALID_MASK BIT(31) +#define ERDMA_CMD_MR_VERSION_MASK GENMASK(30, 28)#define ERDMA_CMD_MR_KEY_MASK GENMASK(27, 20)#define ERDMA_CMD_MR_MPT_IDX_MASK GENMASK(19, 0)
@@ -258,6 +259,7 @@ struct erdma_cmdq_create_cq_req {/* regmr cfg2 */ #define ERDMA_CMD_REGMR_PAGESIZE_MASK GENMASK(31, 27) +#define ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK GENMASK(26, 24)#define ERDMA_CMD_REGMR_MTT_LEVEL_MASK GENMASK(21, 20)#define ERDMA_CMD_REGMR_MTT_CNT_MASK GENMASK(19, 0)
@@ -268,7 +270,14 @@ struct erdma_cmdq_reg_mr_req {u64 start_va;u32 size;u32 cfg2; -u64 phy_addr[4 ]; +union { +u64 phy_addr[4 ]; +struct { +u64 rsvd; +u32 size_h; +u32 mtt_cnt_h; + }; + }; };struct erdma_cmdq_dereg_mr_req {
@@ -309,7 +318,7 @@ struct erdma_cmdq_modify_qp_req {/* create qp mtt_cfg */ #define ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK GENMASK(31, 12)#define ERDMA_CMD_CREATE_QP_MTT_CNT_MASK GENMASK(11, 1) -#define ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK BIT(0) +#define ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK BIT(0)/* create qp db cfg */ #define ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK GENMASK(31, 16)
@@ -364,6 +373,7 @@ struct erdma_cmdq_reflush_req {enum {ERDMA_DEV_CAP_FLAGS_ATOMIC =1 <<7 , +ERDMA_DEV_CAP_FLAGS_MTT_VA =1 <<5 ,ERDMA_DEV_CAP_FLAGS_EXTEND_DB =1 <<3 , };
drivers/infiniband/hw/erdma/erdma_verbs.c +180 -20
315 lines not on this page · raw message
drivers/infiniband/hw/erdma/erdma_verbs.h +2 -2
@@ -73,8 +73,8 @@ struct erdma_pd {#define ERDMA_MR_TYPE_FRMR 1#define ERDMA_MR_TYPE_DMA 2 -#define ERDMA_MR_INLINE_MTT 0 -#define ERDMA_MR_INDIRECT_MTT 1 +#define ERDMA_MR_MTT_0LEVEL 0 +#define ERDMA_MR_MTT_1LEVEL 1#define ERDMA_MR_ACC_RA BIT(0)#define ERDMA_MR_ACC_LR BIT(1)
Signature · 1 line
2.31.1
Leon Romanovsky
AppliedQuoted 10 lines · Cheng Xu
Currently, erdma only supports 0-level or 1-level MTT, which limits the maximum length of MR. It fails to meet the requirements in some scenarios. Therefore, we implement hierachical MTT to support 2-level and 3-level MTT, so that users can register MRs large to 64G to erdma devices.
- #1 ~ #2 make some preparations, such as renaming certain variables and refactoring the storage structure of MTT.
- #3 implements the hierachical MTT.[...]
Applied, thanks!
[1/3] RDMA/erdma: Renaming variable names and field names of struct erdma_mem
https://git.kernel.org/rdma/rdma/c/d7cfbba90b8015 [2/3] RDMA/erdma: Refactor the storage structure of MTT entries https://git.kernel.org/rdma/rdma/c/7244b4aa4221d7 [3/3] RDMA/erdma: Implement hierachical MTT https://git.kernel.org/rdma/rdma/c/ed10435d358314
Best regards,
Signature · 1 line
Leon Romanovsky <[email protected]>