RDMA/hns: Adjust definition of FRMR fields
Subsystems (2)
PATCH v3 1/1 RDMA/hns: Adjust definition of FRMR fields
From: Yixing Liu <[email protected]>
FRMR is not well-supported on HIP08, it is re-designed for HIP09 and the position of related fields is changed. Then the ULPs should be forbidden to use FRMR on older hardwares.
---
This patch is one of series "RDMA/hns: Updates for 5.12", the others have been applied.
Changes since v2:
* Aovid setting IB_DEVICE_MEM_MGT_EXTENSIONS for HIP08 or older hardwares.
* Link: https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/
Changes since v1:
* Add check for HW version and rewrite the description.
* Link: https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/
drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 45 +++++++++++++++++------------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 12 ++++---- drivers/infiniband/hw/hns/hns_roce_main.c | 3 +- 3 files changed, 34 insertions(+), 26 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +26 -19
@@ -99,16 +99,16 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,u64 pbl_ba;/* use ib_access_flags */ -roce_set_bit (rc_sq_wqe->byte_4,V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S , - wr->access &IB_ACCESS_MW_BIND ?1 :0 ); -roce_set_bit (rc_sq_wqe->byte_4,V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S , - wr->access &IB_ACCESS_REMOTE_ATOMIC ?1 :0 ); -roce_set_bit (rc_sq_wqe->byte_4,V2_RC_FRMR_WQE_BYTE_4_RR_S , - wr->access &IB_ACCESS_REMOTE_READ ?1 :0 ); -roce_set_bit (rc_sq_wqe->byte_4,V2_RC_FRMR_WQE_BYTE_4_RW_S , - wr->access &IB_ACCESS_REMOTE_WRITE ?1 :0 ); -roce_set_bit (rc_sq_wqe->byte_4,V2_RC_FRMR_WQE_BYTE_4_LW_S , - wr->access &IB_ACCESS_LOCAL_WRITE ?1 :0 ); +roce_set_bit (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_BIND_EN_S , + !!(wr->access &IB_ACCESS_MW_BIND )); +roce_set_bit (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_ATOMIC_S , + !!(wr->access &IB_ACCESS_REMOTE_ATOMIC )); +roce_set_bit (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_RR_S , + !!(wr->access &IB_ACCESS_REMOTE_READ )); +roce_set_bit (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_RW_S , + !!(wr->access &IB_ACCESS_REMOTE_WRITE )); +roce_set_bit (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_LW_S , + !!(wr->access &IB_ACCESS_LOCAL_WRITE ));/* Data structure reuse may lead to confusion */ pbl_ba = mr->pbl_mtr.hem_cfg.root_ba;
@@ -121,12 +121,10 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, rc_sq_wqe->va =cpu_to_le64 (wr->mr->iova); fseg->pbl_size =cpu_to_le32 (mr->npages); -roce_set_field (fseg->mode_buf_pg_sz, -V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M , +roce_set_field (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M ,V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S ,to_hr_hw_page_shift (mr->pbl_mtr.hem_cfg.buf_pg_shift)); -roce_set_bit (fseg->mode_buf_pg_sz, -V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S ,0 ); +roce_set_bit (fseg->byte_40,V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S ,0 ); } static void set_atomic_seg(const struct ib_send_wr *wr,
@@ -522,10 +520,12 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,return 0 ; } -static int set_rc_opcode (struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, +static int set_rc_opcode (struct hns_roce_dev *hr_dev, +struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,const struct ib_send_wr *wr) {u32 ib_op = wr->opcode; +int ret =0 ; rc_sq_wqe->immtdata =get_immtdata (wr);
@@ -545,7 +545,10 @@ static int set_rc_opcode(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, rc_sq_wqe->va =cpu_to_le64 (atomic_wr (wr)->remote_addr);break ;case IB_WR_REG_MR : -set_frmr_seg (rc_sq_wqe,reg_wr (wr)); +if (hr_dev->pci_dev->revision >=PCI_REVISION_ID_HIP09 ) +set_frmr_seg (rc_sq_wqe,reg_wr (wr)); +else + ret = -EOPNOTSUPP ;break ;case IB_WR_LOCAL_INV :roce_set_bit (rc_sq_wqe->byte_4,V2_RC_SEND_WQE_BYTE_4_SO_S ,1 );
@@ -554,19 +557,23 @@ static int set_rc_opcode(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe, rc_sq_wqe->inv_key =cpu_to_le32 (wr->ex.invalidate_rkey);break ;default : -return -EINVAL ; + ret = -EINVAL ; } +if (unlikely (ret)) +return ret; +roce_set_field (rc_sq_wqe->byte_4,V2_RC_SEND_WQE_BYTE_4_OPCODE_M ,V2_RC_SEND_WQE_BYTE_4_OPCODE_S ,to_hr_opcode (ib_op)); -return 0 ; +return ret; }static inline int set_rc_wqe (struct hns_roce_qp *qp,const struct ib_send_wr *wr,void *wqe,unsigned int *sge_idx,unsigned int owner_bit) { +struct hns_roce_dev *hr_dev =to_hr_dev (qp->ibqp.device);struct hns_roce_v2_rc_send_wqe *rc_sq_wqe = wqe;unsigned int curr_idx = *sge_idx;unsigned int valid_num_sge;
@@ -577,7 +584,7 @@ static inline int set_rc_wqe(struct hns_roce_qp *qp, rc_sq_wqe->msg_len =cpu_to_le32 (msg_len); - ret =set_rc_opcode (rc_sq_wqe, wr); + ret =set_rc_opcode (hr_dev, rc_sq_wqe, wr);if (WARN_ON (ret))return ret;
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +6 -6
@@ -1255,15 +1255,15 @@ struct hns_roce_v2_rc_send_wqe {#define V2_RC_SEND_WQE_BYTE_4_INLINE_S 12 -#define V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S 19 +#define V2_RC_FRMR_WQE_BYTE_40_BIND_EN_S 10 -#define V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S 20 +#define V2_RC_FRMR_WQE_BYTE_40_ATOMIC_S 11 -#define V2_RC_FRMR_WQE_BYTE_4_RR_S 21 +#define V2_RC_FRMR_WQE_BYTE_40_RR_S 12 -#define V2_RC_FRMR_WQE_BYTE_4_RW_S 22 +#define V2_RC_FRMR_WQE_BYTE_40_RW_S 13 -#define V2_RC_FRMR_WQE_BYTE_4_LW_S 23 +#define V2_RC_FRMR_WQE_BYTE_40_LW_S 14#define V2_RC_SEND_WQE_BYTE_4_FLAG_S 31
@@ -1280,7 +1280,7 @@ struct hns_roce_v2_rc_send_wqe {struct hns_roce_wqe_frmr_seg {__le32 pbl_size; -__le32 mode_buf_pg_sz; +__le32 byte_40; };#define V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S 4
drivers/infiniband/hw/hns/hns_roce_main.c +2 -1
@@ -201,7 +201,8 @@ static int hns_roce_query_device(struct ib_device *ib_dev, props->max_srq_sge = hr_dev->caps.max_srq_sges; } -if (hr_dev->caps.flags &HNS_ROCE_CAP_FLAG_FRMR ) { +if (hr_dev->caps.flags &HNS_ROCE_CAP_FLAG_FRMR && + hr_dev->pci_dev->revision >=PCI_REVISION_ID_HIP09 ) { props->device_cap_flags |=IB_DEVICE_MEM_MGT_EXTENSIONS ; props->max_fast_reg_page_list_len =HNS_ROCE_FRMR_MAX_PA ; }
Signature · 1 line
2.8.1
Quoted 16 lines · Weihang Li
From: Yixing Liu <[email protected]>
FRMR is not well-supported on HIP08, it is re-designed for HIP09 and the position of related fields is changed. Then the ULPs should be forbidden to use FRMR on older hardwares.
Signed-off-by Yixing Liu <[email protected]>Signed-off-by Weihang Li <[email protected]>---
This patch is one of series "RDMA/hns: Updates for 5.12", the others have been applied.Changes since v2:
* Aovid settingIB_DEVICE_MEM_MGT_EXTENSIONSfor HIP08 or older hardwares.
* Link: https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/
Quoted 8 lines
Changes since v1:
* Add check for HW version and rewrite the description.
* Link: https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 45 +++++++++++++++++------------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 12 ++++---- drivers/infiniband/hw/hns/hns_roce_main.c | 3 +- 3 files changed, 34 insertions(+), 26 deletions(-)
Applied to for-next, thanks
Jason