RDMA/hns: Add caps flag for UD inline of userspace
Subsystems (2)
PATCH v2 1/1 RDMA/hns: Add caps flag for UD inline of userspace
HIP09 supports UD inline up to size of 1024 Bytes, the caps flag is got from firmware and passed back to userspace when creating QP.
---
Changes since v1:
- Avoid overwriting some fields in set_default_caps().
- Link: https://patchwork.kernel.org/project/linux-rdma/patch/[email protected]/
drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 18 +++++++++++------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 1 + drivers/infiniband/hw/hns/hns_roce_qp.c | 3 +++ include/uapi/rdma/hns-abi.h | 1 + 5 files changed, 17 insertions(+), 7 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
@@ -214,6 +214,7 @@ enum {HNS_ROCE_CAP_FLAG_FRMR =BIT (8 ),HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL =BIT (9 ),HNS_ROCE_CAP_FLAG_ATOMIC =BIT (10 ), +HNS_ROCE_CAP_FLAG_UD_SQ_INL =BIT (13 ),HNS_ROCE_CAP_FLAG_SDI_MODE =BIT (14 ), HNS_ROCE_CAP_FLAG_STASH = BIT(17), };
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +11 -7
@@ -1800,7 +1800,6 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->max_sq_sg =HNS_ROCE_V2_MAX_SQ_SGE_NUM ; caps->max_extend_sg =HNS_ROCE_V2_MAX_EXTEND_SGE_NUM ; caps->max_rq_sg =HNS_ROCE_V2_MAX_RQ_SGE_NUM ; - caps->max_sq_inline =HNS_ROCE_V2_MAX_SQ_INLINE ; caps->num_uars =HNS_ROCE_V2_UAR_NUM ; caps->phy_num_uars =HNS_ROCE_V2_PHY_UAR_NUM ; caps->num_aeq_vectors =HNS_ROCE_V2_AEQE_VEC_NUM ;
@@ -1817,7 +1816,6 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->max_sq_desc_sz =HNS_ROCE_V2_MAX_SQ_DESC_SZ ; caps->max_rq_desc_sz =HNS_ROCE_V2_MAX_RQ_DESC_SZ ; caps->max_srq_desc_sz =HNS_ROCE_V2_MAX_SRQ_DESC_SZ ; - caps->qpc_sz =HNS_ROCE_V2_QPC_SZ ; caps->irrl_entry_sz =HNS_ROCE_V2_IRRL_ENTRY_SZ ; caps->trrl_entry_sz =HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ ; caps->cqc_entry_sz =HNS_ROCE_V2_CQC_ENTRY_SZ ;
@@ -1825,7 +1823,6 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->mtpt_entry_sz =HNS_ROCE_V2_MTPT_ENTRY_SZ ; caps->mtt_entry_sz =HNS_ROCE_V2_MTT_ENTRY_SZ ; caps->idx_entry_sz =HNS_ROCE_V2_IDX_ENTRY_SZ ; - caps->cqe_sz =HNS_ROCE_V2_CQE_SIZE ; caps->page_size_cap =HNS_ROCE_V2_PAGE_SIZE_SUPPORTED ; caps->reserved_lkey =0 ; caps->reserved_pds =0 ;
@@ -1871,11 +1868,8 @@ static void set_default_caps(struct hns_roce_dev *hr_dev)HNS_ROCE_CAP_FLAG_SQ_RECORD_DB ; caps->pkey_table_len[0 ] =1 ; - caps->gid_table_len[0 ] =HNS_ROCE_V2_GID_INDEX_NUM ; caps->ceqe_depth =HNS_ROCE_V2_COMP_EQE_NUM ; caps->aeqe_depth =HNS_ROCE_V2_ASYNC_EQE_NUM ; - caps->aeqe_size =HNS_ROCE_AEQE_SIZE ; - caps->ceqe_size =HNS_ROCE_CEQE_SIZE ; caps->local_ca_ack_delay =0 ; caps->max_mtu =IB_MTU_4096 ;
@@ -1897,7 +1891,6 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->cqc_timer_buf_pg_sz =0 ; caps->cqc_timer_hop_num =HNS_ROCE_HOP_NUM_0 ; - caps->sccc_sz =HNS_ROCE_V2_SCCC_SZ ; caps->sccc_ba_pg_sz =0 ; caps->sccc_buf_pg_sz =0 ; caps->sccc_hop_num =HNS_ROCE_SCCC_HOP_NUM ;
@@ -1916,6 +1909,16 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->gmv_buf_pg_sz =0 ; caps->gid_table_len[0 ] = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz); + caps->flags |=HNS_ROCE_CAP_FLAG_UD_SQ_INL ; + caps->max_sq_inline =HNS_ROCE_V2_MAX_SQ_INL_EXT ; + }else { + caps->aeqe_size =HNS_ROCE_AEQE_SIZE ; + caps->ceqe_size =HNS_ROCE_CEQE_SIZE ; + caps->cqe_sz =HNS_ROCE_V2_CQE_SIZE ; + caps->qpc_sz =HNS_ROCE_V2_QPC_SZ ; + caps->sccc_sz =HNS_ROCE_V2_SCCC_SZ ; + caps->gid_table_len[0 ] =HNS_ROCE_V2_GID_INDEX_NUM ; + caps->max_sq_inline =HNS_ROCE_V2_MAX_SQ_INLINE ; } }
@@ -5084,6 +5087,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, qp_attr->cur_qp_state = qp_attr->qp_state; qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt; qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs; + qp_attr->cap.max_inline_data = hr_qp->max_inline_data;if (!ibqp->uobject) { qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +1 -0
@@ -61,6 +61,7 @@#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000#define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 +#define HNS_ROCE_V2_MAX_SQ_INL_EXT 0x400#define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32#define HNS_ROCE_V2_UAR_NUM 256#define HNS_ROCE_V2_PHY_UAR_NUM 1
drivers/infiniband/hw/hns/hns_roce_qp.c +3 -0
@@ -1020,6 +1020,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, }if (udata) { +if (hr_dev->caps.flags &HNS_ROCE_CAP_FLAG_UD_SQ_INL ) + resp.cap_flags |=HNS_ROCE_QP_CAP_UD_SQ_INL ; + ret =ib_copy_to_udata (udata, &resp,min (udata->outlen,sizeof (resp)));if (ret) {
include/uapi/rdma/hns-abi.h +1 -0
@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3, }; struct hns_roce_ib_create_qp_resp {
Signature · 1 line
2.8.1
Jason Gunthorpe
maintainerinclude/uapi/rdma/hns-abi.h enum hns_roce_qp_cap_flags { · line 81
Quoted 9 lines · Weihang Li
include/uapi/rdma/hns-abi.h +2
-0+++ b/include/uapi/rdma/hns-abi.h@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3, };
Where are the rdma-core patches to support this bit? I don't see them on github?
Jason
Quoted 14 lines · Jason Gunthorpe
include/uapi/rdma/hns-abi.h enum hns_roce_qp_cap_flags { · line 81
Quoted 9 lines · Weihang Li
include/uapi/rdma/hns-abi.h +2
-0+++ b/include/uapi/rdma/hns-abi.h@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3, };Where are the rdma-core patches to support this bit? I don't see them on github?
Quoted 1 line
Jason
I thought we needed to send the userspace part after the kernel part was merged. I sent the rdma-core patches just now:
https://github.com/linux-rdma/rdma-core/pull/934
Thank you
Weihang
Quoted 4 lines
_______________________________________________
Linuxarm mailing list -- [email protected]
To unsubscribe send an email to [email protected]
Quoted 15 lines · liweihang
Quoted 14 lines · Jason Gunthorpe
include/uapi/rdma/hns-abi.h enum hns_roce_qp_cap_flags { · line 81
Quoted 9 lines · Weihang Li
include/uapi/rdma/hns-abi.h +2
-0+++ b/include/uapi/rdma/hns-abi.h@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3, };Where are the rdma-core patches to support this bit? I don't see them on github?
Quoted 6 lines
Quoted 1 line
Jason
I thought we needed to send the userspace part after the kernel part was merged. I sent the rdma-core patches just now:
After kernel part will be accessed, you will need to update the patch
https://github.com/linux-rdma/rdma-core/pull/934/commits/2877713e1fed29305d04e39dd934ea81082e616e
with the correct SHA-1.
Thanks
Quoted 11 lines
Thank you
WeihangQuoted 4 lines
_______________________________________________
Linuxarm mailing list -- [email protected]
To unsubscribe send an email to [email protected]
Quoted 19 lines · Leon Romanovsky
Quoted 18 lines · liweihang
Quoted 13 lines · Jason Gunthorpe
include/uapi/rdma/hns-abi.h enum hns_roce_qp_cap_flags { · line 81
Quoted 9 lines · Weihang Li
include/uapi/rdma/hns-abi.h +2
-0+++ b/include/uapi/rdma/hns-abi.h@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3, };Where are the rdma-core patches to support this bit? I don't see them on github?
Quoted 1 line
Jason
I thought we needed to send the userspace part after the kernel part was merged. I sent the rdma-core patches just now:
Quoted 7 lines
Quoted 1 line
After kernel part will be accessed, you will need to update the patch
https://github.com/linux-rdma/rdma-core/pull/934/commits/2877713e1fed29305d04e39dd934ea81082e616e
with the correct SHA-1.Thanks
OK, I will update it then.
Thanks
Weihang
Jason Gunthorpe
maintainerinclude/uapi/rdma/hns-abi.h enum hns_roce_qp_cap_flags { · line 80
Quoted 8 lines · Weihang Li
include/uapi/rdma/hns-abi.h +2
-0+++ b/include/uapi/rdma/hns-abi.h@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3,
I don't understand why you need this flag.
The # of bytes of inline data should be returned from create_qp in the max_inline_data cap.
If things doesn't support inline data then shouldn't that just return 0?
Jason
Quoted 12 lines · Jason Gunthorpe
include/uapi/rdma/hns-abi.h enum hns_roce_qp_cap_flags { · line 80
Quoted 8 lines · Weihang Li
include/uapi/rdma/hns-abi.h +2
-0+++ b/include/uapi/rdma/hns-abi.h@@ -77,6 +77,7 @@ enum hns_roce_qp_cap_flags {HNS_ROCE_QP_CAP_RQ_RECORD_DB =1 <<0 ,HNS_ROCE_QP_CAP_SQ_RECORD_DB =1 <<1 ,HNS_ROCE_QP_CAP_OWNER_DB =1 <<2 , + HNS_ROCE_QP_CAP_UD_SQ_INL = 1 << 3,I don't understand why you need this flag.
Quoted 8 lines
The # of bytes of inline data should be returned from
create_qpin themax_inline_datacap.If things doesn't support inline data then shouldn't that just return 0?
Jason
Yes, you are right. Please ignore this patch and I will update the userspace part later.
Thanks
Weihang