RDMA/mana_ib: UC QP support
Subsystems (2)
v4 cover RDMA/mana_ib: UC QP support
From: Konstantin Taranov <[email protected]>
Implement UC QP creation in the RNIC HW for user API.
v4: renamed rc_qp destrot helper to rnic_qp. Use the latest robust udata API. Unified QP lookup table to simplify code for inserting UC QPs. That is why the changes were split into 2 patches. The first patch is about the changes related to qp lookup/store.
v3: use new udata helpers
v2: removed udata check and removed enum from mana-abi.h
Konstantin Taranov (2):
RDMA/mana_ib: unify QP lookup table
RDMA/mana_ib: UC QP support for UAPI
drivers/infiniband/hw/mana/main.c | 49 ++++++++- drivers/infiniband/hw/mana/mana_ib.h | 83 +++++++++++++-- drivers/infiniband/hw/mana/qp.c | 154 +++++++++++++++++++-------- include/uapi/rdma/mana-abi.h | 11 ++ 4 files changed, 243 insertions(+), 54 deletions(-)
Signature · 1 line
2.43.0
PATCH v4 1/2 RDMA/mana_ib: unify QP lookup table
From: Konstantin Taranov <[email protected]>
Add helpers to retrieve the send and receive queues of a QP. Use these helpers when storing queue IDs in the lookup table. MANA queue IDs are 2-bit aligned, allowing the two least significant bits to be omitted when storing and looking up queue IDs.
--- drivers/infiniband/hw/mana/mana_ib.h | 30 ++++++++++++ drivers/infiniband/hw/mana/qp.c | 70 ++++++++++++---------------- 2 files changed, 59 insertions(+), 41 deletions(-)
drivers/infiniband/hw/mana/mana_ib.h +30 -0
@@ -26,6 +26,8 @@/* Send queue ID mask */ #define MANA_SENDQ_MASK BIT(31) +/* Queue ID encodes type in the lower 2 bits */ +#define MANA_QID_SUBTYPE_MASK 0x3/* * The hardware limit of number of MRs is greater than maximum number of MRs
@@ -582,12 +584,40 @@ static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)return mdev->gdma_dev->gdma_context; } +static inline struct mana_ib_queue *mana_qp_get_sq (struct mana_ib_qp *qp) +{ +switch (qp->ibqp.qp_type) { +case IB_QPT_RC : +return &qp->rc_qp.queues[MANA_RC_SEND_QUEUE_REQUESTER ]; +case IB_QPT_UD : +case IB_QPT_GSI : +return &qp->ud_qp.queues[MANA_UD_SEND_QUEUE ]; +default : +return NULL ; + } +} + +static inline struct mana_ib_queue *mana_qp_get_rq (struct mana_ib_qp *qp) +{ +switch (qp->ibqp.qp_type) { +case IB_QPT_RC : +return &qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER ]; +case IB_QPT_UD : +case IB_QPT_GSI : +return &qp->ud_qp.queues[MANA_UD_RECV_QUEUE ]; +default : +return NULL ; + } +} +static inline struct mana_ib_qp *mana_get_qp_ref (struct mana_ib_dev *mdev,u32 qid,bool is_sq) {struct mana_ib_qp *qp;unsigned long flag; +/* Remove subtype bits */ + qid &= ~MANA_QID_SUBTYPE_MASK ;if (is_sq) qid |=MANA_SENDQ_MASK ;
drivers/infiniband/hw/mana/qp.c +29 -41
97 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v4 2/2 RDMA/mana_ib: UC QP support for UAPI
In reply to Konstantin TaranovFrom: Konstantin Taranov <[email protected]>
Implement UC QP creation in the RNIC HW for user API. An UC QP is exposed as three work queues: send, receive, and memory management. The latter is used for bind and invalidate WQEs to support memory windows.
--- drivers/infiniband/hw/mana/main.c | 49 ++++++++++++++-- drivers/infiniband/hw/mana/mana_ib.h | 53 ++++++++++++++++-- drivers/infiniband/hw/mana/qp.c | 84 +++++++++++++++++++++++++++- include/uapi/rdma/mana-abi.h | 11 ++++ 4 files changed, 184 insertions(+), 13 deletions(-)
drivers/infiniband/hw/mana/main.c +44 -5
@@ -1046,16 +1046,55 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,return 0 ; } -int mana_ib_gd_destroy_rc_qp (struct mana_ib_dev *mdev,struct mana_ib_qp *qp) +int mana_ib_gd_create_uc_qp (struct mana_ib_dev *mdev,struct mana_ib_qp *qp, +struct ib_qp_init_attr *attr,u32 doorbell,u64 flags) { -struct mana_rnic_destroy_rc_qp_resp resp = {0 }; -struct mana_rnic_destroy_rc_qp_req req = {0 }; + struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); + struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); + struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd);struct gdma_context *gc =mdev_to_gc (mdev); +struct mana_rnic_create_uc_qp_resp resp = {}; +struct mana_rnic_create_uc_qp_req req = {}; +int err, i; -mana_gd_init_req_hdr (&req.hdr,MANA_IB_DESTROY_RC_QP ,sizeof (req),sizeof (resp)); +mana_gd_init_req_hdr (&req.hdr,MANA_IB_CREATE_UC_QP ,sizeof (req),sizeof (resp)); req.hdr.dev_id = mdev->gdma_dev->dev_id; req.adapter = mdev->adapter_handle; - req.rc_qp_handle = qp->qp_handle; + req.pd_handle = pd->pd_handle; + req.send_cq_handle = send_cq->cq_handle; + req.recv_cq_handle = recv_cq->cq_handle; +for (i =0 ; i <MANA_UC_QUEUE_TYPE_MAX ; i++) + req.dma_region[i] = qp->uc_qp.queues[i].gdma_region; + req.doorbell_page = doorbell; + req.max_send_wr = attr->cap.max_send_wr; + req.max_recv_wr = attr->cap.max_recv_wr; + req.max_send_sge = attr->cap.max_send_sge; + req.max_recv_sge = attr->cap.max_recv_sge; + req.flags = flags; + + err =mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); +if (err) +return err; + + qp->qp_handle = resp.qp_handle; +for (i =0 ; i <MANA_UC_QUEUE_TYPE_MAX ; i++) { + qp->uc_qp.queues[i].id = resp.queue_ids[i]; +/* The GDMA regions are now owned by the RNIC QP handle */ + qp->uc_qp.queues[i].gdma_region =GDMA_INVALID_DMA_REGION ; + } +return 0 ; +} + +int mana_ib_gd_destroy_rnic_qp (struct mana_ib_dev *mdev,struct mana_ib_qp *qp) +{ +struct mana_rnic_destroy_rnic_qp_resp resp = {0 }; +struct mana_rnic_destroy_rnic_qp_req req = {0 }; +struct gdma_context *gc =mdev_to_gc (mdev); + +mana_gd_init_req_hdr (&req.hdr,MANA_IB_DESTROY_RNIC_QP ,sizeof (req),sizeof (resp)); + req.hdr.dev_id = mdev->gdma_dev->dev_id; + req.adapter = mdev->adapter_handle; + req.qp_handle = qp->qp_handle;return mana_gd_send_request (gc,sizeof (req), &req,sizeof (resp), &resp); }
drivers/infiniband/hw/mana/mana_ib.h +47 -6
103 lines not on this page · raw message
drivers/infiniband/hw/mana/qp.c +82 -2
126 lines not on this page · raw message
include/uapi/rdma/mana-abi.h +11 -0
@@ -57,6 +57,17 @@ struct mana_ib_create_rc_qp_resp {__u32 queue_id[4 ]; }; +struct mana_ib_create_uc_qp { +__aligned_u64 queue_buf[3 ]; +__u32 queue_size[3 ]; +__u32 comp_mask; +}; + +struct mana_ib_create_uc_qp_resp { +__u32 queue_id[3 ]; +__u32 reserved; +}; +struct mana_ib_create_wq {__aligned_u64 wq_buf_addr;__u32 wq_buf_size;
Signature · 1 line
2.43.0