Provider/rxe: Implement extended verbs APIs
cover Provider/rxe: Implement extended verbs APIs
Implement the following erxtended verbs APIs:
ibv_query_device_ex
ibv_create_cq_ex
ibv_create_qp_ex
Also implement the field parse and set ops in struct ibv_cq and ibv_qp.
Introduce a pair of SW capability bit masks that are exchanged between the user space provider and the kernel space driver during the ibv_alloc_context verb to allow the provider and driver to adjust shared data structures depending on which capabilities are supported. This is an extensible mechanism to avoid changes to ABI version.
This patch set depends on the following patch
0001-Provider-rxe-Cleanup-style-warnings.patch
Bob Pearson (4):
Provider/rxe: Exchange capabilities with driver Provider/rxe: Implement ibv_query_device_ex verb Providers/rxe: Implement ibv_create_cq_ex verb Providers/rxe: Implement ibv_create_qp_ex verb
kernel-headers/rdma/rdma_user_rxe.h | 49 ++ providers/rxe/rxe-abi.h | 8 +- providers/rxe/rxe.c | 976 +++++++++++++++++++++++++++- providers/rxe/rxe.h | 27 +- providers/rxe/rxe_queue.h | 80 ++- 5 files changed, 1092 insertions(+), 48 deletions(-)
Signature · 1 line
2.27.0
PATCH 1/4 Provider/rxe: Exchange capabilities with driver
Exchange capability masks between provider and driver during alloc_context verb.
--- kernel-headers/rdma/rdma_user_rxe.h | 18 ++++++++++++++++++ providers/rxe/rxe-abi.h | 2 ++ providers/rxe/rxe.c | 12 ++++++++---- providers/rxe/rxe.h | 5 +++++ 4 files changed, 33 insertions(+), 4 deletions(-)
kernel-headers/rdma/rdma_user_rxe.h +12 -0
@@ -152,6 +158,18 @@ struct rxe_recv_wqe {struct rxe_dma_info dma; }; +enum rxe_capabilities { +RXE_CAP_NONE =0 , +}; + +struct rxe_alloc_context_cmd { +__aligned_u64 provider_cap; +}; + +struct rxe_alloc_context_resp { +__aligned_u64 driver_cap; +}; +struct rxe_create_cq_resp {struct mminfo mi; };
providers/rxe/rxe-abi.h +2 -0
@@ -39,6 +39,8 @@#include <rdma/rdma_user_rxe.h> #include <kernel-abi/rdma_user_rxe.h> +DECLARE_DRV_CMD (urxe_alloc_context,IB_USER_VERBS_CMD_GET_CONTEXT , + rxe_alloc_context_cmd, rxe_alloc_context_resp);DECLARE_DRV_CMD (urxe_create_cq,IB_USER_VERBS_CMD_CREATE_CQ , empty, rxe_create_cq_resp); DECLARE_DRV_CMD(urxe_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
providers/rxe/rxe.c +8 -4
@@ -865,18 +865,22 @@ static struct verbs_context *rxe_alloc_context(struct ibv_device *ibdev, void *private_data) {struct rxe_context *context; -struct ibv_get_context cmd; -struct ib_uverbs_get_context_resp resp; +struct urxe_alloc_context cmd = {}; +struct urxe_alloc_context_resp resp = {}; context =verbs_init_and_alloc_context (ibdev, cmd_fd, context, ibv_ctx,RDMA_DRIVER_RXE );if (!context)return NULL ; -if (ibv_cmd_get_context (&context->ibv_ctx, &cmd,sizeof (cmd), - &resp,sizeof (resp))) + cmd.provider_cap =RXE_PROVIDER_CAP ; + +if (ibv_cmd_get_context (&context->ibv_ctx, &cmd.ibv_cmd,sizeof (cmd), + &resp.ibv_resp,sizeof (resp))) goto out; + context->capabilities = cmd.provider_cap & resp.driver_cap; +verbs_set_ops (&context->ibv_ctx, &rxe_ctx_ops);return &context->ibv_ctx;
providers/rxe/rxe.h +5 -0
@@ -48,6 +48,10 @@ enum rdma_network_type {RDMA_NETWORK_IPV6 }; +enum rxe_provider_cap { +RXE_PROVIDER_CAP =RXE_CAP_NONE , +}; +struct rxe_device {struct verbs_device ibv_dev;int abi_version;
@@ -55,6 +59,7 @@ struct rxe_device {struct rxe_context {struct verbs_context ibv_ctx; +uint64_t capabilities; };struct rxe_cq {
Signature · 1 line
2.27.0
PATCH 2/4 Provider/rxe: Implement ibv_query_device_ex verb
In reply to Bob PearsonImplement ibv_query_device_ex verb. Make it depend on a RXE_CAP_CMD_EX capability bit supported by both provider and driver.
--- kernel-headers/rdma/rdma_user_rxe.h | 1 + providers/rxe/rxe.c | 35 +++++++++++++++++++++++++++++ providers/rxe/rxe.h | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-)
kernel-headers/rdma/rdma_user_rxe.h +1 -0
@@ -160,6 +160,7 @@ struct rxe_recv_wqe {enum rxe_capabilities {RXE_CAP_NONE =0 , +RXE_CAP_CMD_EX =1ULL <<0 , };struct rxe_alloc_context_cmd {
providers/rxe/rxe.c +35 -0
@@ -87,6 +87,34 @@ static int rxe_query_device(struct ibv_context *context,return 0 ; } +static int rxe_query_device_ex (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr, +size_t attr_size) +{ +int ret; +uint64_t raw_fw_ver; +unsigned int major, minor, sub_minor; +struct ibv_query_device_ex cmd = {}; +struct ib_uverbs_ex_query_device_resp resp = {}; + +fprintf (stderr,"%s: called\n" , __func__); + ret =ibv_cmd_query_device_ex (context, input, attr,sizeof (*attr), + &raw_fw_ver, &cmd,sizeof (cmd), + &resp,sizeof (resp)); +if (ret) +return ret; + + major = (raw_fw_ver >>32 ) &0xffff ; + minor = (raw_fw_ver >>16 ) &0xffff ; + sub_minor = raw_fw_ver &0xffff ; + +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver), +"%d.%d.%d" , major, minor, sub_minor); + +return 0 ; +} +static int rxe_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr) {
@@ -860,6 +888,10 @@ static const struct verbs_context_ops rxe_ctx_ops = { .free_context = rxe_free_context, }; +static const struct verbs_context_ops rxe_ctx_ops_cmd_ex = { + .query_device_ex = rxe_query_device_ex, +}; +static struct verbs_context *rxe_alloc_context (struct ibv_device *ibdev,int cmd_fd,void *private_data)
@@ -883,6 +915,9 @@ static struct verbs_context *rxe_alloc_context(struct ibv_device *ibdev,verbs_set_ops (&context->ibv_ctx, &rxe_ctx_ops); +if (context->capabilities &RXE_CAP_CMD_EX ) +verbs_set_ops (&context->ibv_ctx, &rxe_ctx_ops_cmd_ex); +return &context->ibv_ctx; out:
providers/rxe/rxe.h +1 -1
@@ -49,7 +49,7 @@ enum rdma_network_type { };enum rxe_provider_cap { -RXE_PROVIDER_CAP =RXE_CAP_NONE , +RXE_PROVIDER_CAP =RXE_CAP_CMD_EX , };struct rxe_device {
Signature · 1 line
2.27.0
Jason Gunthorpe
maintainerRe: [PATCH 2/4] Provider/rxe: Implement ibv_query_device_ex verb
Quoted 13 lines · Bob Pearson
Implement
ibv_query_device_exverb. Make it depend on aRXE_CAP_CMD_EXcapability bit supported by both provider and driver.Signed-off-by Bob Pearson <[email protected]>kernel-headers/rdma/rdma_user_rxe.h | 1 + providers/rxe/rxe.c | 35 +++++++++++++++++++++++++++++ providers/rxe/rxe.h | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-)kernel-headers/rdma/rdma_user_rxe.h +1
-0+++ b/kernel-headers/rdma/rdma_user_rxe.h
kernel-headers/rdma/rdma_user_rxe.h struct rxe_recv_wqe { · line 164
Quoted 5 lines
kernel-headers/rdma/rdma_user_rxe.h +1
-0@@ -160,6 +160,7 @@ struct rxe_recv_wqe {enum rxe_capabilities {RXE_CAP_NONE =0 , +RXE_CAP_CMD_EX =1ULL <<0 , };
All the kernel-headers/ changes need to be in one patch at the start, the kernel-headers/update script will make the required commit.
Keeping this 100% in sync with the kernel is important
Quoted 10 lines
kernel-headers/rdma/rdma_user_rxe.h +0
-0struct rxe_alloc_context_cmd {providers/rxe/rxe.c +3
-0+++ b/providers/rxe/rxe.c@@ -87,6 +87,34 @@ static int rxe_query_device(struct ibv_context *context,return 0 ; } +static int rxe_query_device_ex(struct ibv_context *context, + const struct ibv_query_device_ex_input *input,
providers/rxe/rxe.c static int rxe_query_device(struct ibv_context *context, · line 101
Quoted 10 lines
providers/rxe/rxe.c +10
-0+struct ibv_device_attr_ex *attr, + size_t attr_size) +{ + int ret; +uint64_t raw_fw_ver; +unsigned int major, minor, sub_minor; +struct ibv_query_device_ex cmd = {}; +struct ib_uverbs_ex_query_device_resp resp = {}; + +fprintf (stderr,"%s: called\n" , __func__);
Don't send debugging prints in patches
Quoted 31 lines
providers/rxe/rxe.c +20
-0+ ret =ibv_cmd_query_device_ex (context, input, attr,sizeof (*attr), + &raw_fw_ver, &cmd,sizeof (cmd), + &resp,sizeof (resp)); +if (ret) +return ret; + + major = (raw_fw_ver >>32 ) &0xffff ; + minor = (raw_fw_ver >>16 ) &0xffff ; + sub_minor = raw_fw_ver &0xffff ; + +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver), +"%d.%d.%d" , major, minor, sub_minor); + +return 0 ; +} +static int rxe_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr) {@@ -860,6 +888,10 @@ static const struct verbs_context_ops rxe_ctx_ops = { .free_context = rxe_free_context, }; +static const struct verbs_context_ops rxe_ctx_ops_cmd_ex = { + .query_device_ex = rxe_query_device_ex, +}; +static struct verbs_context *rxe_alloc_context (struct ibv_device *ibdev,int cmd_fd,void *private_data)
providers/rxe/rxe.c static struct verbs_context *rxe_alloc_context(struct ibv_device *ibdev, · line 919
Quoted 5 lines
providers/rxe/rxe.c +2
-0@@ -883,6 +915,9 @@ static struct verbs_context *rxe_alloc_context(struct ibv_device *ibdev,verbs_set_ops (&context->ibv_ctx, &rxe_ctx_ops); +if (context->capabilities &RXE_CAP_CMD_EX ) +verbs_set_ops (&context->ibv_ctx, &rxe_ctx_ops_cmd_ex);
This isn't needed, we know if ibv_cmd_query_device_ex() is not supported because the kernel returns -EOPNOTSUP when we call it.
What is needed is to just call the fallback like dummy ops does:
if (ret == -EOPNOTSUPP) {
if (input && input->comp_mask)
return EINVAL;
if (attr_size < sizeof(attr->orig_attr))
return EOPNOTSUPP;
memset(attr, 0, attr_size);
return ibv_query_device(context, &attr->orig_attr);
}And I wonder if we should make ibv_cmd_query_device_ex() just do this? This whole thing is a mess now that the kernel always supports ibv_cmd_query_device_ex() on all drivers.
I don't have time to fix it all properly, so I suggest you just use the above fragment for now instead of the RXE_CAP_CMD_EX
Jason
PATCH 3/4 Providers/rxe: Implement ibv_create_cq_ex verb
In reply to Bob PearsonTogether with the matching commit for the rxe driver implement the ibv_create_cq_ex verb.
Also implement the operations in ibv_cq_ex struct.
--- kernel-headers/rdma/rdma_user_rxe.h | 30 ++++ providers/rxe/rxe-abi.h | 4 +- providers/rxe/rxe.c | 267 +++++++++++++++++++++++++++- providers/rxe/rxe.h | 12 +- providers/rxe/rxe_queue.h | 59 +++++- 5 files changed, 357 insertions(+), 15 deletions(-)
kernel-headers/rdma/rdma_user_rxe.h +30 -0
@@ -158,6 +158,32 @@ struct rxe_recv_wqe {struct rxe_dma_info dma; }; +struct rxe_uverbs_wc { +/* keep these the same as ib_uverbs_wc */ +__aligned_u64 wr_id; +__u32 status; +__u32 opcode; +__u32 vendor_err; +__u32 byte_len; +union { +__be32 imm_data; +__u32 invalidate_rkey; + } ex; +__u32 qp_num; +__u32 src_qp; +__u32 wc_flags; +__u16 pkey_index; +__u16 slid; +__u8 sl; +__u8 dlid_path_bits; +__u8 port_num; +__u8 reserved; + +/* any extras go here */ +__aligned_u64 timestamp; +__aligned_u64 realtime; +}; +enum rxe_capabilities {RXE_CAP_NONE =0 ,RXE_CAP_CMD_EX =1ULL <<0 ,
@@ -171,6 +197,10 @@ struct rxe_alloc_context_resp {__aligned_u64 driver_cap; }; +struct rxe_create_cq_cmd { +__aligned_u64 is_ex; +}; +struct rxe_create_cq_resp {struct mminfo mi; };
providers/rxe/rxe-abi.h +3 -1
@@ -42,7 +42,9 @@DECLARE_DRV_CMD (urxe_alloc_context,IB_USER_VERBS_CMD_GET_CONTEXT , rxe_alloc_context_cmd, rxe_alloc_context_resp);DECLARE_DRV_CMD (urxe_create_cq,IB_USER_VERBS_CMD_CREATE_CQ , - empty, rxe_create_cq_resp); + rxe_create_cq_cmd, rxe_create_cq_resp); +DECLARE_DRV_CMD (urxe_create_cq_ex,IB_USER_VERBS_EX_CMD_CREATE_CQ , + rxe_create_cq_cmd, rxe_create_cq_resp);DECLARE_DRV_CMD (urxe_create_qp,IB_USER_VERBS_CMD_CREATE_QP , empty, rxe_create_qp_resp); DECLARE_DRV_CMD(urxe_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
providers/rxe/rxe.c +262 -4
303 lines not on this page · raw message
providers/rxe/rxe.h +9 -3
@@ -62,11 +62,17 @@ struct rxe_context {uint64_t capabilities; }; +/* common between cq and cq_ex */ struct rxe_cq { -struct ibv_cq ibv_cq; +struct verbs_cq vcq;struct mminfo mmap_info; -struct rxe_queue *queue; +struct rxe_queue *queue;pthread_spinlock_t lock; + +/* new API support */ +struct rxe_uverbs_wc *wc; +size_t wc_size; +uint32_t cur_index; };struct rxe_ah {
@@ -113,7 +119,7 @@ static inline struct rxe_device *to_rdev(struct ibv_device *ibdev)static inline struct rxe_cq *to_rcq (struct ibv_cq *ibcq) { -return to_rxxx (cq, cq); + return container_of(ibcq, struct rxe_cq, vcq.cq); }static inline struct rxe_qp *to_rqp (struct ibv_qp *ibqp)
providers/rxe/rxe_queue.h +53 -6
117 lines not on this page · raw message
Signature · 1 line
2.27.0
Jason Gunthorpe
maintainerRe: [PATCH 3/4] Providers/rxe: Implement ibv_create_cq_ex verb
kernel-headers/rdma/rdma_user_rxe.h struct rxe_alloc_context_resp { · line 202
Quoted 7 lines · Bob Pearson
kernel-headers/rdma/rdma_user_rxe.h +3
-0@@ -171,6 +197,10 @@ struct rxe_alloc_context_resp {__aligned_u64 driver_cap; }; +struct rxe_create_cq_cmd { +__aligned_u64 is_ex; +};
would be cleaer to call this 'wc_format' or something more specific;
u8 wc_format
u8 reserved[7]
Is fine
Quoted 5 lines
providers/rxe/rxe.c +0
-1@@ -210,15 +352,129 @@ static struct ibv_cq *rxe_create_cq(struct ibv_context *context, int cqe, cq->queue =mmap (NULL , resp.mi.size,PROT_READ |PROT_WRITE ,MAP_SHARED , context->cmd_fd, resp.mi.offset);if ((void *)cq->queue ==MAP_FAILED ) { -ibv_cmd_destroy_cq (&cq->ibv_cq);
providers/rxe/rxe.c static struct ibv_cq *rxe_create_cq(struct ibv_context *context, int cqe, · line 364
Quoted 10 lines
providers/rxe/rxe.c +10
-0+ibv_cmd_destroy_cq (&cq->vcq.cq); +free (cq); +return NULL ; + } + + cq->wc_size =1ULL << cq->queue->log2_elem_size; + +if (cq->wc_size <sizeof (struct ib_uverbs_wc )) { +fprintf (stderr,"cq wc size too small %ld need %ld\n" , + cq->wc_size,sizeof (struct ib_uverbs_wc ));
No prints like this in libraries
Seems reasonable other wise
Jason
PATCH 4/4 Providers/rxe: Implement ibv_create_qp_ex verb
In reply to Bob PearsonAdd ibv_create_qp_ex verb.
Add WQ operations in verbs_qp struct.
--- providers/rxe/rxe-abi.h | 2 + providers/rxe/rxe.c | 664 ++++++++++++++++++++++++++++++++++++-- providers/rxe/rxe.h | 10 +- providers/rxe/rxe_queue.h | 21 ++ 4 files changed, 667 insertions(+), 30 deletions(-)
providers/rxe/rxe-abi.h +2 -0
@@ -47,6 +47,8 @@ DECLARE_DRV_CMD(urxe_create_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, rxe_create_cq_cmd, rxe_create_cq_resp);DECLARE_DRV_CMD (urxe_create_qp,IB_USER_VERBS_CMD_CREATE_QP , empty, rxe_create_qp_resp); +DECLARE_DRV_CMD (urxe_create_qp_ex,IB_USER_VERBS_EX_CMD_CREATE_QP , + empty, rxe_create_qp_resp);DECLARE_DRV_CMD (urxe_create_srq,IB_USER_VERBS_CMD_CREATE_SRQ , empty, rxe_create_srq_resp); DECLARE_DRV_CMD(urxe_modify_srq, IB_USER_VERBS_CMD_MODIFY_SRQ,
providers/rxe/rxe.c +637 -27
724 lines not on this page · raw message
providers/rxe/rxe.h +7 -3
@@ -88,15 +88,19 @@ struct rxe_wq { };struct rxe_qp { -struct ibv_qp ibv_qp; +struct verbs_qp vqp;struct mminfo rq_mmap_info;struct rxe_wq rq;struct mminfo sq_mmap_info;struct rxe_wq sq;unsigned int ssn; + +/* new API support */ +uint32_t cur_index; +int err; }; -#define qp_type (qp) ((qp)->ibv_qp.qp_type) +#define qp_type (qp) ((qp)->vqp.qp.qp_type)struct rxe_srq {struct ibv_srq ibv_srq;
@@ -124,7 +128,7 @@ static inline struct rxe_cq *to_rcq(struct ibv_cq *ibcq)static inline struct rxe_qp *to_rqp (struct ibv_qp *ibqp) { -return to_rxxx (qp, qp); + return container_of(ibqp, struct rxe_qp, vqp.qp); }static inline struct rxe_srq *to_rsrq (struct ibv_srq *ibsrq)
providers/rxe/rxe_queue.h +21 -0
@@ -172,4 +172,25 @@ static inline int check_cq_queue_empty(struct rxe_cq *cq) return (cq->cur_index == producer_index); } +static inline void advance_qp_cur_index (struct rxe_qp *qp) +{ +struct rxe_queue *q = qp->sq.queue; + + qp->cur_index = (qp->cur_index +1 ) & q->index_mask; +} + +static inline int check_qp_queue_full (struct rxe_qp *qp) +{ +struct rxe_queue *q = qp->sq.queue; +uint32_t consumer_index =atomic_load (&q->consumer_index); + +if (qp->err) + goto err; + +if ((qp->cur_index +1 - consumer_index) % q->index_mask ==0 ) + qp->err =ENOSPC ; +err: +return qp->err; +} +#endif /* H_RXE_PCQ */
Signature · 1 line
2.27.0