Simplify query_device() in libibverbs
Jason Gunthorpe
maintainercover Simplify query_device() in libibverbs
Now that that kernel implements query_device_ex for every driver we should do the same in userspace. Have providers implement only query_device_ex() and feed query_device() down the same path. This eliminates some duplicated code in the providers and allows every provider to fully support query_device_ex with any new kernel features added.
This is a PR:
https://github.com/linux-rdma/rdma-core/pull/890
Jason Gunthorpe (9):
verbs: Simplify query_device_ex
verbs: Add ibv_cmd_query_device_any()
mlx5: Move context initialization out of mlx5_query_device_ex()
efa: Move the context intialization out of efa_query_device_ex()
mlx4: Move the context intialization out of mlx4_query_device_ex()
providers: Remove normal query_device() from providers that have _ex
providers: Convert all providers to implement query_device_ex
verbs: Remove dead code
verbs: Delete query_device() internal support
libibverbs/cmd.c | 212 ----------------------------- libibverbs/cmd_device.c | 155 +++++++++++++++++++++ libibverbs/device.c | 1 + libibverbs/driver.h | 19 +-- libibverbs/dummy_ops.c | 21 +-- libibverbs/libibverbs.map.in | 2 +- libibverbs/verbs.c | 5 +- libibverbs/verbs.h | 3 +- providers/bnxt_re/main.c | 2 +- providers/bnxt_re/verbs.c | 30 ++-- providers/bnxt_re/verbs.h | 5 +- providers/cxgb4/dev.c | 10 +- providers/cxgb4/libcxgb4.h | 3 +- providers/cxgb4/verbs.c | 14 +- providers/efa/efa.c | 15 +- providers/efa/verbs.c | 82 ++++++----- providers/efa/verbs.h | 2 +- providers/hfi1verbs/hfiverbs.c | 2 +- providers/hfi1verbs/hfiverbs.h | 5 +- providers/hfi1verbs/verbs.c | 13 +- providers/hns/hns_roce_u.c | 8 +- providers/hns/hns_roce_u.h | 3 +- providers/hns/hns_roce_u_verbs.c | 15 +- providers/i40iw/i40iw_umain.c | 2 +- providers/i40iw/i40iw_umain.h | 4 +- providers/i40iw/i40iw_uverbs.c | 18 ++- providers/ipathverbs/ipathverbs.c | 2 +- providers/ipathverbs/ipathverbs.h | 5 +- providers/ipathverbs/verbs.c | 13 +- providers/mlx4/mlx4.c | 34 +---- providers/mlx4/mlx4.h | 3 +- providers/mlx4/verbs.c | 86 +++++++----- providers/mlx5/mlx5.c | 11 +- providers/mlx5/mlx5.h | 3 +- providers/mlx5/verbs.c | 129 +++++++++--------- providers/mthca/mthca.c | 2 +- providers/mthca/mthca.h | 3 +- providers/mthca/verbs.c | 13 +- providers/ocrdma/ocrdma_main.c | 2 +- providers/ocrdma/ocrdma_main.h | 4 +- providers/ocrdma/ocrdma_verbs.c | 20 +-- providers/qedr/qelr_main.c | 2 +- providers/qedr/qelr_verbs.c | 21 +-- providers/qedr/qelr_verbs.h | 3 +- providers/rxe/rxe.c | 15 +- providers/siw/siw.c | 20 +-- providers/vmw_pvrdma/pvrdma.h | 3 +- providers/vmw_pvrdma/pvrdma_main.c | 2 +- providers/vmw_pvrdma/verbs.c | 13 +- util/util.h | 3 + 50 files changed, 511 insertions(+), 552 deletions(-)
Signature · 1 line
2.29.2
Jason Gunthorpe
maintainerPATCH 1/9 verbs: Simplify query_device_ex
The obtuse logic here is hard to read, simplify it with a small macro and add offsetofend()
--- libibverbs/cmd.c | 146 ++++++++++++++++------------------------------- util/util.h | 3 + 2 files changed, 52 insertions(+), 97 deletions(-)
libibverbs/cmd.c +49 -97
@@ -44,6 +44,7 @@#include <infiniband/cmd_write.h> #include "ibverbs.h" #include <ccan/minmax.h> +#include <util/util.h> bool verbs_allow_disassociate_destroy;
@@ -144,117 +145,68 @@ int ibv_cmd_query_device_ex(struct ibv_context *context,/* Report back supported comp_mask bits. For now no comp_mask bit is * defined */ attr->comp_mask = resp->comp_mask &0 ; -if (attr_size >= offsetof(struct ibv_device_attr_ex , odp_caps) + -sizeof (attr->odp_caps)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , odp_caps) + -sizeof (resp->odp_caps)) { - attr->odp_caps.general_caps = resp->odp_caps.general_caps; - attr->odp_caps.per_transport_caps.rc_odp_caps = - resp->odp_caps.per_transport_caps.rc_odp_caps; - attr->odp_caps.per_transport_caps.uc_odp_caps = - resp->odp_caps.per_transport_caps.uc_odp_caps; - attr->odp_caps.per_transport_caps.ud_odp_caps = - resp->odp_caps.per_transport_caps.ud_odp_caps; - } - } -if (attr_size >= offsetof(struct ibv_device_attr_ex , - completion_timestamp_mask) + -sizeof (attr->completion_timestamp_mask)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , timestamp_mask) + -sizeof (resp->timestamp_mask)) - attr->completion_timestamp_mask = resp->timestamp_mask; +#define CAN_COPY (_ibv_attr, _uverbs_attr) \ + (attr_size >= offsetofend(struct ibv_device_attr_ex, _ibv_attr) && \ + resp->response_length >= \ + offsetofend(struct ib_uverbs_ex_query_device_resp, \ + _uverbs_attr)) + +if (CAN_COPY (odp_caps, odp_caps)) { + attr->odp_caps.general_caps = resp->odp_caps.general_caps; + attr->odp_caps.per_transport_caps.rc_odp_caps = + resp->odp_caps.per_transport_caps.rc_odp_caps; + attr->odp_caps.per_transport_caps.uc_odp_caps = + resp->odp_caps.per_transport_caps.uc_odp_caps; + attr->odp_caps.per_transport_caps.ud_odp_caps = + resp->odp_caps.per_transport_caps.ud_odp_caps; } -if (attr_size >= offsetof(struct ibv_device_attr_ex , hca_core_clock) + -sizeof (attr->hca_core_clock)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , hca_core_clock) + -sizeof (resp->hca_core_clock)) - attr->hca_core_clock = resp->hca_core_clock; - } +if (CAN_COPY (completion_timestamp_mask, timestamp_mask)) + attr->completion_timestamp_mask = resp->timestamp_mask; -if (attr_size >= offsetof(struct ibv_device_attr_ex , device_cap_flags_ex) + -sizeof (attr->device_cap_flags_ex)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , device_cap_flags_ex) + -sizeof (resp->device_cap_flags_ex)) - attr->device_cap_flags_ex = resp->device_cap_flags_ex; - } +if (CAN_COPY (hca_core_clock, hca_core_clock)) + attr->hca_core_clock = resp->hca_core_clock; -if (attr_size >= offsetof(struct ibv_device_attr_ex , rss_caps) + -sizeof (attr->rss_caps)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , rss_caps) + -sizeof (resp->rss_caps)) { - attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; - attr->rss_caps.max_rwq_indirection_tables = resp->rss_caps.max_rwq_indirection_tables; - attr->rss_caps.max_rwq_indirection_table_size = resp->rss_caps.max_rwq_indirection_table_size; - } - } +if (CAN_COPY (device_cap_flags_ex, device_cap_flags_ex)) + attr->device_cap_flags_ex = resp->device_cap_flags_ex; -if (attr_size >= offsetof(struct ibv_device_attr_ex , max_wq_type_rq) + -sizeof (attr->max_wq_type_rq)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , max_wq_type_rq) + -sizeof (resp->max_wq_type_rq)) - attr->max_wq_type_rq = resp->max_wq_type_rq; +if (CAN_COPY (rss_caps, rss_caps)) { + attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; + attr->rss_caps.max_rwq_indirection_tables = + resp->rss_caps.max_rwq_indirection_tables; + attr->rss_caps.max_rwq_indirection_table_size = + resp->rss_caps.max_rwq_indirection_table_size; } -if (attr_size >= offsetof(struct ibv_device_attr_ex , raw_packet_caps) + -sizeof (attr->raw_packet_caps)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , raw_packet_caps) + -sizeof (resp->raw_packet_caps)) - attr->raw_packet_caps = resp->raw_packet_caps; - } +if (CAN_COPY (max_wq_type_rq, max_wq_type_rq)) + attr->max_wq_type_rq = resp->max_wq_type_rq; -if (attr_size >= offsetof(struct ibv_device_attr_ex , tm_caps) + -sizeof (attr->tm_caps)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , tm_caps) + -sizeof (resp->tm_caps)) { - attr->tm_caps.max_rndv_hdr_size = - resp->tm_caps.max_rndv_hdr_size; - attr->tm_caps.max_num_tags = - resp->tm_caps.max_num_tags; - attr->tm_caps.flags = resp->tm_caps.flags; - attr->tm_caps.max_ops = - resp->tm_caps.max_ops; - attr->tm_caps.max_sge = - resp->tm_caps.max_sge; - } - } +if (CAN_COPY (raw_packet_caps, raw_packet_caps)) + attr->raw_packet_caps = resp->raw_packet_caps; -if (attr_size >= offsetof(struct ibv_device_attr_ex , cq_mod_caps) + -sizeof (attr->cq_mod_caps)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , cq_moderation_caps) + -sizeof (resp->cq_moderation_caps)) { - attr->cq_mod_caps.max_cq_count = resp->cq_moderation_caps.max_cq_moderation_count; - attr->cq_mod_caps.max_cq_period = resp->cq_moderation_caps.max_cq_moderation_period; - } +if (CAN_COPY (tm_caps, tm_caps)) { + attr->tm_caps.max_rndv_hdr_size = + resp->tm_caps.max_rndv_hdr_size; + attr->tm_caps.max_num_tags = resp->tm_caps.max_num_tags; + attr->tm_caps.flags = resp->tm_caps.flags; + attr->tm_caps.max_ops = resp->tm_caps.max_ops; + attr->tm_caps.max_sge = resp->tm_caps.max_sge; } -if (attr_size >= offsetof(struct ibv_device_attr_ex , max_dm_size) + -sizeof (attr->max_dm_size)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , max_dm_size) + -sizeof (resp->max_dm_size)) { - attr->max_dm_size = resp->max_dm_size; - } +if (CAN_COPY (cq_mod_caps, cq_moderation_caps)) { + attr->cq_mod_caps.max_cq_count = + resp->cq_moderation_caps.max_cq_moderation_count; + attr->cq_mod_caps.max_cq_period = + resp->cq_moderation_caps.max_cq_moderation_period; } -if (attr_size >= offsetof(struct ibv_device_attr_ex , xrc_odp_caps) + -sizeof (attr->xrc_odp_caps)) { -if (resp->response_length >= - offsetof(struct ib_uverbs_ex_query_device_resp , xrc_odp_caps) + -sizeof (resp->xrc_odp_caps)) { - attr->xrc_odp_caps = resp->xrc_odp_caps; - } - } +if (CAN_COPY (max_dm_size, max_dm_size)) + attr->max_dm_size = resp->max_dm_size; + +if (CAN_COPY (xrc_odp_caps, xrc_odp_caps)) + attr->xrc_odp_caps = resp->xrc_odp_caps; +#undef CAN_COPYreturn 0 ; }
util/util.h +3 -0
@@ -23,6 +23,9 @@ static inline bool __good_snprintf(size_t len, int rc) ((a)->tv_nsec CMP (b)->tv_nsec) : \ ((a)->tv_sec CMP (b)->tv_sec)) +#define offsetofend (_type, _member) \ + (offsetof(_type, _member) + sizeof(((_type *)0)->_member)) +static inline unsigned long align (unsigned long val,unsigned long align) {return (val + align -1 ) & ~(align -1 );
Signature · 1 line
2.29.2
Jason Gunthorpe
maintainerPATCH 2/9 verbs: Add ibv_cmd_query_device_any()
In reply to Jason GunthorpeThis implements all the query_device command flows under a single call.
--- libibverbs/cmd_device.c | 155 +++++++++++++++++++++++++++++++++++ libibverbs/driver.h | 5 ++ libibverbs/libibverbs.map.in | 1 + 3 files changed, 161 insertions(+)
libibverbs/cmd_device.c +155 -0
@@ -35,6 +35,7 @@#include <stdlib.h> #include <dirent.h> #include <infiniband/cmd_write.h> +#include <util/util.h> #include <net/if.h>
@@ -516,3 +517,157 @@ ssize_t _ibv_query_gid_table(struct ibv_context *context,return num_entries; } + +int ibv_cmd_query_device_any (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size, +struct ib_uverbs_ex_query_device_resp *resp, +size_t *resp_size) +{ +struct ib_uverbs_ex_query_device_resp internal_resp; +size_t internal_resp_size; +int err; + +if (input && input->comp_mask) +return EINVAL ; +if (attr_size <sizeof (attr->orig_attr)) +return EINVAL ; + +if (!resp) { + resp = &internal_resp; + internal_resp_size =sizeof (internal_resp); + resp_size = &internal_resp_size; + } +memset (attr,0 , attr_size); +memset (resp,0 , *resp_size); + +if (attr_size >sizeof (attr->orig_attr)) { +struct ibv_query_device_ex cmd = {}; + + err =execute_cmd_write_ex (context, +IB_USER_VERBS_EX_CMD_QUERY_DEVICE , + &cmd,sizeof (cmd), resp, *resp_size); +if (err) { +if (err !=EOPNOTSUPP ) +return err; + attr_size =sizeof (attr->orig_attr); + } + } + +if (attr_size ==sizeof (attr->orig_attr)) { +struct ibv_query_device cmd = {}; + + err =execute_cmd_write (context,IB_USER_VERBS_CMD_QUERY_DEVICE , + &cmd,sizeof (cmd), &resp->base, +sizeof (resp->base)); +if (err) +return err; + resp->response_length =sizeof (resp->base); + } + + *resp_size = resp->response_length; + attr->orig_attr.node_guid = resp->base.node_guid; + attr->orig_attr.sys_image_guid = resp->base.sys_image_guid; + attr->orig_attr.max_mr_size = resp->base.max_mr_size; + attr->orig_attr.page_size_cap = resp->base.page_size_cap; + attr->orig_attr.vendor_id = resp->base.vendor_id; + attr->orig_attr.vendor_part_id = resp->base.vendor_part_id; + attr->orig_attr.hw_ver = resp->base.hw_ver; + attr->orig_attr.max_qp = resp->base.max_qp; + attr->orig_attr.max_qp_wr = resp->base.max_qp_wr; + attr->orig_attr.device_cap_flags = resp->base.device_cap_flags; + attr->orig_attr.max_sge = resp->base.max_sge; + attr->orig_attr.max_sge_rd = resp->base.max_sge_rd; + attr->orig_attr.max_cq = resp->base.max_cq; + attr->orig_attr.max_cqe = resp->base.max_cqe; + attr->orig_attr.max_mr = resp->base.max_mr; + attr->orig_attr.max_pd = resp->base.max_pd; + attr->orig_attr.max_qp_rd_atom = resp->base.max_qp_rd_atom; + attr->orig_attr.max_ee_rd_atom = resp->base.max_ee_rd_atom; + attr->orig_attr.max_res_rd_atom = resp->base.max_res_rd_atom; + attr->orig_attr.max_qp_init_rd_atom = resp->base.max_qp_init_rd_atom; + attr->orig_attr.max_ee_init_rd_atom = resp->base.max_ee_init_rd_atom; + attr->orig_attr.atomic_cap = resp->base.atomic_cap; + attr->orig_attr.max_ee = resp->base.max_ee; + attr->orig_attr.max_rdd = resp->base.max_rdd; + attr->orig_attr.max_mw = resp->base.max_mw; + attr->orig_attr.max_raw_ipv6_qp = resp->base.max_raw_ipv6_qp; + attr->orig_attr.max_raw_ethy_qp = resp->base.max_raw_ethy_qp; + attr->orig_attr.max_mcast_grp = resp->base.max_mcast_grp; + attr->orig_attr.max_mcast_qp_attach = resp->base.max_mcast_qp_attach; + attr->orig_attr.max_total_mcast_qp_attach = + resp->base.max_total_mcast_qp_attach; + attr->orig_attr.max_ah = resp->base.max_ah; + attr->orig_attr.max_fmr = resp->base.max_fmr; + attr->orig_attr.max_map_per_fmr = resp->base.max_map_per_fmr; + attr->orig_attr.max_srq = resp->base.max_srq; + attr->orig_attr.max_srq_wr = resp->base.max_srq_wr; + attr->orig_attr.max_srq_sge = resp->base.max_srq_sge; + attr->orig_attr.max_pkeys = resp->base.max_pkeys; + attr->orig_attr.local_ca_ack_delay = resp->base.local_ca_ack_delay; + attr->orig_attr.phys_port_cnt = resp->base.phys_port_cnt; + +#define CAN_COPY (_ibv_attr, _uverbs_attr) \ + (attr_size >= offsetofend(struct ibv_device_attr_ex, _ibv_attr) && \ + resp->response_length >= \ + offsetofend(struct ib_uverbs_ex_query_device_resp, \ + _uverbs_attr)) + +if (CAN_COPY (odp_caps, odp_caps)) { + attr->odp_caps.general_caps = resp->odp_caps.general_caps; + attr->odp_caps.per_transport_caps.rc_odp_caps = + resp->odp_caps.per_transport_caps.rc_odp_caps; + attr->odp_caps.per_transport_caps.uc_odp_caps = + resp->odp_caps.per_transport_caps.uc_odp_caps; + attr->odp_caps.per_transport_caps.ud_odp_caps = + resp->odp_caps.per_transport_caps.ud_odp_caps; + } + +if (CAN_COPY (completion_timestamp_mask, timestamp_mask)) + attr->completion_timestamp_mask = resp->timestamp_mask; + +if (CAN_COPY (hca_core_clock, hca_core_clock)) + attr->hca_core_clock = resp->hca_core_clock; + +if (CAN_COPY (device_cap_flags_ex, device_cap_flags_ex)) + attr->device_cap_flags_ex = resp->device_cap_flags_ex; + +if (CAN_COPY (rss_caps, rss_caps)) { + attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; + attr->rss_caps.max_rwq_indirection_tables = + resp->rss_caps.max_rwq_indirection_tables; + attr->rss_caps.max_rwq_indirection_table_size = + resp->rss_caps.max_rwq_indirection_table_size; + } + +if (CAN_COPY (max_wq_type_rq, max_wq_type_rq)) + attr->max_wq_type_rq = resp->max_wq_type_rq; + +if (CAN_COPY (raw_packet_caps, raw_packet_caps)) + attr->raw_packet_caps = resp->raw_packet_caps; + +if (CAN_COPY (tm_caps, tm_caps)) { + attr->tm_caps.max_rndv_hdr_size = + resp->tm_caps.max_rndv_hdr_size; + attr->tm_caps.max_num_tags = resp->tm_caps.max_num_tags; + attr->tm_caps.flags = resp->tm_caps.flags; + attr->tm_caps.max_ops = resp->tm_caps.max_ops; + attr->tm_caps.max_sge = resp->tm_caps.max_sge; + } + +if (CAN_COPY (cq_mod_caps, cq_moderation_caps)) { + attr->cq_mod_caps.max_cq_count = + resp->cq_moderation_caps.max_cq_moderation_count; + attr->cq_mod_caps.max_cq_period = + resp->cq_moderation_caps.max_cq_moderation_period; + } + +if (CAN_COPY (max_dm_size, max_dm_size)) + attr->max_dm_size = resp->max_dm_size; + +if (CAN_COPY (xrc_odp_caps, xrc_odp_caps)) + attr->xrc_odp_caps = resp->xrc_odp_caps; +#undef CAN_COPY + +return 0 ; +}
libibverbs/driver.h +5 -0
@@ -460,6 +460,11 @@ int ibv_cmd_create_flow_action_esp(struct ibv_context *ctx,int ibv_cmd_modify_flow_action_esp (struct verbs_flow_action *flow_action,struct ibv_flow_action_esp_attr *attr,struct ibv_command_buffer *driver); +int ibv_cmd_query_device_any (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size, +struct ib_uverbs_ex_query_device_resp *resp, +size_t *resp_size); int ibv_cmd_query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size,
libibverbs/libibverbs.map.in +1 -0
@@ -204,6 +204,7 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_post_srq_recv; ibv_cmd_query_context; ibv_cmd_query_device; + ibv_cmd_query_device_any; ibv_cmd_query_device_ex; ibv_cmd_query_mr; ibv_cmd_query_port;
Signature · 1 line
2.29.2
Re: [PATCH 2/9] verbs: Add ibv_cmd_query_device_any()
Quoted 47 lines · Jason Gunthorpe
This implements all the
query_devicecommand flows under a single call.Signed-off-by Jason Gunthorpe <[email protected]>--- libibverbs/cmd_device.c | 155 +++++++++++++++++++++++++++++++++++ libibverbs/driver.h | 5 ++ libibverbs/libibverbs.map.in | 1 + 3 files changed, 161 insertions(+)libibverbs/cmd_device.c +23
-0@@ -35,6 +35,7 @@#include <stdlib.h> #include <dirent.h> #include <infiniband/cmd_write.h> +#include <util/util.h> #include <net/if.h> @@ -516,3 +517,157 @@ ssize_t _ibv_query_gid_table(struct ibv_context *context,return num_entries; } + +int ibv_cmd_query_device_any (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size, +struct ib_uverbs_ex_query_device_resp *resp, +size_t *resp_size) +{ +struct ib_uverbs_ex_query_device_resp internal_resp; +size_t internal_resp_size; +int err; + +if (input && input->comp_mask) +return EINVAL ; +if (attr_size <sizeof (attr->orig_attr)) +return EINVAL ; + +if (!resp) { + resp = &internal_resp; + internal_resp_size =sizeof (internal_resp); + resp_size = &internal_resp_size; + } +memset (attr,0 , attr_size);
libibverbs/cmd_device.c ssize_t _ibv_query_gid_table(struct ibv_context *context, · line 551
Quoted 10 lines
libibverbs/cmd_device.c +10
-0+memset (resp,0 , *resp_size); + +if (attr_size >sizeof (attr->orig_attr)) { +struct ibv_query_device_ex cmd = {}; + + err =execute_cmd_write_ex (context, +IB_USER_VERBS_EX_CMD_QUERY_DEVICE , + &cmd,sizeof (cmd), resp, *resp_size); +if (err) { +if (err !=EOPNOTSUPP )
Are you sure about that?
I think older kernels return ENOSYS.
Jason Gunthorpe
maintainerRe: [PATCH 2/9] verbs: Add ibv_cmd_query_device_any()
Quoted 28 lines · Gal Pressman
Quoted 26 lines · Jason Gunthorpe
libibverbs/cmd_device.c +26
-0+int ibv_cmd_query_device_any (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size, +struct ib_uverbs_ex_query_device_resp *resp, +size_t *resp_size) +{ +struct ib_uverbs_ex_query_device_resp internal_resp; +size_t internal_resp_size; +int err; + +if (input && input->comp_mask) +return EINVAL ; +if (attr_size <sizeof (attr->orig_attr)) +return EINVAL ; + +if (!resp) { + resp = &internal_resp; + internal_resp_size =sizeof (internal_resp); + resp_size = &internal_resp_size; + } +memset (attr,0 , attr_size); +memset (resp,0 , *resp_size); + +if (attr_size >sizeof (attr->orig_attr)) { +struct ibv_query_device_ex cmd = {}; +
Quoted 8 lines
libibverbs/cmd_device.c ssize_t _ibv_query_gid_table(struct ibv_context *context, · line 551
Quoted 5 lines
libibverbs/cmd_device.c +5
-0+ err =execute_cmd_write_ex (context, +IB_USER_VERBS_EX_CMD_QUERY_DEVICE , + &cmd,sizeof (cmd), resp, *resp_size); +if (err) { +if (err !=EOPNOTSUPP )Are you sure about that?
I think older kernels return ENOSYS.
Oh, that is possibly true, we changed those at one point, I didn't test that far back.
Jason
Jason Gunthorpe
maintainerPATCH 3/9 mlx5: Move context initialization out of mlx5_query_device_ex()
In reply to Jason GunthorpeWhen the user calls mlx5_query_device_ex() it should not cause the context values to be mutated, only the attribute should be returned.
Move this code to a dedicated function that is only called during context setup.
--- providers/mlx5/mlx5.c | 10 +------ providers/mlx5/mlx5.h | 1 + providers/mlx5/verbs.c | 62 ++++++++++++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 29 deletions(-)
providers/mlx5/mlx5.c +1 -9
@@ -1373,7 +1373,6 @@ static int mlx5_set_context(struct mlx5_context *context, {struct verbs_context *v_ctx = &context->ibv_ctx;struct ibv_port_attr port_attr = {}; -struct ibv_device_attr_ex device_attr = {};int cmd_fd = v_ctx->context.cmd_fd;struct mlx5_device *mdev =to_mdev (v_ctx->context.device);struct ibv_device *ibdev = v_ctx->context.device;
@@ -1518,14 +1517,7 @@ bf_done: goto err_free; } -if (!mlx5_query_device_ex (&v_ctx->context,NULL , &device_attr, -sizeof (struct ibv_device_attr_ex ))) { - context->cached_device_cap_flags = - device_attr.orig_attr.device_cap_flags; - context->atomic_cap = device_attr.orig_attr.atomic_cap; - context->cached_tso_caps = device_attr.tso_caps; - context->max_dm_size = device_attr.max_dm_size; - } +mlx5_query_device_ctx (context);for (j =0 ; j <min (MLX5_MAX_PORTS_NUM , context->num_ports); ++j) {memset (&port_attr,0 ,sizeof (port_attr));
providers/mlx5/mlx5.h +1 -0
@@ -878,6 +878,7 @@ __be32 *mlx5_alloc_dbrec(struct mlx5_context *context, struct ibv_pd *pd,void mlx5_free_db (struct mlx5_context *context,__be32 *db,struct ibv_pd *pd,bool custom_alloc); +void mlx5_query_device_ctx (struct mlx5_context *mctx);int mlx5_query_device (struct ibv_context *context,struct ibv_device_attr *attr); int mlx5_query_device_ex(struct ibv_context *context,
providers/mlx5/verbs.c +42 -20
99 lines not on this page · raw message
Signature · 1 line
2.29.2
Re: [PATCH 3/9] mlx5: Move context initialization out of mlx5_query_device_ex()
Quoted 99 lines · Jason Gunthorpe
When the user calls
mlx5_query_device_ex()it should not cause the context values to be mutated, only the attribute should be returned.Move this code to a dedicated function that is only called during context setup.
Signed-off-by Jason Gunthorpe <[email protected]>--- providers/mlx5/mlx5.c | 10 +------ providers/mlx5/mlx5.h | 1 + providers/mlx5/verbs.c | 62 ++++++++++++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 29 deletions(-)providers/mlx5/mlx5.c +1
-9@@ -1373,7 +1373,6 @@ static int mlx5_set_context(struct mlx5_context *context, {struct verbs_context *v_ctx = &context->ibv_ctx;struct ibv_port_attr port_attr = {}; -struct ibv_device_attr_ex device_attr = {};int cmd_fd = v_ctx->context.cmd_fd;struct mlx5_device *mdev =to_mdev (v_ctx->context.device);struct ibv_device *ibdev = v_ctx->context.device;@@ -1518,14 +1517,7 @@ bf_done: goto err_free; } -if (!mlx5_query_device_ex (&v_ctx->context,NULL , &device_attr, -sizeof (struct ibv_device_attr_ex ))) { - context->cached_device_cap_flags = - device_attr.orig_attr.device_cap_flags; - context->atomic_cap = device_attr.orig_attr.atomic_cap; - context->cached_tso_caps = device_attr.tso_caps; - context->max_dm_size = device_attr.max_dm_size; - } +mlx5_query_device_ctx (context);for (j =0 ; j <min (MLX5_MAX_PORTS_NUM , context->num_ports); ++j) {memset (&port_attr,0 ,sizeof (port_attr));providers/mlx5/mlx5.h +1
-0@@ -878,6 +878,7 @@ __be32 *mlx5_alloc_dbrec(struct mlx5_context *context, struct ibv_pd *pd,void mlx5_free_db (struct mlx5_context *context,__be32 *db,struct ibv_pd *pd,bool custom_alloc); +void mlx5_query_device_ctx (struct mlx5_context *mctx);int mlx5_query_device (struct ibv_context *context,struct ibv_device_attr *attr); int mlx5_query_device_ex(struct ibv_context *context,providers/mlx5/verbs.c +18
-3@@ -3450,19 +3450,19 @@ static void get_pci_atomic_caps(struct ibv_context *context, } } -static void get_lag_caps (struct ibv_context *ctx) +static void get_lag_caps (struct mlx5_context *mctx) {uint16_t opmod =MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE |HCA_CAP_OPMOD_GET_CUR ;uint32_t out[DEVX_ST_SZ_DW (query_hca_cap_out)] = {};uint32_t in[DEVX_ST_SZ_DW (query_hca_cap_in)] = {}; -struct mlx5_context *mctx =to_mctx (ctx);int ret;DEVX_SET (query_hca_cap_in, in, opcode,MLX5_CMD_OP_QUERY_HCA_CAP );DEVX_SET (query_hca_cap_in, in, op_mod, opmod); - ret =mlx5dv_devx_general_cmd (ctx, in,sizeof (in), out,sizeof (out)); + ret =mlx5dv_devx_general_cmd (&mctx->ibv_ctx.context, in,sizeof (in), + out,sizeof (out));if (ret)return ;@@ -3512,6 +3512,41 @@ int mlx5_query_device_ex(struct ibv_context *context, attr->packet_pacing_caps.supported_qpts = resp.packet_pacing_caps.supported_qpts; + major = (raw_fw_ver >>32 ) &0xffff ; + minor = (raw_fw_ver >>16 ) &0xffff ; + sub_minor = raw_fw_ver &0xffff ; + a = &attr->orig_attr; +snprintf (a->fw_ver,sizeof (a->fw_ver),"%d.%d.%04d" , + major, minor, sub_minor); + +if (attr_size >= offsetof(struct ibv_device_attr_ex , pci_atomic_caps) + +sizeof (attr->pci_atomic_caps)) +get_pci_atomic_caps (context, attr); + +return 0 ; +} + +void mlx5_query_device_ctx (struct mlx5_context *mctx)
providers/mlx5/verbs.c int mlx5_query_device_ex(struct ibv_context *context, · line 3539
Quoted 10 lines
providers/mlx5/verbs.c +10
-0+{ +struct ibv_device_attr_ex device_attr; +struct mlx5_query_device_ex_resp resp; +size_t resp_size =sizeof (resp); + +get_lag_caps (mctx); + +if (!(mctx->cmds_supp_uhw &MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE )) +return ; +
Any reason to not read some applicable context fields (e.g. max_dm_size) over uverbs by reducing the resp_size to the ib part ?
providers/mlx5/verbs.c int mlx5_query_device_ex(struct ibv_context *context, · line 3547
Quoted 8 lines
providers/mlx5/verbs.c +8
-0+if (ibv_cmd_query_device_any (&mctx->ibv_ctx.context,NULL , &device_attr, +sizeof (device_attr), &resp.ibv_resp, + &resp_size)) +return ; + + mctx->cached_device_cap_flags = device_attr.orig_attr.device_cap_flags; + mctx->atomic_cap = device_attr.orig_attr.atomic_cap; + mctx->cached_tso_caps = device_attr.tso_caps;
The device_attr.tso_caps is not set over uverbs / cmd_device.c, it comes as UHW. The below should be done instead.
mctx->cached_tso_caps.max_tso = resp.tso_caps.max_tso; mctx->cached_tso_caps.supported_qpts = resp.tso_caps.supported_qpts;
Quoted 43 lines
providers/mlx5/verbs.c +6
-17+ mctx->max_dm_size = device_attr.max_dm_size; +if (resp.mlx5_ib_support_multi_pkt_send_wqes &MLX5_IB_ALLOW_MPW ) mctx->vendor_cap_flags |=MLX5_VENDOR_CAP_FLAGS_MPW_ALLOWED ;@@ -3519,7 +3554,8 @@ int mlx5_query_device_ex(struct ibv_context *context, mctx->vendor_cap_flags |=MLX5_VENDOR_CAP_FLAGS_ENHANCED_MPW ; mctx->cqe_comp_caps.max_num = resp.cqe_comp_caps.max_num; - mctx->cqe_comp_caps.supported_format = resp.cqe_comp_caps.supported_format; + mctx->cqe_comp_caps.supported_format = + resp.cqe_comp_caps.supported_format; mctx->sw_parsing_caps.sw_parsing_offloads = resp.sw_parsing_caps.sw_parsing_offloads; mctx->sw_parsing_caps.supported_qpts =@@ -3544,25 +3580,11 @@ int mlx5_query_device_ex(struct ibv_context *context, mctx->vendor_cap_flags |=MLX5_VENDOR_CAP_FLAGS_CQE_128B_PAD ;if (resp.flags &MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE ) - mctx->vendor_cap_flags |=MLX5_VENDOR_CAP_FLAGS_PACKET_BASED_CREDIT_MODE ; + mctx->vendor_cap_flags |= +MLX5_VENDOR_CAP_FLAGS_PACKET_BASED_CREDIT_MODE ;if (resp.flags &MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT ) mctx->vendor_cap_flags |=MLX5_VENDOR_CAP_FLAGS_SCAT2CQE_DCT ; - - major = (raw_fw_ver >>32 ) &0xffff ; - minor = (raw_fw_ver >>16 ) &0xffff ; - sub_minor = raw_fw_ver &0xffff ; - a = &attr->orig_attr; -snprintf (a->fw_ver,sizeof (a->fw_ver),"%d.%d.%04d" , - major, minor, sub_minor); - -if (attr_size >= offsetof(struct ibv_device_attr_ex , pci_atomic_caps) + -sizeof (attr->pci_atomic_caps)) -get_pci_atomic_caps (context, attr); - -get_lag_caps (context); - -return 0 ; }static int rwq_sig_enabled (struct ibv_context *context)
Jason Gunthorpe
maintainerRe: [PATCH 3/9] mlx5: Move context initialization out of mlx5_query_device_ex()
Quoted 6 lines · Yishai Hadas
Quoted 6 lines
providers/mlx5/verbs.c +6
-0+void mlx5_query_device_ctx (struct mlx5_context *mctx) +{ +struct ibv_device_attr_ex device_attr; +struct mlx5_query_device_ex_resp resp; +size_t resp_size =sizeof (resp); +
Quoted 8 lines
providers/mlx5/verbs.c int mlx5_query_device_ex(struct ibv_context *context, · line 3539
Quoted 5 lines
providers/mlx5/verbs.c +5
-0+get_lag_caps (mctx); + +if (!(mctx->cmds_supp_uhw &MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE )) +return ; +Any reason to not read some applicable context fields (e.g.
max_dm_size) over uverbs by reducing theresp_sizeto the ib part ?
No, I missed the device_attr detail
Quoted 11 lines
providers/mlx5/verbs.c int mlx5_query_device_ex(struct ibv_context *context, · line 3547
Quoted 8 lines
providers/mlx5/verbs.c +8
-0+if (ibv_cmd_query_device_any (&mctx->ibv_ctx.context,NULL , &device_attr, +sizeof (device_attr), &resp.ibv_resp, + &resp_size)) +return ; + + mctx->cached_device_cap_flags = device_attr.orig_attr.device_cap_flags; + mctx->atomic_cap = device_attr.orig_attr.atomic_cap; + mctx->cached_tso_caps = device_attr.tso_caps;The device_attr.tso_caps is not set over uverbs /
cmd_device.c, it comes as UHW. The below should be done instead.
Got it
I updated the github
Thanks,
Jason
Jason Gunthorpe
maintainerPATCH 4/9 efa: Move the context intialization out of efa_query_device_ex()
In reply to Jason GunthorpeWhen the user calls efa_query_device_ex() it should not cause the context values to be mutated, only the attribute shuld be returned.
Move this code to a dedicated function that is only called during context setup.
--- providers/efa/efa.c | 14 +------------ providers/efa/verbs.c | 46 +++++++++++++++++++++++++++++++++++-------- providers/efa/verbs.h | 1 + 3 files changed, 40 insertions(+), 21 deletions(-)
providers/efa/efa.c +1 -13
@@ -54,10 +54,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev, {struct efa_alloc_ucontext_resp resp = {};struct efa_alloc_ucontext cmd = {}; -struct ibv_device_attr_ex attr; -unsigned int qp_table_sz;struct efa_context *ctx; -int err; cmd.comp_mask |=EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH ; cmd.comp_mask |=EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR ;
@@ -86,17 +83,8 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,verbs_set_ops (&ctx->ibvctx, &efa_ctx_ops); - err =efa_query_device_ex (&ctx->ibvctx.context,NULL , &attr, -sizeof (attr)); -if (err) +if (!efa_query_device_ctx (ctx)) goto err_free_spinlock; - - qp_table_sz =roundup_pow_of_two (attr.orig_attr.max_qp); - ctx->qp_table_sz_m1 = qp_table_sz -1 ; - ctx->qp_table =calloc (qp_table_sz,sizeof (*ctx->qp_table)); -if (!ctx->qp_table) - goto err_free_spinlock; -return &ctx->ibvctx; err_free_spinlock:
providers/efa/verbs.c +38 -8
58 lines not on this page · raw message
providers/efa/verbs.h +1 -0
@@ -9,6 +9,7 @@#include <infiniband/driver.h> #include <infiniband/verbs.h> +int efa_query_device_ctx (struct efa_context *ctx);int efa_query_device (struct ibv_context *uctx,struct ibv_device_attr *attr);int efa_query_port (struct ibv_context *uctx,uint8_t port,struct ibv_port_attr *attr);
Signature · 1 line
2.29.2
Re: [PATCH 4/9] efa: Move the context intialization out of efa_query_device_ex()
Quoted 8 lines · Jason Gunthorpe
When the user calls
efa_query_device_ex()it should not cause the context values to be mutated, only the attribute shuld be returned.Move this code to a dedicated function that is only called during context setup.
Cc Gal Pressman <[email protected]>Signed-off-by Jason Gunthorpe <[email protected]>
Didn't get a chance to review yet, but this one breaks EFA. I'll try to provide more info today/tomorrow.
(BTW: typo in the subject line "intialization", and "shuld" in the commit message)
Re: [PATCH 4/9] efa: Move the context intialization out of efa_query_device_ex()
In reply to Jason GunthorpeQuoted 30 lines · Jason Gunthorpe
When the user calls
efa_query_device_ex()it should not cause the context values to be mutated, only the attribute shuld be returned.Move this code to a dedicated function that is only called during context setup.
Cc Gal Pressman <[email protected]>Signed-off-by Jason Gunthorpe <[email protected]>--- providers/efa/efa.c | 14 +------------ providers/efa/verbs.c | 46 +++++++++++++++++++++++++++++++++++-------- providers/efa/verbs.h | 1 + 3 files changed, 40 insertions(+), 21 deletions(-)providers/efa/efa.c +0
-3@@ -54,10 +54,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev, {struct efa_alloc_ucontext_resp resp = {};struct efa_alloc_ucontext cmd = {}; -struct ibv_device_attr_ex attr; -unsigned int qp_table_sz;struct efa_context *ctx; -int err; cmd.comp_mask |=EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH ; cmd.comp_mask |=EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR ;
providers/efa/efa.c static struct verbs_context *efa_alloc_context(struct ibv_device *vdev, · line 87
Quoted 8 lines
providers/efa/efa.c +1
-3@@ -86,17 +83,8 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,verbs_set_ops (&ctx->ibvctx, &efa_ctx_ops); - err =efa_query_device_ex (&ctx->ibvctx.context,NULL , &attr, -sizeof (attr)); -if (err) +if (!efa_query_device_ctx (ctx)) goto err_free_spinlock;
This return error on success seems to be the issue, will verify.
Re: [PATCH 4/9] efa: Move the context intialization out of efa_query_device_ex()
In reply to Jason GunthorpeQuoted 30 lines · Jason Gunthorpe
When the user calls
efa_query_device_ex()it should not cause the context values to be mutated, only the attribute shuld be returned.Move this code to a dedicated function that is only called during context setup.
Cc Gal Pressman <[email protected]>Signed-off-by Jason Gunthorpe <[email protected]>--- providers/efa/efa.c | 14 +------------ providers/efa/verbs.c | 46 +++++++++++++++++++++++++++++++++++-------- providers/efa/verbs.h | 1 + 3 files changed, 40 insertions(+), 21 deletions(-)providers/efa/efa.c +0
-3@@ -54,10 +54,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev, {struct efa_alloc_ucontext_resp resp = {};struct efa_alloc_ucontext cmd = {}; -struct ibv_device_attr_ex attr; -unsigned int qp_table_sz;struct efa_context *ctx; -int err; cmd.comp_mask |=EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH ; cmd.comp_mask |=EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR ;
providers/efa/efa.c static struct verbs_context *efa_alloc_context(struct ibv_device *vdev, · line 86
Quoted 7 lines
providers/efa/efa.c +1
-3@@ -86,17 +83,8 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,verbs_set_ops (&ctx->ibvctx, &efa_ctx_ops); - err =efa_query_device_ex (&ctx->ibvctx.context,NULL , &attr, -sizeof (attr)); -if (err) +if (!efa_query_device_ctx (ctx))
Remove the not.
Quoted 31 lines
providers/efa/efa.c +0
-7goto err_free_spinlock; - - qp_table_sz =roundup_pow_of_two (attr.orig_attr.max_qp); - ctx->qp_table_sz_m1 = qp_table_sz -1 ; - ctx->qp_table =calloc (qp_table_sz,sizeof (*ctx->qp_table)); -if (!ctx->qp_table) - goto err_free_spinlock; -return &ctx->ibvctx; err_free_spinlock:providers/efa/verbs.c +0
-8@@ -106,14 +106,6 @@ int efa_query_device_ex(struct ibv_context *context,if (err)return err; - ctx->device_caps = resp.device_caps; - ctx->max_sq_wr = resp.max_sq_wr; - ctx->max_rq_wr = resp.max_rq_wr; - ctx->max_sq_sge = resp.max_sq_sge; - ctx->max_rq_sge = resp.max_rq_sge; - ctx->max_rdma_size = resp.max_rdma_size; - ctx->max_wr_rdma_sge = a->max_sge_rd; - a->max_qp_wr =min_t (int, a->max_qp_wr, ctx->max_llq_size /sizeof (struct efa_io_tx_wqe )); snprintf(a->fw_ver, sizeof(a->fw_ver), "%u.%u.%u.%u",
providers/efa/verbs.c int efa_query_device_ex(struct ibv_context *context, · line 120
Quoted 7 lines
providers/efa/verbs.c +4
-0@@ -122,6 +114,44 @@ int efa_query_device_ex(struct ibv_context *context,return 0 ; } +int efa_query_device_ctx (struct efa_context *ctx) +{ +struct ibv_device_attr_ex attr; +struct efa_query_device_ex_resp resp;
Preferably I would put this first.
Quoted 8 lines
providers/efa/verbs.c +8
-0+size_t resp_size =sizeof (resp); +unsigned int qp_table_sz; +int err; + +if (ctx->cmds_supp_udata_mask &EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE ) { + err =ibv_cmd_query_device_any (&ctx->ibvctx.context,NULL , + &attr,sizeof (attr), + &resp.ibv_resp, &resp_size);
providers/efa/verbs.c int efa_query_device_ex(struct ibv_context *context, · line 138
Quoted 10 lines
providers/efa/verbs.c +10
-0+if (err) +return err; + + ctx->device_caps = resp.device_caps; + ctx->max_sq_wr = resp.max_sq_wr; + ctx->max_rq_wr = resp.max_rq_wr; + ctx->max_sq_sge = resp.max_sq_sge; + ctx->max_rq_sge = resp.max_rq_sge; + ctx->max_rdma_size = resp.max_rdma_size; + ctx->max_wr_rdma_sge = attr.orig_attr.max_sge_rd;
max_wr_rdma_sge assignment can be done in the else clause as well.
Quoted 15 lines
providers/efa/verbs.c +15
-0+ }else { + err =ibv_cmd_query_device_any (&ctx->ibvctx.context,NULL , + &attr,sizeof (attr.orig_attr), +NULL ,NULL ); +if (err) +return err; + } + + qp_table_sz =roundup_pow_of_two (attr.orig_attr.max_qp); + ctx->qp_table_sz_m1 = qp_table_sz -1 ; + ctx->qp_table =calloc (qp_table_sz,sizeof (*ctx->qp_table)); +if (!ctx->qp_table) +return ENOMEM ; +return 0 ; +}
Jason Gunthorpe
maintainerRe: [PATCH 4/9] efa: Move the context intialization out of efa_query_device_ex()
Quoted 12 lines · Gal Pressman
Quoted 12 lines
providers/efa/verbs.c +12
-0+size_t resp_size =sizeof (resp); +unsigned int qp_table_sz; +int err; + +if (ctx->cmds_supp_udata_mask &EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE ) { + err =ibv_cmd_query_device_any (&ctx->ibvctx.context,NULL , + &attr,sizeof (attr), + &resp.ibv_resp, &resp_size); +if (err) +return err; + + ctx->device_caps = resp.device_caps;
Quoted 8 lines
providers/efa/verbs.c int efa_query_device_ex(struct ibv_context *context, · line 138
Quoted 6 lines
providers/efa/verbs.c +6
-0+ ctx->max_sq_wr = resp.max_sq_wr; + ctx->max_rq_wr = resp.max_rq_wr; + ctx->max_sq_sge = resp.max_sq_sge; + ctx->max_rq_sge = resp.max_rq_sge; + ctx->max_rdma_size = resp.max_rdma_size; + ctx->max_wr_rdma_sge = attr.orig_attr.max_sge_rd;
max_wr_rdma_sgeassignment can be done in the else clause as well.
Yes, it is the same mistake as I did in mlx5
I updated everything, thanks
Jason
Jason Gunthorpe
maintainerPATCH 5/9 mlx4: Move the context intialization out of mlx4_query_device_ex()
In reply to Jason GunthorpeWhen the user calls mlx4_query_device_ex() it should not cause the context values to be mutated, only the attribute shuld be returned.
Move this code to a dedicated function that is only called during context setup.
--- providers/mlx4/mlx4.c | 33 +------------------------------- providers/mlx4/mlx4.h | 1 + providers/mlx4/verbs.c | 43 +++++++++++++++++++++++++++++++++++------- 3 files changed, 38 insertions(+), 39 deletions(-)
providers/mlx4/mlx4.c +1 -32
@@ -136,28 +136,6 @@ static const struct verbs_context_ops mlx4_ctx_ops = { .free_context = mlx4_free_context, }; -static int mlx4_map_internal_clock (struct mlx4_device *mdev, -struct ibv_context *ibv_ctx) -{ -struct mlx4_context *context =to_mctx (ibv_ctx); -void *hca_clock_page; - - hca_clock_page =mmap (NULL , mdev->page_size, -PROT_READ ,MAP_SHARED , ibv_ctx->cmd_fd, - mdev->page_size *3 ); - -if (hca_clock_page ==MAP_FAILED ) { -fprintf (stderr,PFX -"Warning: Timestamp available,\n" -"but failed to mmap() hca core clock page.\n" ); -return -1 ; - } - - context->hca_core_clock = hca_clock_page + - (context->core_clock.offset & (mdev->page_size -1 )); -return 0 ; -} -static struct verbs_context *mlx4_alloc_context (struct ibv_device *ibdev,int cmd_fd,void *private_data)
@@ -170,7 +148,6 @@ static struct verbs_context *mlx4_alloc_context(struct ibv_device *ibdev,__u16 bf_reg_size;struct mlx4_device *dev =to_mdev (ibdev);struct verbs_context *verbs_ctx; -struct ibv_device_attr_ex dev_attrs; context =verbs_init_and_alloc_context (ibdev, cmd_fd, context, ibv_ctx,RDMA_DRIVER_MLX4 );
@@ -242,15 +219,7 @@ static struct verbs_context *mlx4_alloc_context(struct ibv_device *ibdev,verbs_set_ops (verbs_ctx, &mlx4_ctx_ops); - context->hca_core_clock =NULL ; -memset (&dev_attrs,0 ,sizeof (dev_attrs)); -if (!mlx4_query_device_ex (&verbs_ctx->context,NULL , &dev_attrs, -sizeof (struct ibv_device_attr_ex ))) { - context->max_qp_wr = dev_attrs.orig_attr.max_qp_wr; - context->max_sge = dev_attrs.orig_attr.max_sge; -if (context->core_clock.offset_valid) -mlx4_map_internal_clock (dev, &verbs_ctx->context); - } +mlx4_query_device_ctx (dev, context);return verbs_ctx;
providers/mlx4/mlx4.h +1 -0
@@ -304,6 +304,7 @@ __be32 *mlx4_alloc_db(struct mlx4_context *context, enum mlx4_db_type type);void mlx4_free_db (struct mlx4_context *context,enum mlx4_db_type type,__be32 *db); +void mlx4_query_device_ctx (struct mlx4_device *mdev,struct mlx4_context *mctx);int mlx4_query_device (struct ibv_context *context,struct ibv_device_attr *attr); int mlx4_query_device_ex(struct ibv_context *context,
providers/mlx4/verbs.c +36 -7
67 lines not on this page · raw message
Signature · 1 line
2.29.2
Jason Gunthorpe
maintainerPATCH 6/9 providers: Remove normal query_device() from providers that have _ex
In reply to Jason GunthorpeThe ex callback can implement both versions, no reason for duplicate code in two paths. Have the core code call the _ex version instead.
--- libibverbs/dummy_ops.c | 9 ++++- providers/efa/efa.c | 1 - providers/efa/verbs.c | 36 +++++--------------- providers/efa/verbs.h | 1 - providers/mlx4/mlx4.c | 1 - providers/mlx4/mlx4.h | 2 -- providers/mlx4/verbs.c | 45 +++++++++---------------- providers/mlx5/mlx5.c | 1 - providers/mlx5/mlx5.h | 2 -- providers/mlx5/verbs.c | 75 +++++++++++++++++------------------------- 10 files changed, 61 insertions(+), 112 deletions(-)
libibverbs/dummy_ops.c +8 -1
@@ -380,7 +380,14 @@ static int post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr,static int query_device (struct ibv_context *context,struct ibv_device_attr *device_attr) { -return EOPNOTSUPP ; +const struct verbs_context_ops *ops =get_ops (context); + +if (!ops->query_device_ex) +return EOPNOTSUPP ; +return ops->query_device_ex ( + context,NULL , + container_of(device_attr, struct ibv_device_attr_ex, orig_attr), +sizeof (*device_attr)); }/* Provide a generic implementation for all providers that don't implement
providers/efa/efa.c +0 -1
@@ -40,7 +40,6 @@ static const struct verbs_context_ops efa_ctx_ops = { .poll_cq = efa_poll_cq, .post_recv = efa_post_recv, .post_send = efa_post_send, - .query_device = efa_query_device, .query_device_ex = efa_query_device_ex, .query_port = efa_query_port, .query_qp = efa_query_qp,
providers/efa/verbs.c +8 -28
@@ -56,27 +56,6 @@ struct efa_wq_init_attr {uint16_t sub_cq_idx; }; -int efa_query_device (struct ibv_context *ibvctx, -struct ibv_device_attr *dev_attr) -{ -struct efa_context *ctx =to_efa_context (ibvctx); -struct ibv_query_device cmd; -uint8_t fw_ver[8 ]; -int err; - - err =ibv_cmd_query_device (ibvctx, dev_attr, (uint64_t *)&fw_ver, - &cmd,sizeof (cmd)); -if (err) -return err; - - dev_attr->max_qp_wr =min_t (int, dev_attr->max_qp_wr, - ctx->max_llq_size /sizeof (struct efa_io_tx_wqe )); -snprintf (dev_attr->fw_ver,sizeof (dev_attr->fw_ver),"%u.%u.%u.%u" , - fw_ver[0 ], fw_ver[1 ], fw_ver[2 ], fw_ver[3 ]); - -return 0 ; -} -int efa_query_port (struct ibv_context *ibvctx,uint8_t port,struct ibv_port_attr *port_attr) {
@@ -91,23 +70,24 @@ int efa_query_device_ex(struct ibv_context *context, size_t attr_size) {struct efa_context *ctx =to_efa_context (context); -int cmd_supp_uhw = ctx->cmds_supp_udata_mask & -EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE ;struct ibv_device_attr *a = &attr->orig_attr;struct efa_query_device_ex_resp resp = {}; -struct ibv_query_device_ex cmd = {}; +size_t resp_size = (ctx->cmds_supp_udata_mask & +EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE ) ? +sizeof (resp) : +sizeof (resp.ibv_resp);uint8_t fw_ver[8 ];int err; - err =ibv_cmd_query_device_ex ( - context, input, attr, attr_size, (uint64_t *)&fw_ver, &cmd, -sizeof (cmd), &resp.ibv_resp, - cmd_supp_uhw ?sizeof (resp) :sizeof (resp.ibv_resp)); + err =ibv_cmd_query_device_any (context, input, attr, attr_size, + &resp.ibv_resp, &resp_size);if (err)return err; a->max_qp_wr =min_t (int, a->max_qp_wr, ctx->max_llq_size /sizeof (struct efa_io_tx_wqe )); +memcpy (fw_ver, &resp.ibv_resp.base.fw_ver, +sizeof (resp.ibv_resp.base.fw_ver));snprintf (a->fw_ver,sizeof (a->fw_ver),"%u.%u.%u.%u" , fw_ver[0 ], fw_ver[1 ], fw_ver[2 ], fw_ver[3 ]);
providers/efa/verbs.h +0 -1
@@ -10,7 +10,6 @@#include <infiniband/verbs.h> int efa_query_device_ctx (struct efa_context *ctx); -int efa_query_device (struct ibv_context *uctx,struct ibv_device_attr *attr);int efa_query_port (struct ibv_context *uctx,uint8_t port,struct ibv_port_attr *attr); int efa_query_device_ex(struct ibv_context *context,
providers/mlx4/mlx4.c +0 -1
@@ -84,7 +84,6 @@ static const struct verbs_match_ent hca_table[] = { };static const struct verbs_context_ops mlx4_ctx_ops = { - .query_device = mlx4_query_device, .query_port = mlx4_query_port, .alloc_pd = mlx4_alloc_pd, .dealloc_pd = mlx4_free_pd,
providers/mlx4/mlx4.h +0 -2
@@ -305,8 +305,6 @@ void mlx4_free_db(struct mlx4_context *context, enum mlx4_db_type type, __be32 *db);void mlx4_query_device_ctx (struct mlx4_device *mdev,struct mlx4_context *mctx); -int mlx4_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); int mlx4_query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr,
providers/mlx4/verbs.c +15 -30
66 lines not on this page · raw message
providers/mlx5/mlx5.c +0 -1
@@ -90,7 +90,6 @@ uint32_t mlx5_debug_mask = 0;int mlx5_freeze_on_error_cqe;static const struct verbs_context_ops mlx5_ctx_common_ops = { - .query_device = mlx5_query_device, .query_port = mlx5_query_port, .alloc_pd = mlx5_alloc_pd, .async_event = mlx5_async_event,
providers/mlx5/mlx5.h +0 -2
@@ -879,8 +879,6 @@ void mlx5_free_db(struct mlx5_context *context, __be32 *db, struct ibv_pd *pd, bool custom_alloc);void mlx5_query_device_ctx (struct mlx5_context *mctx); -int mlx5_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); int mlx5_query_device_ex(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr,
providers/mlx5/verbs.c +30 -45
104 lines not on this page · raw message
Signature · 1 line
2.29.2
Re: [PATCH 6/9] providers: Remove normal query_device() from providers that have _ex
Quoted 4 lines · Jason Gunthorpe
The ex callback can implement both versions, no reason for duplicate code in two paths. Have the core code call the
_exversion instead.Signed-off-by Jason Gunthorpe <[email protected]>
Jason Gunthorpe
maintainerPATCH 7/9 providers: Convert all providers to implement query_device_ex
In reply to Jason GunthorpeThe kernel now supports query_device_ex for all drivers, there is no reason to have a weird split where providers do different things.
All providers implement only query_device_ex and call ibv_cmd_query_device_any() to get as much of the device_attr's as the kernel can return.
The user facing ibv_query_device is emulated by requesting only the first portion of the ibv_device_attr_ex structure using a shorter size.
Nearly all providers have a fairly simple pattern where they just call ibv_cmd_query_device_any() and the manipulate the fw_version into a string. A few return a device udata and process that as well.
--- providers/bnxt_re/main.c | 2 +- providers/bnxt_re/verbs.c | 30 +++++++++++++++++++----------- providers/bnxt_re/verbs.h | 5 +++-- providers/cxgb4/dev.c | 2 +- providers/cxgb4/libcxgb4.h | 3 ++- providers/cxgb4/verbs.c | 14 +++++++++----- providers/hfi1verbs/hfiverbs.c | 2 +- providers/hfi1verbs/hfiverbs.h | 5 +++-- providers/hfi1verbs/verbs.c | 13 ++++++++----- providers/hns/hns_roce_u.c | 8 ++++++-- providers/hns/hns_roce_u.h | 3 ++- providers/hns/hns_roce_u_verbs.c | 15 +++++++++------ providers/i40iw/i40iw_umain.c | 2 +- providers/i40iw/i40iw_umain.h | 4 +++- providers/i40iw/i40iw_uverbs.c | 18 +++++++++++------- providers/ipathverbs/ipathverbs.c | 2 +- providers/ipathverbs/ipathverbs.h | 5 +++-- providers/ipathverbs/verbs.c | 13 ++++++++----- providers/mthca/mthca.c | 2 +- providers/mthca/mthca.h | 3 ++- providers/mthca/verbs.c | 13 +++++++++---- providers/ocrdma/ocrdma_main.c | 2 +- providers/ocrdma/ocrdma_main.h | 4 +++- providers/ocrdma/ocrdma_verbs.c | 20 ++++++++++++-------- providers/qedr/qelr_main.c | 2 +- providers/qedr/qelr_verbs.c | 21 ++++++++++++--------- providers/qedr/qelr_verbs.h | 3 ++- providers/rxe/rxe.c | 15 +++++++++------ providers/siw/siw.c | 20 +++++++++++--------- providers/vmw_pvrdma/pvrdma.h | 3 ++- providers/vmw_pvrdma/pvrdma_main.c | 2 +- providers/vmw_pvrdma/verbs.c | 13 ++++++++----- 32 files changed, 165 insertions(+), 104 deletions(-)
providers/bnxt_re/main.c +1 -1
@@ -91,7 +91,7 @@ static const struct verbs_match_ent cna_table[] = { };static const struct verbs_context_ops bnxt_re_cntx_ops = { - .query_device = bnxt_re_query_device, + .query_device_ex = bnxt_re_query_device, .query_port = bnxt_re_query_port, .alloc_pd = bnxt_re_alloc_pd, .dealloc_pd = bnxt_re_free_pd,
providers/bnxt_re/verbs.c +19 -11
@@ -53,19 +53,24 @@#include "main.h" #include "verbs.h" -int bnxt_re_query_device (struct ibv_context *ibvctx, -struct ibv_device_attr *dev_attr) +int bnxt_re_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint8_t fw_ver[8 ]; -int status; +int err; -memset (dev_attr,0 ,sizeof (struct ibv_device_attr )); - status =ibv_cmd_query_device (ibvctx, dev_attr, (uint64_t *)&fw_ver, - &cmd,sizeof (cmd)); -snprintf (dev_attr->fw_ver,64 ,"%d.%d.%d.%d" , - fw_ver[0 ], fw_ver[1 ], fw_ver[2 ], fw_ver[3 ]); -return status; + err =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size); +if (err) +return err; + +memcpy (fw_ver, &resp.base.fw_ver,sizeof (resp.base.fw_ver)); +snprintf (attr->orig_attr.fw_ver,64 ,"%d.%d.%d.%d" , fw_ver[0 ], + fw_ver[1 ], fw_ver[2 ], fw_ver[3 ]); +return 0 ; } int bnxt_re_query_port(struct ibv_context *ibvctx, uint8_t port,
@@ -773,7 +778,10 @@ static int bnxt_re_check_qp_limits(struct bnxt_re_context *cntx,struct ibv_device_attr devattr;int ret; - ret =bnxt_re_query_device (&cntx->ibvctx.context, &devattr); + ret =bnxt_re_query_device ( + &cntx->ibvctx.context,NULL , + container_of(&devattr, struct ibv_device_attr_ex, orig_attr), +sizeof (devattr));if (ret)return ret;if (attr->cap.max_send_sge > devattr.max_sge)
providers/bnxt_re/verbs.h +3 -2
@@ -54,8 +54,9 @@#include <infiniband/driver.h> #include <infiniband/verbs.h> -int bnxt_re_query_device (struct ibv_context *uctx, -struct ibv_device_attr *attr); +int bnxt_re_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int bnxt_re_query_port (struct ibv_context *uctx,uint8_t port,struct ibv_port_attr *attr);struct ibv_pd *bnxt_re_alloc_pd (struct ibv_context *uctx);
providers/cxgb4/dev.c +1 -1
@@ -75,7 +75,7 @@ int t5_en_wc = 1;static LIST_HEAD (devices);static const struct verbs_context_ops c4iw_ctx_common_ops = { - .query_device = c4iw_query_device, + .query_device_ex = c4iw_query_device, .query_port = c4iw_query_port, .alloc_pd = c4iw_alloc_pd, .dealloc_pd = c4iw_free_pd,
providers/cxgb4/libcxgb4.h +2 -1
@@ -191,7 +191,8 @@ static inline unsigned long_log2(unsigned long x) }int c4iw_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int c4iw_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/cxgb4/verbs.c +9 -5
@@ -47,24 +47,28 @@ bool is_64b_cqe;#define MASKED (x) (void *)((unsigned long)(x) & c4iw_page_mask) -int c4iw_query_device (struct ibv_context *context,struct ibv_device_attr *attr) +int c4iw_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;u8 major, minor, sub_minor, build;int ret; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, &cmd, -sizeof cmd); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>24 ) &0xff ; minor = (raw_fw_ver >>16 ) &0xff ; sub_minor = (raw_fw_ver >>8 ) &0xff ; build = raw_fw_ver &0xff ; -snprintf (attr->fw_ver,sizeof attr->fw_ver, +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%d.%d" , major, minor, sub_minor, build);return 0 ;
providers/hfi1verbs/hfiverbs.c +1 -1
@@ -90,7 +90,7 @@ static const struct verbs_match_ent hca_table[] = {static const struct verbs_context_ops hfi1_ctx_common_ops = { .free_context = hfi1_free_context, - .query_device = hfi1_query_device, + .query_device_ex = hfi1_query_device, .query_port = hfi1_query_port, .alloc_pd = hfi1_alloc_pd,
providers/hfi1verbs/hfiverbs.h +3 -2
@@ -194,8 +194,9 @@ static inline struct hfi1_rwqe *get_rwqe_ptr(struct hfi1_rq *rq, rq->max_sge * sizeof(struct ibv_sge)) * n); } -extern int hfi1_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +int hfi1_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);extern int hfi1_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/hfi1verbs/verbs.c +8 -5
@@ -68,23 +68,26 @@#include "hfi-abi.h" int hfi1_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;unsigned major, minor, sub_minor;int ret; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, - &cmd,sizeof cmd); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof attr->fw_ver, +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%d" , major, minor, sub_minor);return 0 ;
providers/hns/hns_roce_u.c +6 -2
@@ -74,7 +74,7 @@ static const struct verbs_context_ops hns_common_ops = { .dereg_mr = hns_roce_u_dereg_mr, .destroy_cq = hns_roce_u_destroy_cq, .modify_cq = hns_roce_u_modify_cq, - .query_device = hns_roce_u_query_device, + .query_device_ex = hns_roce_u_query_device, .query_port = hns_roce_u_query_port, .query_qp = hns_roce_u_query_qp, .reg_mr = hns_roce_u_reg_mr,
@@ -147,7 +147,11 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,verbs_set_ops (&context->ibv_ctx, &hns_common_ops);verbs_set_ops (&context->ibv_ctx, &hr_dev->u_hw->hw_ops); -if (hns_roce_u_query_device (&context->ibv_ctx.context, &dev_attrs)) +if (hns_roce_u_query_device (&context->ibv_ctx.context,NULL , +container_of (&dev_attrs, + struct ibv_device_attr_ex, + orig_attr), +sizeof (dev_attrs))) goto tptr_free; context->max_qp_wr = dev_attrs.max_qp_wr;
providers/hns/hns_roce_u.h +2 -1
@@ -316,7 +316,8 @@ static inline struct hns_roce_qp *to_hr_qp(struct ibv_qp *ibv_qp) }int hns_roce_u_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int hns_roce_u_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/hns/hns_roce_u_verbs.c +9 -6
@@ -54,24 +54,27 @@ void hns_roce_init_qp_indices(struct hns_roce_qp *qp) }int hns_roce_u_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);int ret; -struct ibv_query_device cmd;uint64_t raw_fw_ver;unsigned int major, minor, sub_minor; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, &cmd, -sizeof (cmd)); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof (attr->fw_ver),"%d.%d.%03d" , major, minor, - sub_minor); +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver), +"%d.%d.%03d" , major, minor, sub_minor);return 0 ; }
providers/i40iw/i40iw_umain.c +1 -1
@@ -95,7 +95,7 @@ static const struct verbs_match_ent hca_table[] = { };static const struct verbs_context_ops i40iw_uctx_ops = { - .query_device = i40iw_uquery_device, + .query_device_ex = i40iw_uquery_device, .query_port = i40iw_uquery_port, .alloc_pd = i40iw_ualloc_pd, .dealloc_pd = i40iw_ufree_pd,
providers/i40iw/i40iw_umain.h +3 -1
@@ -151,7 +151,9 @@ static inline struct i40iw_uqp *to_i40iw_uqp(struct ibv_qp *ibqp) }/* i40iw_uverbs.c */ -int i40iw_uquery_device (struct ibv_context *,struct ibv_device_attr *); +int i40iw_uquery_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int i40iw_uquery_port (struct ibv_context *,uint8_t ,struct ibv_port_attr *);struct ibv_pd *i40iw_ualloc_pd (struct ibv_context *);int i40iw_ufree_pd (struct ibv_pd *);
providers/i40iw/i40iw_uverbs.c +11 -7
@@ -55,23 +55,27 @@* @context: user context for the device * @attr: where to save all the mx resources from the driver **/ -int i40iw_uquery_device (struct ibv_context *context,struct ibv_device_attr *attr) +int i40iw_uquery_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t i40iw_fw_ver;int ret;unsigned int minor, major; - ret =ibv_cmd_query_device (context, attr, &i40iw_fw_ver, &cmd,sizeof (cmd)); -if (ret) { -fprintf (stderr,PFX "%s: query device failed and returned status code: %d\n" , __func__, ret); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size); +if (ret)return ret; - } + i40iw_fw_ver = resp.base.fw_ver; major = (i40iw_fw_ver >>16 ) &0xffff ; minor = i40iw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof (attr->fw_ver),"%d.%d" , major, minor); +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver), +"%d.%d" , major, minor);return 0 ; }
providers/ipathverbs/ipathverbs.c +1 -1
@@ -89,7 +89,7 @@ static const struct verbs_match_ent hca_table[] = {static const struct verbs_context_ops ipath_ctx_common_ops = { .free_context = ipath_free_context, - .query_device = ipath_query_device, + .query_device_ex = ipath_query_device, .query_port = ipath_query_port, .alloc_pd = ipath_alloc_pd,
providers/ipathverbs/ipathverbs.h +3 -2
@@ -173,8 +173,9 @@ static inline struct ipath_rwqe *get_rwqe_ptr(struct ipath_rq *rq, rq->max_sge * sizeof(struct ibv_sge)) * n); } -extern int ipath_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +int ipath_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);extern int ipath_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/ipathverbs/verbs.c +8 -5
@@ -48,23 +48,26 @@#include "ipath-abi.h" int ipath_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;unsigned major, minor, sub_minor;int ret; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, - &cmd,sizeof cmd); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof attr->fw_ver, +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%d" , major, minor, sub_minor);return 0 ;
providers/mthca/mthca.c +1 -1
@@ -92,7 +92,7 @@ static const struct verbs_match_ent hca_table[] = { };static const struct verbs_context_ops mthca_ctx_common_ops = { - .query_device = mthca_query_device, + .query_device_ex = mthca_query_device, .query_port = mthca_query_port, .alloc_pd = mthca_alloc_pd, .dealloc_pd = mthca_free_pd,
providers/mthca/mthca.h +2 -1
@@ -273,7 +273,8 @@ struct mthca_db_table *mthca_alloc_db_tab(int uarc_size);void mthca_free_db_tab (struct mthca_db_table *db_tab);int mthca_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int mthca_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/mthca/verbs.c +9 -4
@@ -42,22 +42,27 @@#include "mthca.h" #include "mthca-abi.h" -int mthca_query_device (struct ibv_context *context,struct ibv_device_attr *attr) +int mthca_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;unsigned major, minor, sub_minor;int ret; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, &cmd,sizeof cmd); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof attr->fw_ver, +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%d" , major, minor, sub_minor);return 0 ;
providers/ocrdma/ocrdma_main.c +1 -1
@@ -68,7 +68,7 @@ static const struct verbs_match_ent ucna_table[] = { };static const struct verbs_context_ops ocrdma_ctx_ops = { - .query_device = ocrdma_query_device, + .query_device_ex = ocrdma_query_device, .query_port = ocrdma_query_port, .alloc_pd = ocrdma_alloc_pd, .dealloc_pd = ocrdma_free_pd,
providers/ocrdma/ocrdma_main.h +3 -1
@@ -265,7 +265,9 @@ static inline struct ocrdma_ah *get_ocrdma_ah(struct ibv_ah *ibah) }void ocrdma_init_ahid_tbl (struct ocrdma_devctx *ctx); -int ocrdma_query_device (struct ibv_context *,struct ibv_device_attr *); +int ocrdma_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int ocrdma_query_port (struct ibv_context *,uint8_t ,struct ibv_port_attr *);struct ibv_pd *ocrdma_alloc_pd (struct ibv_context *);int ocrdma_free_pd (struct ibv_pd *);
providers/ocrdma/ocrdma_verbs.c +12 -8
@@ -68,17 +68,21 @@ static inline void ocrdma_swap_cpu_to_le(void *dst, uint32_t len)* ocrdma_query_device */ int ocrdma_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; -uint64_t fw_ver;struct ocrdma_device *dev =get_ocrdma_dev (context->device); -int status; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp); +int ret; -bzero (attr,sizeof *attr); - status =ibv_cmd_query_device (context, attr, &fw_ver, &cmd,sizeof cmd); -memcpy (attr->fw_ver, dev->fw_ver,sizeof (dev->fw_ver)); -return status; + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size); +if (ret) +return ret; + +memcpy (attr->orig_attr.fw_ver, dev->fw_ver,sizeof (dev->fw_ver)); +return 0 ; }/*
providers/qedr/qelr_main.c +1 -1
@@ -87,7 +87,7 @@ static const struct verbs_match_ent hca_table[] = { };static const struct verbs_context_ops qelr_ctx_ops = { - .query_device = qelr_query_device, + .query_device_ex = qelr_query_device, .query_port = qelr_query_port, .alloc_pd = qelr_alloc_pd, .dealloc_pd = qelr_dealloc_pd,
providers/qedr/qelr_verbs.c +12 -9
@@ -75,26 +75,29 @@ static inline int qelr_wq_is_full(struct qelr_qp_hwq_info *info) }int qelr_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t fw_ver;unsigned int major, minor, revision, eng; -int status; +int ret; -bzero (attr,sizeof (*attr)); - status =ibv_cmd_query_device (context, attr, &fw_ver, &cmd, -sizeof (cmd)); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size); +if (ret) +return ret; + fw_ver = resp.base.fw_ver; major = (fw_ver >>24 ) &0xff ; minor = (fw_ver >>16 ) &0xff ; revision = (fw_ver >>8 ) &0xff ; eng = fw_ver &0xff ; -snprintf (attr->fw_ver,sizeof (attr->fw_ver), +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%d.%d" , major, minor, revision, eng); - -return status; +return 0 ; } int qelr_query_port(struct ibv_context *context, uint8_t port,
providers/qedr/qelr_verbs.h +2 -1
@@ -41,7 +41,8 @@#include <util/udma_barrier.h> int qelr_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int qelr_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/rxe/rxe.c +9 -6
@@ -65,23 +65,26 @@ static const struct verbs_match_ent hca_table[] = { };static int rxe_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;unsigned int major, minor, sub_minor;int ret; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, - &cmd,sizeof (cmd)); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof (attr->fw_ver), +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%d" , major, minor, sub_minor);return 0 ;
@@ -831,7 +834,7 @@ static int rxe_destroy_ah(struct ibv_ah *ibah) }static const struct verbs_context_ops rxe_ctx_ops = { - .query_device = rxe_query_device, + .query_device_ex = rxe_query_device, .query_port = rxe_query_port, .alloc_pd = rxe_alloc_pd, .dealloc_pd = rxe_dealloc_pd,
providers/siw/siw.c +11 -9
@@ -20,26 +20,28 @@static const int siw_debug;static void siw_free_context (struct ibv_context *ibv_ctx); -static int siw_query_device (struct ibv_context *ctx, -struct ibv_device_attr *attr) +static int siw_query_device (struct ibv_context *context, +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;unsigned int major, minor, sub_minor;int rv; -memset (&cmd,0 ,sizeof (cmd)); - - rv =ibv_cmd_query_device (ctx, attr, &raw_fw_ver, &cmd,sizeof (cmd)); + rv =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (rv)return rv; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof (attr->fw_ver),"%d.%d.%d" , major, minor, - sub_minor); +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver), +"%d.%d.%d" , major, minor, sub_minor);return 0 ; }
@@ -832,7 +834,7 @@ static const struct verbs_context_ops siw_context_ops = { .post_recv = siw_post_recv, .post_send = siw_post_send, .post_srq_recv = siw_post_srq_recv, - .query_device = siw_query_device, + .query_device_ex = siw_query_device, .query_port = siw_query_port, .query_qp = siw_query_qp, .reg_mr = siw_reg_mr,
providers/vmw_pvrdma/pvrdma.h +2 -1
@@ -275,7 +275,8 @@ int pvrdma_alloc_buf(struct pvrdma_buf *buf, size_t size, int page_size);void pvrdma_free_buf (struct pvrdma_buf *buf);int pvrdma_query_device (struct ibv_context *context, -struct ibv_device_attr *attr); +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size);int pvrdma_query_port (struct ibv_context *context,uint8_t port,struct ibv_port_attr *attr);
providers/vmw_pvrdma/pvrdma_main.c +1 -1
@@ -55,7 +55,7 @@ static void pvrdma_free_context(struct ibv_context *ibctx);static const struct verbs_context_ops pvrdma_ctx_ops = { .free_context = pvrdma_free_context, - .query_device = pvrdma_query_device, + .query_device_ex = pvrdma_query_device, .query_port = pvrdma_query_port, .alloc_pd = pvrdma_alloc_pd, .dealloc_pd = pvrdma_free_pd,
providers/vmw_pvrdma/verbs.c +8 -5
@@ -47,23 +47,26 @@#include "pvrdma.h" int pvrdma_query_device (struct ibv_context *context, -struct ibv_device_attr *attr) +const struct ibv_query_device_ex_input *input, +struct ibv_device_attr_ex *attr,size_t attr_size) { -struct ibv_query_device cmd; +struct ib_uverbs_ex_query_device_resp resp; +size_t resp_size =sizeof (resp);uint64_t raw_fw_ver;unsigned major, minor, sub_minor;int ret; - ret =ibv_cmd_query_device (context, attr, &raw_fw_ver, - &cmd,sizeof (cmd)); + ret =ibv_cmd_query_device_any (context, input, attr, attr_size, &resp, + &resp_size);if (ret)return ret; + raw_fw_ver = resp.base.fw_ver; major = (raw_fw_ver >>32 ) &0xffff ; minor = (raw_fw_ver >>16 ) &0xffff ; sub_minor = raw_fw_ver &0xffff ; -snprintf (attr->fw_ver,sizeof (attr->fw_ver), +snprintf (attr->orig_attr.fw_ver,sizeof (attr->orig_attr.fw_ver),"%d.%d.%03d" , major, minor, sub_minor);return 0 ;
Signature · 1 line
2.29.2
Jason Gunthorpe
maintainerPATCH 8/9 verbs: Remove dead code
In reply to Jason GunthorpeRemove the old query_device support code, it is now replaced by ibv_cmd_query_device_any()
--- libibverbs/cmd.c | 99 ------------------------------------ libibverbs/driver.h | 8 --- libibverbs/libibverbs.map.in | 1 - 3 files changed, 108 deletions(-)
libibverbs/cmd.c +0 -99
@@ -44,7 +44,6 @@#include <infiniband/cmd_write.h> #include "ibverbs.h" #include <ccan/minmax.h> -#include <util/util.h> bool verbs_allow_disassociate_destroy;
@@ -113,104 +112,6 @@ int ibv_cmd_query_device(struct ibv_context *context,return 0 ; } -int ibv_cmd_query_device_ex (struct ibv_context *context, -const struct ibv_query_device_ex_input *input, -struct ibv_device_attr_ex *attr,size_t attr_size, -uint64_t *raw_fw_ver, -struct ibv_query_device_ex *cmd, -size_t cmd_size, -struct ib_uverbs_ex_query_device_resp *resp, -size_t resp_size) -{ -int err; - -if (input && input->comp_mask) -return EINVAL ; - -if (attr_size < offsetof(struct ibv_device_attr_ex , comp_mask) + -sizeof (attr->comp_mask)) -return EINVAL ; - - cmd->comp_mask =0 ; - cmd->reserved =0 ; -memset (attr->orig_attr.fw_ver,0 ,sizeof (attr->orig_attr.fw_ver)); -memset (&attr->comp_mask,0 , attr_size -sizeof (attr->orig_attr)); - - err =execute_cmd_write_ex (context,IB_USER_VERBS_EX_CMD_QUERY_DEVICE , - cmd, cmd_size, resp, resp_size); -if (err) -return err; - -copy_query_dev_fields (&attr->orig_attr, &resp->base, raw_fw_ver); -/* Report back supported comp_mask bits. For now no comp_mask bit is -* defined */ - attr->comp_mask = resp->comp_mask &0 ; - -#define CAN_COPY (_ibv_attr, _uverbs_attr) \ - (attr_size >= offsetofend(struct ibv_device_attr_ex, _ibv_attr) && \ - resp->response_length >= \ - offsetofend(struct ib_uverbs_ex_query_device_resp, \ - _uverbs_attr)) - -if (CAN_COPY (odp_caps, odp_caps)) { - attr->odp_caps.general_caps = resp->odp_caps.general_caps; - attr->odp_caps.per_transport_caps.rc_odp_caps = - resp->odp_caps.per_transport_caps.rc_odp_caps; - attr->odp_caps.per_transport_caps.uc_odp_caps = - resp->odp_caps.per_transport_caps.uc_odp_caps; - attr->odp_caps.per_transport_caps.ud_odp_caps = - resp->odp_caps.per_transport_caps.ud_odp_caps; - } - -if (CAN_COPY (completion_timestamp_mask, timestamp_mask)) - attr->completion_timestamp_mask = resp->timestamp_mask; - -if (CAN_COPY (hca_core_clock, hca_core_clock)) - attr->hca_core_clock = resp->hca_core_clock; - -if (CAN_COPY (device_cap_flags_ex, device_cap_flags_ex)) - attr->device_cap_flags_ex = resp->device_cap_flags_ex; - -if (CAN_COPY (rss_caps, rss_caps)) { - attr->rss_caps.supported_qpts = resp->rss_caps.supported_qpts; - attr->rss_caps.max_rwq_indirection_tables = - resp->rss_caps.max_rwq_indirection_tables; - attr->rss_caps.max_rwq_indirection_table_size = - resp->rss_caps.max_rwq_indirection_table_size; - } - -if (CAN_COPY (max_wq_type_rq, max_wq_type_rq)) - attr->max_wq_type_rq = resp->max_wq_type_rq; - -if (CAN_COPY (raw_packet_caps, raw_packet_caps)) - attr->raw_packet_caps = resp->raw_packet_caps; - -if (CAN_COPY (tm_caps, tm_caps)) { - attr->tm_caps.max_rndv_hdr_size = - resp->tm_caps.max_rndv_hdr_size; - attr->tm_caps.max_num_tags = resp->tm_caps.max_num_tags; - attr->tm_caps.flags = resp->tm_caps.flags; - attr->tm_caps.max_ops = resp->tm_caps.max_ops; - attr->tm_caps.max_sge = resp->tm_caps.max_sge; - } - -if (CAN_COPY (cq_mod_caps, cq_moderation_caps)) { - attr->cq_mod_caps.max_cq_count = - resp->cq_moderation_caps.max_cq_moderation_count; - attr->cq_mod_caps.max_cq_period = - resp->cq_moderation_caps.max_cq_moderation_period; - } - -if (CAN_COPY (max_dm_size, max_dm_size)) - attr->max_dm_size = resp->max_dm_size; - -if (CAN_COPY (xrc_odp_caps, xrc_odp_caps)) - attr->xrc_odp_caps = resp->xrc_odp_caps; -#undef CAN_COPY - -return 0 ; -} -int ibv_cmd_alloc_pd (struct ibv_context *context,struct ibv_pd *pd,struct ibv_alloc_pd *cmd,size_t cmd_size,struct ib_uverbs_alloc_pd_resp *resp,size_t resp_size)
libibverbs/driver.h +0 -8
@@ -465,14 +465,6 @@ int ibv_cmd_query_device_any(struct ibv_context *context, struct ibv_device_attr_ex *attr, size_t attr_size, struct ib_uverbs_ex_query_device_resp *resp, size_t *resp_size); -int ibv_cmd_query_device_ex (struct ibv_context *context, -const struct ibv_query_device_ex_input *input, -struct ibv_device_attr_ex *attr,size_t attr_size, -uint64_t *raw_fw_ver, -struct ibv_query_device_ex *cmd, -size_t cmd_size, -struct ib_uverbs_ex_query_device_resp *resp, -size_t resp_size);int ibv_cmd_query_port (struct ibv_context *context,uint8_t port_num,struct ibv_port_attr *port_attr,struct ibv_query_port *cmd,size_t cmd_size);
libibverbs/libibverbs.map.in +0 -1
@@ -205,7 +205,6 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ { ibv_cmd_query_context; ibv_cmd_query_device; ibv_cmd_query_device_any; - ibv_cmd_query_device_ex; ibv_cmd_query_mr; ibv_cmd_query_port; ibv_cmd_query_qp;
Signature · 1 line
2.29.2
Re: [PATCH 8/9] verbs: Remove dead code
Quoted 4 lines · Jason Gunthorpe
Remove the old
query_devicesupport code, it is now replaced byibv_cmd_query_device_any()Signed-off-by Jason Gunthorpe <[email protected]>
Shouldn't the legacy fallback in ibv_query_device_ex() be removed as well?
/**
* ibv_query_device_ex - Get extended device properties */
static inline int
ibv_query_device_ex(struct ibv_context *context,
const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr)
{
struct verbs_context *vctx;
int ret;
if (input && input->comp_mask)
return EINVAL;
vctx = verbs_get_ctx_op(context, query_device_ex);
if (!vctx)
goto legacy;
ret = vctx->query_device_ex(context, input, attr, sizeof(*attr));
if (ret == EOPNOTSUPP || ret == ENOSYS)
goto legacy;
return ret;legacy:
memset(attr, 0, sizeof(*attr)); ret = ibv_query_device(context, &attr->orig_attr); return ret; }
Jason Gunthorpe
maintainerRe: [PATCH 8/9] verbs: Remove dead code
Quoted 7 lines · Gal Pressman
Quoted 4 lines · Jason Gunthorpe
Remove the old
query_devicesupport code, it is now replaced byibv_cmd_query_device_any()Signed-off-by Jason Gunthorpe <[email protected]>Shouldn't the legacy fallback in
ibv_query_device_ex()be removed as well?
I thought about it, and yes we could convert that into a real function call and remove all the junk
But if we do that it causes a symbol ver dependencies and if we are going that far then we should really just do all of the verbs_get_ctx_op() inlines and be done with that mess entirely.
This would mean anything linked with a new rdma-core would not be able to load on any of the older ones..
I've been saving that gem for some future adventure since it seems to cause some general pain. It would be good to couple it with some other verbs.h clean up as well
Jason
Jason Gunthorpe
maintainerPATCH 9/9 verbs: Delete query_device() internal support
In reply to Jason GunthorpeNow that all providers only implement the _ex API have the external API call query_device_ex() directly and remove everything to do with the internal query_device op.
--- libibverbs/cmd.c | 65 ------------------------------------------ libibverbs/device.c | 1 + libibverbs/driver.h | 6 ---- libibverbs/dummy_ops.c | 28 +----------------- libibverbs/verbs.c | 5 +++- libibverbs/verbs.h | 3 +- providers/cxgb4/dev.c | 8 +++--- 7 files changed, 12 insertions(+), 104 deletions(-)
libibverbs/cmd.c +0 -65
@@ -47,71 +47,6 @@bool verbs_allow_disassociate_destroy; -static void copy_query_dev_fields (struct ibv_device_attr *device_attr, -struct ib_uverbs_query_device_resp *resp, -uint64_t *raw_fw_ver) -{ - *raw_fw_ver = resp->fw_ver; - device_attr->node_guid = resp->node_guid; - device_attr->sys_image_guid = resp->sys_image_guid; - device_attr->max_mr_size = resp->max_mr_size; - device_attr->page_size_cap = resp->page_size_cap; - device_attr->vendor_id = resp->vendor_id; - device_attr->vendor_part_id = resp->vendor_part_id; - device_attr->hw_ver = resp->hw_ver; - device_attr->max_qp = resp->max_qp; - device_attr->max_qp_wr = resp->max_qp_wr; - device_attr->device_cap_flags = resp->device_cap_flags; - device_attr->max_sge = resp->max_sge; - device_attr->max_sge_rd = resp->max_sge_rd; - device_attr->max_cq = resp->max_cq; - device_attr->max_cqe = resp->max_cqe; - device_attr->max_mr = resp->max_mr; - device_attr->max_pd = resp->max_pd; - device_attr->max_qp_rd_atom = resp->max_qp_rd_atom; - device_attr->max_ee_rd_atom = resp->max_ee_rd_atom; - device_attr->max_res_rd_atom = resp->max_res_rd_atom; - device_attr->max_qp_init_rd_atom = resp->max_qp_init_rd_atom; - device_attr->max_ee_init_rd_atom = resp->max_ee_init_rd_atom; - device_attr->atomic_cap = resp->atomic_cap; - device_attr->max_ee = resp->max_ee; - device_attr->max_rdd = resp->max_rdd; - device_attr->max_mw = resp->max_mw; - device_attr->max_raw_ipv6_qp = resp->max_raw_ipv6_qp; - device_attr->max_raw_ethy_qp = resp->max_raw_ethy_qp; - device_attr->max_mcast_grp = resp->max_mcast_grp; - device_attr->max_mcast_qp_attach = resp->max_mcast_qp_attach; - device_attr->max_total_mcast_qp_attach = resp->max_total_mcast_qp_attach; - device_attr->max_ah = resp->max_ah; - device_attr->max_fmr = resp->max_fmr; - device_attr->max_map_per_fmr = resp->max_map_per_fmr; - device_attr->max_srq = resp->max_srq; - device_attr->max_srq_wr = resp->max_srq_wr; - device_attr->max_srq_sge = resp->max_srq_sge; - device_attr->max_pkeys = resp->max_pkeys; - device_attr->local_ca_ack_delay = resp->local_ca_ack_delay; - device_attr->phys_port_cnt = resp->phys_port_cnt; -} - -int ibv_cmd_query_device (struct ibv_context *context, -struct ibv_device_attr *device_attr, -uint64_t *raw_fw_ver, -struct ibv_query_device *cmd,size_t cmd_size) -{ -struct ib_uverbs_query_device_resp resp; -int ret; - - ret =execute_cmd_write (context,IB_USER_VERBS_CMD_QUERY_DEVICE , cmd, - cmd_size, &resp,sizeof (resp)); -if (ret) -return ret; - -memset (device_attr->fw_ver,0 ,sizeof device_attr->fw_ver); -copy_query_dev_fields (device_attr, &resp, raw_fw_ver); - -return 0 ; -} -int ibv_cmd_alloc_pd (struct ibv_context *context,struct ibv_pd *pd,struct ibv_alloc_pd *cmd,size_t cmd_size,struct ib_uverbs_alloc_pd_resp *resp,size_t resp_size)
libibverbs/device.c +1 -0
@@ -320,6 +320,7 @@ static void set_lib_ops(struct verbs_context *vctx)#undef ibv_query_port vctx->context.ops._compat_query_port = ibv_query_port; vctx->query_port = __lib_query_port; + vctx->context.ops._compat_query_device = ibv_query_device;/* * In order to maintain backward/forward binary compatibility
libibverbs/driver.h +0 -6
@@ -354,8 +354,6 @@ struct verbs_context_ops { struct ibv_ops_wr **bad_op);int (*post_srq_recv)(struct ibv_srq *srq,struct ibv_recv_wr *recv_wr,struct ibv_recv_wr **bad_recv_wr); -int (*query_device)(struct ibv_context *context, -struct ibv_device_attr *device_attr); int (*query_device_ex)(struct ibv_context *context, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr,
@@ -449,10 +447,6 @@ int ibv_cmd_get_context(struct verbs_context *context, struct ib_uverbs_get_context_resp *resp, size_t resp_size);int ibv_cmd_query_context (struct ibv_context *ctx,struct ibv_command_buffer *driver); -int ibv_cmd_query_device (struct ibv_context *context, -struct ibv_device_attr *device_attr, -uint64_t *raw_fw_ver, -struct ibv_query_device *cmd,size_t cmd_size); int ibv_cmd_create_flow_action_esp(struct ibv_context *ctx, struct ibv_flow_action_esp_attr *attr, struct verbs_flow_action *flow_action,
libibverbs/dummy_ops.c +1 -27
@@ -377,35 +377,11 @@ static int post_srq_recv(struct ibv_srq *srq, struct ibv_recv_wr *recv_wr,return EOPNOTSUPP ; } -static int query_device (struct ibv_context *context, -struct ibv_device_attr *device_attr) -{ -const struct verbs_context_ops *ops =get_ops (context); - -if (!ops->query_device_ex) -return EOPNOTSUPP ; -return ops->query_device_ex ( - context,NULL , - container_of(device_attr, struct ibv_device_attr_ex, orig_attr), -sizeof (*device_attr)); -} - -/* Provide a generic implementation for all providers that don't implement -* query_device_ex. -*/ static int query_device_ex (struct ibv_context *context,const struct ibv_query_device_ex_input *input,struct ibv_device_attr_ex *attr,size_t attr_size) { -if (input && input->comp_mask) -return EINVAL ; - -if (attr_size <sizeof (attr->orig_attr)) -return EOPNOTSUPP ; - -memset (attr,0 , attr_size); - -return ibv_query_device (context, &attr->orig_attr); +return EOPNOTSUPP ; }static int query_ece (struct ibv_qp *qp,struct ibv_ece *ece)
@@ -558,7 +534,6 @@ const struct verbs_context_ops verbs_dummy_ops = { post_send, post_srq_ops, post_srq_recv, - query_device, query_device_ex, query_ece, query_port,
@@ -680,7 +655,6 @@ void verbs_set_ops(struct verbs_context *vctx,SET_OP (ctx, post_send);SET_OP (vctx, post_srq_ops);SET_OP (ctx, post_srq_recv); -SET_PRIV_OP (ctx, query_device);SET_OP (vctx, query_device_ex);SET_PRIV_OP_IC (vctx, query_ece);SET_PRIV_OP_IC (ctx, query_port);
libibverbs/verbs.c +4 -1
@@ -156,7 +156,10 @@ LATEST_SYMVER_FUNC(ibv_query_device, 1_1, "IBVERBS_1.1", struct ibv_context *context, struct ibv_device_attr *device_attr) { - return get_ops(context)->query_device(context, device_attr); + return get_ops(context)->query_device_ex( + context, NULL, + container_of(device_attr, struct ibv_device_attr_ex, orig_attr), + sizeof(*device_attr)); } int __lib_query_port(struct ibv_context *context, uint8_t port_num,
libibverbs/verbs.h +2 -1
@@ -1922,7 +1922,8 @@ struct ibv_device {struct _compat_ibv_port_attr ;struct ibv_context_ops { -void *(*_compat_query_device)(void ); +int (*_compat_query_device)(struct ibv_context *context, +struct ibv_device_attr *device_attr);int (*_compat_query_port)(struct ibv_context *context,uint8_t port_num,struct _compat_ibv_port_attr *port_attr);
providers/cxgb4/dev.c +4 -4
@@ -114,8 +114,6 @@ static struct verbs_context *c4iw_alloc_context(struct ibv_device *ibdev,struct ibv_get_context cmd;struct uc4iw_alloc_ucontext_resp resp;struct c4iw_dev *rhp =to_c4iw_dev (ibdev); -struct ibv_query_device qcmd; -uint64_t raw_fw_ver;struct ibv_device_attr attr; context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
@@ -143,8 +141,10 @@ static struct verbs_context *c4iw_alloc_context(struct ibv_device *ibdev, }verbs_set_ops (&context->ibv_ctx, &c4iw_ctx_common_ops); -if (ibv_cmd_query_device (&context->ibv_ctx.context, &attr, - &raw_fw_ver, &qcmd,sizeof (qcmd))) +if (c4iw_query_device (&context->ibv_ctx.context,NULL , + container_of(&attr, struct ibv_device_attr_ex, + orig_attr), +sizeof (attr))) goto err_unmap;if (!rhp->mmid2ptr) {
Signature · 1 line
2.29.2