mlx5 QP cleanup (cont.)
Subsystems (2)
cover mlx5 QP cleanup (cont.)
From: Leon Romanovsky <[email protected]>
Hi,
This short series continue to cleanup qp.c file, which grew to be completely unmaintainable.
Thanks
Leon Romanovsky (2):
RDMA/mlx5: Update mlx5_ib to use new cmd interface RDMA/mlx5: Move all WR logic from qp.c to separate file
Max Gurtovoy (1):
RDMA/mlx5: Refactor mlx5_post_send() to improve readability
drivers/infiniband/hw/mlx5/Makefile | 3 +- drivers/infiniband/hw/mlx5/cmd.c | 114 +- drivers/infiniband/hw/mlx5/cmd.h | 4 +- drivers/infiniband/hw/mlx5/cong.c | 4 +- drivers/infiniband/hw/mlx5/main.c | 10 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 4 - drivers/infiniband/hw/mlx5/odp.c | 5 +- drivers/infiniband/hw/mlx5/qp.c | 1488 +------------------------ drivers/infiniband/hw/mlx5/srq_cmd.c | 115 +- drivers/infiniband/hw/mlx5/wr.c | 1504 ++++++++++++++++++++++++++ drivers/infiniband/hw/mlx5/wr.h | 76 ++ 11 files changed, 1680 insertions(+), 1647 deletions(-) create mode 100644 drivers/infiniband/hw/mlx5/wr.c create mode 100644 drivers/infiniband/hw/mlx5/wr.h
--
2.26.2
PATCH 1/3 RDMA/mlx5: Update mlx5_ib to use new cmd interface
From: Leon Romanovsky <[email protected]>
Reuse newly introduced mlx5_cmd_exec_in() and mlx5_cmd_exec_inout() to reduce code duplication in mlx5_ib module.
--- drivers/infiniband/hw/mlx5/cmd.c | 114 ++++++++------------------ drivers/infiniband/hw/mlx5/cmd.h | 4 +- drivers/infiniband/hw/mlx5/cong.c | 4 +- drivers/infiniband/hw/mlx5/main.c | 5 +- drivers/infiniband/hw/mlx5/odp.c | 5 +- drivers/infiniband/hw/mlx5/srq_cmd.c | 115 ++++++++++++--------------- 6 files changed, 91 insertions(+), 156 deletions(-)
drivers/infiniband/hw/mlx5/cmd.c +35 -79
@@ -1,46 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* -* Copyright (c) 2017, Mellanox Technologies. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenIB.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. +* Copyright (c) 2017-2020, Mellanox Technologies inc. All rights reserved. */ #include "cmd.h" int mlx5_cmd_dump_fill_mkey (struct mlx5_core_dev *dev,u32 *mkey) { -u32 out[MLX5_ST_SZ_DW (query_special_contexts_out)] = {0 }; -u32 in[MLX5_ST_SZ_DW (query_special_contexts_in)] = {0 }; +u32 out[MLX5_ST_SZ_DW (query_special_contexts_out)] = {}; +u32 in[MLX5_ST_SZ_DW (query_special_contexts_in)] = {};int err;MLX5_SET (query_special_contexts_in, in, opcode,MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS ); - err =mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); + err =mlx5_cmd_exec_inout (dev, query_special_contexts, in, out);if (!err) *mkey =MLX5_GET (query_special_contexts_out, out, dump_fill_mkey);
@@ -50,12 +23,12 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey)int mlx5_cmd_null_mkey (struct mlx5_core_dev *dev,u32 *null_mkey) {u32 out[MLX5_ST_SZ_DW (query_special_contexts_out)] = {}; -u32 in[MLX5_ST_SZ_DW (query_special_contexts_in)] = {}; +u32 in[MLX5_ST_SZ_DW (query_special_contexts_in)] = {};int err;MLX5_SET (query_special_contexts_in, in, opcode,MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS ); - err =mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); + err =mlx5_cmd_exec_inout (dev, query_special_contexts, in, out);if (!err) *null_mkey =MLX5_GET (query_special_contexts_out, out, null_mkey);
@@ -63,23 +36,15 @@ int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey) }int mlx5_cmd_query_cong_params (struct mlx5_core_dev *dev,int cong_point, -void *out,int out_size) +void *out) { -u32 in[MLX5_ST_SZ_DW (query_cong_params_in)] = { }; +u32 in[MLX5_ST_SZ_DW (query_cong_params_in)] = {};MLX5_SET (query_cong_params_in, in, opcode,MLX5_CMD_OP_QUERY_CONG_PARAMS );MLX5_SET (query_cong_params_in, in, cong_protocol, cong_point); -return mlx5_cmd_exec (dev, in,sizeof (in), out, out_size); -} - -int mlx5_cmd_modify_cong_params (struct mlx5_core_dev *dev, -void *in,int in_size) -{ -u32 out[MLX5_ST_SZ_DW (modify_cong_params_out)] = { }; - -return mlx5_cmd_exec (dev, in, in_size, out,sizeof (out)); +return mlx5_cmd_exec_inout (dev, query_cong_params, in, out); } int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
@@ -133,7 +98,7 @@ int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,MLX5_SET64 (alloc_memic_in, in, range_start_addr, hw_start_addr + (page_idx *PAGE_SIZE )); - ret =mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); + ret =mlx5_cmd_exec_inout (dev, alloc_memic, in, out);if (ret) {spin_lock (&dm->lock); bitmap_clear(dm->memic_alloc_pages,
@@ -162,8 +127,7 @@ void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)struct mlx5_core_dev *dev = dm->dev;u64 hw_start_addr =MLX5_CAP64_DEV_MEM (dev, memic_bar_start_addr);u32 num_pages =DIV_ROUND_UP (length,PAGE_SIZE ); -u32 out[MLX5_ST_SZ_DW (dealloc_memic_out)] = {0 }; -u32 in[MLX5_ST_SZ_DW (dealloc_memic_in)] = {0 }; +u32 in[MLX5_ST_SZ_DW (dealloc_memic_in)] = {};u64 start_page_idx;int err;
@@ -174,7 +138,7 @@ void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)MLX5_SET64 (dealloc_memic_in, in, memic_start_addr, addr);MLX5_SET (dealloc_memic_in, in, memic_size, length); - err =mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); + err =mlx5_cmd_exec_in (dev, dealloc_memic, in);if (err)return ;
@@ -198,49 +162,46 @@ int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out)void mlx5_cmd_destroy_tir (struct mlx5_core_dev *dev,u32 tirn,u16 uid) { -u32 in[MLX5_ST_SZ_DW (destroy_tir_in)] = {}; -u32 out[MLX5_ST_SZ_DW (destroy_tir_out)] = {}; +u32 in[MLX5_ST_SZ_DW (destroy_tir_in)] = {};MLX5_SET (destroy_tir_in, in, opcode,MLX5_CMD_OP_DESTROY_TIR );MLX5_SET (destroy_tir_in, in, tirn, tirn);MLX5_SET (destroy_tir_in, in, uid, uid); -mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +mlx5_cmd_exec_in (dev, destroy_tir, in); }void mlx5_cmd_destroy_tis (struct mlx5_core_dev *dev,u32 tisn,u16 uid) { -u32 in[MLX5_ST_SZ_DW (destroy_tis_in)] = {0 }; -u32 out[MLX5_ST_SZ_DW (destroy_tis_out)] = {0 }; +u32 in[MLX5_ST_SZ_DW (destroy_tis_in)] = {};MLX5_SET (destroy_tis_in, in, opcode,MLX5_CMD_OP_DESTROY_TIS );MLX5_SET (destroy_tis_in, in, tisn, tisn);MLX5_SET (destroy_tis_in, in, uid, uid); -mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +mlx5_cmd_exec_in (dev, destroy_tis, in); }void mlx5_cmd_destroy_rqt (struct mlx5_core_dev *dev,u32 rqtn,u16 uid) { -u32 in[MLX5_ST_SZ_DW (destroy_rqt_in)] = {}; -u32 out[MLX5_ST_SZ_DW (destroy_rqt_out)] = {}; +u32 in[MLX5_ST_SZ_DW (destroy_rqt_in)] = {};MLX5_SET (destroy_rqt_in, in, opcode,MLX5_CMD_OP_DESTROY_RQT );MLX5_SET (destroy_rqt_in, in, rqtn, rqtn);MLX5_SET (destroy_rqt_in, in, uid, uid); -mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +mlx5_cmd_exec_in (dev, destroy_rqt, in); }int mlx5_cmd_alloc_transport_domain (struct mlx5_core_dev *dev,u32 *tdn,u16 uid) { -u32 in[MLX5_ST_SZ_DW (alloc_transport_domain_in)] = {0 }; -u32 out[MLX5_ST_SZ_DW (alloc_transport_domain_out)] = {0 }; +u32 in[MLX5_ST_SZ_DW (alloc_transport_domain_in)] = {}; +u32 out[MLX5_ST_SZ_DW (alloc_transport_domain_out)] = {};int err;MLX5_SET (alloc_transport_domain_in, in, opcode,MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN );MLX5_SET (alloc_transport_domain_in, in, uid, uid); - err =mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); + err =mlx5_cmd_exec_inout (dev, alloc_transport_domain, in, out);if (!err) *tdn =MLX5_GET (alloc_transport_domain_out, out, transport_domain);
@@ -251,32 +212,29 @@ int mlx5_cmd_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn,void mlx5_cmd_dealloc_transport_domain (struct mlx5_core_dev *dev,u32 tdn,u16 uid) { -u32 in[MLX5_ST_SZ_DW (dealloc_transport_domain_in)] = {0 }; -u32 out[MLX5_ST_SZ_DW (dealloc_transport_domain_out)] = {0 }; +u32 in[MLX5_ST_SZ_DW (dealloc_transport_domain_in)] = {};MLX5_SET (dealloc_transport_domain_in, in, opcode,MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN );MLX5_SET (dealloc_transport_domain_in, in, uid, uid);MLX5_SET (dealloc_transport_domain_in, in, transport_domain, tdn); -mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +mlx5_cmd_exec_in (dev, dealloc_transport_domain, in); }void mlx5_cmd_dealloc_pd (struct mlx5_core_dev *dev,u32 pdn,u16 uid) { -u32 out[MLX5_ST_SZ_DW (dealloc_pd_out)] = {}; -u32 in[MLX5_ST_SZ_DW (dealloc_pd_in)] = {}; +u32 in[MLX5_ST_SZ_DW (dealloc_pd_in)] = {};MLX5_SET (dealloc_pd_in, in, opcode,MLX5_CMD_OP_DEALLOC_PD );MLX5_SET (dealloc_pd_in, in, pd, pdn);MLX5_SET (dealloc_pd_in, in, uid, uid); -mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +mlx5_cmd_exec_in (dev, dealloc_pd, in); }int mlx5_cmd_attach_mcg (struct mlx5_core_dev *dev,union ib_gid *mgid,u32 qpn,u16 uid) { -u32 out[MLX5_ST_SZ_DW (attach_to_mcg_out)] = {}; -u32 in[MLX5_ST_SZ_DW (attach_to_mcg_in)] = {}; +u32 in[MLX5_ST_SZ_DW (attach_to_mcg_in)] = {};void *gid;MLX5_SET (attach_to_mcg_in, in, opcode,MLX5_CMD_OP_ATTACH_TO_MCG );
@@ -284,14 +242,13 @@ int mlx5_cmd_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,MLX5_SET (attach_to_mcg_in, in, uid, uid); gid =MLX5_ADDR_OF (attach_to_mcg_in, in, multicast_gid);memcpy (gid, mgid,sizeof (*mgid)); -return mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +return mlx5_cmd_exec_in (dev, attach_to_mcg, in); }int mlx5_cmd_detach_mcg (struct mlx5_core_dev *dev,union ib_gid *mgid,u32 qpn,u16 uid) { -u32 out[MLX5_ST_SZ_DW (detach_from_mcg_out)] = {}; -u32 in[MLX5_ST_SZ_DW (detach_from_mcg_in)] = {}; +u32 in[MLX5_ST_SZ_DW (detach_from_mcg_in)] = {};void *gid;MLX5_SET (detach_from_mcg_in, in, opcode,MLX5_CMD_OP_DETACH_FROM_MCG );
@@ -299,18 +256,18 @@ int mlx5_cmd_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid,MLX5_SET (detach_from_mcg_in, in, uid, uid); gid =MLX5_ADDR_OF (detach_from_mcg_in, in, multicast_gid);memcpy (gid, mgid,sizeof (*mgid)); -return mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +return mlx5_cmd_exec_in (dev, detach_from_mcg, in); }int mlx5_cmd_xrcd_alloc (struct mlx5_core_dev *dev,u32 *xrcdn,u16 uid) {u32 out[MLX5_ST_SZ_DW (alloc_xrcd_out)] = {}; -u32 in[MLX5_ST_SZ_DW (alloc_xrcd_in)] = {}; +u32 in[MLX5_ST_SZ_DW (alloc_xrcd_in)] = {};int err;MLX5_SET (alloc_xrcd_in, in, opcode,MLX5_CMD_OP_ALLOC_XRCD );MLX5_SET (alloc_xrcd_in, in, uid, uid); - err =mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); + err =mlx5_cmd_exec_inout (dev, alloc_xrcd, in, out);if (!err) *xrcdn =MLX5_GET (alloc_xrcd_out, out, xrcd);return err;
@@ -318,13 +275,12 @@ int mlx5_cmd_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn, u16 uid)int mlx5_cmd_xrcd_dealloc (struct mlx5_core_dev *dev,u32 xrcdn,u16 uid) { -u32 out[MLX5_ST_SZ_DW (dealloc_xrcd_out)] = {}; -u32 in[MLX5_ST_SZ_DW (dealloc_xrcd_in)] = {}; +u32 in[MLX5_ST_SZ_DW (dealloc_xrcd_in)] = {};MLX5_SET (dealloc_xrcd_in, in, opcode,MLX5_CMD_OP_DEALLOC_XRCD );MLX5_SET (dealloc_xrcd_in, in, xrcd, xrcdn);MLX5_SET (dealloc_xrcd_in, in, uid, uid); -return mlx5_cmd_exec (dev, in,sizeof (in), out,sizeof (out)); +return mlx5_cmd_exec_in (dev, dealloc_xrcd, in); } int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
@@ -350,7 +306,7 @@ int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb, data =MLX5_ADDR_OF (mad_ifc_in, in, mad);memcpy (data, inb,MLX5_FLD_SZ_BYTES (mad_ifc_in, mad)); - err =mlx5_cmd_exec (dev, in, inlen, out, outlen); + err =mlx5_cmd_exec_inout (dev, mad_ifc, in, out);if (err) goto out;
drivers/infiniband/hw/mlx5/cmd.h +1 -3
@@ -40,10 +40,8 @@int mlx5_cmd_dump_fill_mkey (struct mlx5_core_dev *dev,u32 *mkey);int mlx5_cmd_null_mkey (struct mlx5_core_dev *dev,u32 *null_mkey);int mlx5_cmd_query_cong_params (struct mlx5_core_dev *dev,int cong_point, -void *out,int out_size); +void *out);int mlx5_cmd_query_ext_ppcnt_counters (struct mlx5_core_dev *dev,void *out); -int mlx5_cmd_modify_cong_params (struct mlx5_core_dev *mdev, -void *in,int in_size);int mlx5_cmd_alloc_memic (struct mlx5_dm *dm,phys_addr_t *addr,u64 length,u32 alignment);void mlx5_cmd_dealloc_memic (struct mlx5_dm *dm,phys_addr_t addr,u64 length);
drivers/infiniband/hw/mlx5/cong.c +2 -2
@@ -290,7 +290,7 @@ static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u8 port_num, node =mlx5_ib_param_to_node (offset); - err =mlx5_cmd_query_cong_params (mdev, node, out, outlen); + err =mlx5_cmd_query_cong_params (mdev, node, out);if (err) goto free;
@@ -339,7 +339,7 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u8 port_num,MLX5_SET (field_select_r_roce_rp, field, field_select_r_roce_rp, attr_mask); - err =mlx5_cmd_modify_cong_params (mdev, in, inlen); + err =mlx5_cmd_exec_in (dev->mdev, modify_cong_params, in);kvfree (in); alloc_err:mlx5_ib_put_native_port_mdev (dev, port_num +1 );
drivers/infiniband/hw/mlx5/main.c +2 -3
@@ -2562,7 +2562,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)struct mlx5_ib_alloc_pd_resp resp;int err;u32 out[MLX5_ST_SZ_DW (alloc_pd_out)] = {}; -u32 in[MLX5_ST_SZ_DW (alloc_pd_in)] = {}; +u32 in[MLX5_ST_SZ_DW (alloc_pd_in)] = {};u16 uid =0 ;struct mlx5_ib_ucontext *context =rdma_udata_to_drv_context ( udata, struct mlx5_ib_ucontext, ibucontext);
@@ -2570,8 +2570,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) uid = context ? context->devx_uid :0 ;MLX5_SET (alloc_pd_in, in, opcode,MLX5_CMD_OP_ALLOC_PD );MLX5_SET (alloc_pd_in, in, uid, uid); - err =mlx5_cmd_exec (to_mdev (ibdev)->mdev, in,sizeof (in), - out,sizeof (out)); + err =mlx5_cmd_exec_inout (to_mdev (ibdev)->mdev, alloc_pd, in, out);if (err)return err;
drivers/infiniband/hw/mlx5/odp.c +2 -3
@@ -447,8 +447,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, {int wq_num = pfault->event_subtype ==MLX5_PFAULT_SUBTYPE_WQE ? pfault->wqe.wq_num : pfault->token; -u32 out[MLX5_ST_SZ_DW (page_fault_resume_out)] = { }; -u32 in[MLX5_ST_SZ_DW (page_fault_resume_in)] = { }; +u32 in[MLX5_ST_SZ_DW (page_fault_resume_in)] = {};int err;MLX5_SET (page_fault_resume_in, in, opcode,MLX5_CMD_OP_PAGE_FAULT_RESUME );
@@ -457,7 +456,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,MLX5_SET (page_fault_resume_in, in, wq_number, wq_num);MLX5_SET (page_fault_resume_in, in, error, !!error); - err =mlx5_cmd_exec (dev->mdev, in,sizeof (in), out,sizeof (out)); + err =mlx5_cmd_exec_in (dev->mdev, page_fault_resume, in);if (err)mlx5_ib_err (dev,"Failed to resolve the page fault on WQ 0x%x err %d\n" , wq_num, err);
drivers/infiniband/hw/mlx5/srq_cmd.c +49 -66
230 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH 2/3 RDMA/mlx5: Refactor mlx5_post_send() to improve readability
In reply to Leon RomanovskyFrom: Max Gurtovoy <[email protected]>
Add small helpers in order to avoid code duplication and improve code readability. Decrease the amount of code in the gigantic post_send function and divide it to readable methods that will help in code maintenance in the future.
--- drivers/infiniband/hw/mlx5/qp.c | 490 ++++++++++++++++++-------------- 1 file changed, 276 insertions(+), 214 deletions(-)
drivers/infiniband/hw/mlx5/qp.c +276 -214
@@ -5234,18 +5234,279 @@ static void finish_wqe(struct mlx5_ib_qp *qp, cur_edge; } +static void handle_rdma_op (const struct ib_send_wr *wr,void **seg,int *size) +{ +set_raddr_seg (*seg,rdma_wr (wr)->remote_addr,rdma_wr (wr)->rkey); + *seg +=sizeof (struct mlx5_wqe_raddr_seg ); + *size +=sizeof (struct mlx5_wqe_raddr_seg ) /16 ; +} + +static void handle_local_inv (struct mlx5_ib_qp *qp,const struct ib_send_wr *wr, +struct mlx5_wqe_ctrl_seg **ctrl,void **seg, +int *size,void **cur_edge,unsigned int idx) +{ + qp->sq.wr_data[idx] =IB_WR_LOCAL_INV ; + (*ctrl)->imm =cpu_to_be32 (wr->ex.invalidate_rkey); +set_linv_wr (qp, seg, size, cur_edge); +} + +static int handle_reg_mr (struct mlx5_ib_qp *qp,const struct ib_send_wr *wr, +struct mlx5_wqe_ctrl_seg **ctrl,void **seg,int *size, +void **cur_edge,unsigned int idx) +{ + qp->sq.wr_data[idx] =IB_WR_REG_MR ; + (*ctrl)->imm =cpu_to_be32 (reg_wr (wr)->key); +return set_reg_wr (qp,reg_wr (wr), seg, size, cur_edge, true); +} + +static int handle_psv (struct mlx5_ib_dev *dev,struct mlx5_ib_qp *qp, +const struct ib_send_wr *wr, +struct mlx5_wqe_ctrl_seg **ctrl,void **seg,int *size, +void **cur_edge,unsigned int *idx,int nreq, +struct ib_sig_domain *domain,u32 psv_index, +u8 next_fence) +{ +int err; + +/* +* SET_PSV WQEs are not signaled and solicited on error. +*/ + err =__begin_wqe (qp, seg, ctrl, wr, idx, size, cur_edge, nreq, + false, true); +if (unlikely (err)) { +mlx5_ib_warn (dev,"\n" ); + err = -ENOMEM ; + goto out; + } + err =set_psv_wr (domain, psv_index, seg, size); +if (unlikely (err)) { +mlx5_ib_warn (dev,"\n" ); + goto out; + } +finish_wqe (qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq, + next_fence,MLX5_OPCODE_SET_PSV ); + +out: +return err; +} + +static int handle_reg_mr_integrity (struct mlx5_ib_dev *dev, +struct mlx5_ib_qp *qp,const struct ib_send_wr *wr, +struct mlx5_wqe_ctrl_seg **ctrl,void **seg,int *size, +void **cur_edge,unsigned int *idx,int nreq,u8 fence, +u8 next_fence) +{ +struct mlx5_ib_mr *mr; +struct mlx5_ib_mr *pi_mr; +struct mlx5_ib_mr pa_pi_mr; +struct ib_sig_attrs *sig_attrs; +struct ib_reg_wr reg_pi_wr; +int err; + + qp->sq.wr_data[*idx] =IB_WR_REG_MR_INTEGRITY ; + + mr =to_mmr (reg_wr (wr)->mr); + pi_mr = mr->pi_mr; + +if (pi_mr) { +memset (®_pi_wr,0 , +sizeof (struct ib_reg_wr )); + + reg_pi_wr.mr = &pi_mr->ibmr; + reg_pi_wr.access =reg_wr (wr)->access; + reg_pi_wr.key = pi_mr->ibmr.rkey; + + (*ctrl)->imm =cpu_to_be32 (reg_pi_wr.key); +/* UMR for data + prot registration */ + err =set_reg_wr (qp, ®_pi_wr, seg, size, cur_edge, false); +if (unlikely (err)) + goto out; + +finish_wqe (qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, + nreq, fence,MLX5_OPCODE_UMR ); + + err =begin_wqe (qp, seg, ctrl, wr, idx, size, cur_edge, nreq); +if (unlikely (err)) { +mlx5_ib_warn (dev,"\n" ); + err = -ENOMEM ; + goto out; + } + }else { +memset (&pa_pi_mr,0 ,sizeof (struct mlx5_ib_mr )); +/* No UMR, use local_dma_lkey */ + pa_pi_mr.ibmr.lkey = mr->ibmr.pd->local_dma_lkey; + pa_pi_mr.ndescs = mr->ndescs; + pa_pi_mr.data_length = mr->data_length; + pa_pi_mr.data_iova = mr->data_iova; +if (mr->meta_ndescs) { + pa_pi_mr.meta_ndescs = mr->meta_ndescs; + pa_pi_mr.meta_length = mr->meta_length; + pa_pi_mr.pi_iova = mr->pi_iova; + } + + pa_pi_mr.ibmr.length = mr->ibmr.length; + mr->pi_mr = &pa_pi_mr; + } + (*ctrl)->imm =cpu_to_be32 (mr->ibmr.rkey); +/* UMR for sig MR */ + err =set_pi_umr_wr (wr, qp, seg, size, cur_edge); +if (unlikely (err)) { +mlx5_ib_warn (dev,"\n" ); + goto out; + } +finish_wqe (qp, *ctrl, *seg, *size, *cur_edge, *idx, wr->wr_id, nreq, + fence,MLX5_OPCODE_UMR ); + + sig_attrs = mr->ibmr.sig_attrs; + err =handle_psv (dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, + &sig_attrs->mem, mr->sig->psv_memory.psv_idx, + next_fence); +if (unlikely (err)) + goto out; + + err =handle_psv (dev, qp, wr, ctrl, seg, size, cur_edge, idx, nreq, + &sig_attrs->wire, mr->sig->psv_wire.psv_idx, + next_fence); +if (unlikely (err)) + goto out; + + qp->next_fence =MLX5_FENCE_MODE_INITIATOR_SMALL ; + +out: +return err; +} + +static int handle_qpt_rc (struct mlx5_ib_dev *dev,struct mlx5_ib_qp *qp, +const struct ib_send_wr *wr, +struct mlx5_wqe_ctrl_seg **ctrl,void **seg,int *size, +void **cur_edge,unsigned int *idx,int nreq,u8 fence, +u8 next_fence,int *num_sge) +{ +int err =0 ; + +switch (wr->opcode) { +case IB_WR_RDMA_READ : +case IB_WR_RDMA_WRITE : +case IB_WR_RDMA_WRITE_WITH_IMM : +handle_rdma_op (wr, seg, size); +break ; + +case IB_WR_ATOMIC_CMP_AND_SWP : +case IB_WR_ATOMIC_FETCH_AND_ADD : +case IB_WR_MASKED_ATOMIC_CMP_AND_SWP : +mlx5_ib_warn (dev,"Atomic operations are not supported yet\n" ); + err = -EOPNOTSUPP ; + goto out; + +case IB_WR_LOCAL_INV : +handle_local_inv (qp, wr, ctrl, seg, size, cur_edge, *idx); + *num_sge =0 ; +break ; + +case IB_WR_REG_MR : + err =handle_reg_mr (qp, wr, ctrl, seg, size, cur_edge, *idx); +if (unlikely (err)) + goto out; + *num_sge =0 ; +break ; + +case IB_WR_REG_MR_INTEGRITY : + err =handle_reg_mr_integrity (dev, qp, wr, ctrl, seg, size, + cur_edge, idx, nreq, fence, + next_fence); +if (unlikely (err)) + goto out; + *num_sge =0 ; +break ; + +default : +break ; + } + +out: +return err; +} + +static void handle_qpt_uc (const struct ib_send_wr *wr,void **seg,int *size) +{ +switch (wr->opcode) { +case IB_WR_RDMA_WRITE : +case IB_WR_RDMA_WRITE_WITH_IMM : +handle_rdma_op (wr, seg, size); +break ; +default : +break ; + } +} + +static void handle_qpt_hw_gsi (struct mlx5_ib_qp *qp, +const struct ib_send_wr *wr,void **seg, +int *size,void **cur_edge) +{ +set_datagram_seg (*seg, wr); + *seg +=sizeof (struct mlx5_wqe_datagram_seg ); + *size +=sizeof (struct mlx5_wqe_datagram_seg ) /16 ; +handle_post_send_edge (&qp->sq, seg, *size, cur_edge); +} + +static void handle_qpt_ud (struct mlx5_ib_qp *qp,const struct ib_send_wr *wr, +void **seg,int *size,void **cur_edge) +{ +set_datagram_seg (*seg, wr); + *seg +=sizeof (struct mlx5_wqe_datagram_seg ); + *size +=sizeof (struct mlx5_wqe_datagram_seg ) /16 ; +handle_post_send_edge (&qp->sq, seg, *size, cur_edge); + +/* handle qp that supports ud offload */ +if (qp->flags &IB_QP_CREATE_IPOIB_UD_LSO ) { +struct mlx5_wqe_eth_pad *pad; + + pad = *seg; +memset (pad,0 ,sizeof (struct mlx5_wqe_eth_pad )); + *seg +=sizeof (struct mlx5_wqe_eth_pad ); + *size +=sizeof (struct mlx5_wqe_eth_pad ) /16 ; +set_eth_seg (wr, qp, seg, size, cur_edge); +handle_post_send_edge (&qp->sq, seg, *size, cur_edge); + } +} + +static int handle_qpt_reg_umr (struct mlx5_ib_dev *dev,struct mlx5_ib_qp *qp, +const struct ib_send_wr *wr, +struct mlx5_wqe_ctrl_seg **ctrl,void **seg, +int *size,void **cur_edge,unsigned int idx) +{ +int err =0 ; + +if (unlikely (wr->opcode !=MLX5_IB_WR_UMR )) { + err = -EINVAL ; +mlx5_ib_warn (dev,"bad opcode %d\n" , wr->opcode); + goto out; + } + + qp->sq.wr_data[idx] =MLX5_IB_WR_UMR ; + (*ctrl)->imm =cpu_to_be32 (umr_wr (wr)->mkey); + err =set_reg_umr_segment (dev, *seg, wr, + !!(MLX5_CAP_GEN (dev->mdev, atomic))); +if (unlikely (err)) + goto out; + *seg +=sizeof (struct mlx5_wqe_umr_ctrl_seg ); + *size +=sizeof (struct mlx5_wqe_umr_ctrl_seg ) /16 ; +handle_post_send_edge (&qp->sq, seg, *size, cur_edge); +set_reg_mkey_segment (*seg, wr); + *seg +=sizeof (struct mlx5_mkey_seg ); + *size +=sizeof (struct mlx5_mkey_seg ) /16 ; +handle_post_send_edge (&qp->sq, seg, *size, cur_edge); +out: +return err; +} +static int _mlx5_ib_post_send (struct ib_qp *ibqp,const struct ib_send_wr *wr,const struct ib_send_wr **bad_wr,bool drain) {struct mlx5_wqe_ctrl_seg *ctrl =NULL ;/* compiler warning */ struct mlx5_ib_dev *dev =to_mdev (ibqp->device);struct mlx5_core_dev *mdev = dev->mdev; -struct ib_reg_wr reg_pi_wr;struct mlx5_ib_qp *qp; -struct mlx5_ib_mr *mr; -struct mlx5_ib_mr *pi_mr; -struct mlx5_ib_mr pa_pi_mr; -struct ib_sig_attrs *sig_attrs;struct mlx5_wqe_xrc_seg *xrc;struct mlx5_bf *bf;void *cur_edge;
@@ -5321,186 +5582,20 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, size +=sizeof (*xrc) /16 ;/* fall through */ case IB_QPT_RC : -switch (wr->opcode) { -case IB_WR_RDMA_READ : -case IB_WR_RDMA_WRITE : -case IB_WR_RDMA_WRITE_WITH_IMM : -set_raddr_seg (seg,rdma_wr (wr)->remote_addr, -rdma_wr (wr)->rkey); - seg +=sizeof (struct mlx5_wqe_raddr_seg ); - size +=sizeof (struct mlx5_wqe_raddr_seg ) /16 ; -break ; - -case IB_WR_ATOMIC_CMP_AND_SWP : -case IB_WR_ATOMIC_FETCH_AND_ADD : -case IB_WR_MASKED_ATOMIC_CMP_AND_SWP : -mlx5_ib_warn (dev,"Atomic operations are not supported yet\n" ); - err = -ENOSYS ; + err =handle_qpt_rc (dev, qp, wr, &ctrl, &seg, &size, + &cur_edge, &idx, nreq, fence, + next_fence, &num_sge); +if (unlikely (err)) { *bad_wr = wr; goto out; - -case IB_WR_LOCAL_INV : - qp->sq.wr_data[idx] =IB_WR_LOCAL_INV ; - ctrl->imm =cpu_to_be32 (wr->ex.invalidate_rkey); -set_linv_wr (qp, &seg, &size, &cur_edge); - num_sge =0 ; -break ; - -case IB_WR_REG_MR : - qp->sq.wr_data[idx] =IB_WR_REG_MR ; - ctrl->imm =cpu_to_be32 (reg_wr (wr)->key); - err =set_reg_wr (qp,reg_wr (wr), &seg, &size, - &cur_edge, true); -if (err) { - *bad_wr = wr; - goto out; - } - num_sge =0 ; -break ; - -case IB_WR_REG_MR_INTEGRITY : - qp->sq.wr_data[idx] =IB_WR_REG_MR_INTEGRITY ; - - mr =to_mmr (reg_wr (wr)->mr); - pi_mr = mr->pi_mr; - -if (pi_mr) { -memset (®_pi_wr,0 , -sizeof (struct ib_reg_wr )); - - reg_pi_wr.mr = &pi_mr->ibmr; - reg_pi_wr.access =reg_wr (wr)->access; - reg_pi_wr.key = pi_mr->ibmr.rkey; - - ctrl->imm =cpu_to_be32 (reg_pi_wr.key); -/* UMR for data + prot registration */ - err =set_reg_wr (qp, ®_pi_wr, &seg, - &size, &cur_edge, - false); -if (err) { - *bad_wr = wr; - goto out; - } -finish_wqe (qp, ctrl, seg, size, - cur_edge, idx, wr->wr_id, - nreq, fence, -MLX5_OPCODE_UMR ); - - err =begin_wqe (qp, &seg, &ctrl, wr, - &idx, &size, &cur_edge, - nreq); -if (err) { -mlx5_ib_warn (dev,"\n" ); - err = -ENOMEM ; - *bad_wr = wr; - goto out; - } - }else { -memset (&pa_pi_mr,0 , -sizeof (struct mlx5_ib_mr )); -/* No UMR, use local_dma_lkey */ - pa_pi_mr.ibmr.lkey = - mr->ibmr.pd->local_dma_lkey; - - pa_pi_mr.ndescs = mr->ndescs; - pa_pi_mr.data_length = mr->data_length; - pa_pi_mr.data_iova = mr->data_iova; -if (mr->meta_ndescs) { - pa_pi_mr.meta_ndescs = - mr->meta_ndescs; - pa_pi_mr.meta_length = - mr->meta_length; - pa_pi_mr.pi_iova = mr->pi_iova; - } - - pa_pi_mr.ibmr.length = mr->ibmr.length; - mr->pi_mr = &pa_pi_mr; - } - ctrl->imm =cpu_to_be32 (mr->ibmr.rkey); -/* UMR for sig MR */ - err =set_pi_umr_wr (wr, qp, &seg, &size, - &cur_edge); -if (err) { -mlx5_ib_warn (dev,"\n" ); - *bad_wr = wr; - goto out; - } -finish_wqe (qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, -MLX5_OPCODE_UMR ); - -/* -* SET_PSV WQEs are not signaled and solicited -* on error -*/ - sig_attrs = mr->ibmr.sig_attrs; - err =__begin_wqe (qp, &seg, &ctrl, wr, &idx, - &size, &cur_edge, nreq, false, - true); -if (err) { -mlx5_ib_warn (dev,"\n" ); - err = -ENOMEM ; - *bad_wr = wr; - goto out; - } - err =set_psv_wr (&sig_attrs->mem, - mr->sig->psv_memory.psv_idx, - &seg, &size); -if (err) { -mlx5_ib_warn (dev,"\n" ); - *bad_wr = wr; - goto out; - } -finish_wqe (qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, next_fence, -MLX5_OPCODE_SET_PSV ); - - err =__begin_wqe (qp, &seg, &ctrl, wr, &idx, - &size, &cur_edge, nreq, false, - true); -if (err) { -mlx5_ib_warn (dev,"\n" ); - err = -ENOMEM ; - *bad_wr = wr; - goto out; - } - err =set_psv_wr (&sig_attrs->wire, - mr->sig->psv_wire.psv_idx, - &seg, &size); -if (err) { -mlx5_ib_warn (dev,"\n" ); - *bad_wr = wr; - goto out; - } -finish_wqe (qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, next_fence, -MLX5_OPCODE_SET_PSV ); - - qp->next_fence = -MLX5_FENCE_MODE_INITIATOR_SMALL ; - num_sge =0 ; + }else if (wr->opcode ==IB_WR_REG_MR_INTEGRITY ) { goto skip_psv; - -default : -break ; }break ;case IB_QPT_UC : -switch (wr->opcode) { -case IB_WR_RDMA_WRITE : -case IB_WR_RDMA_WRITE_WITH_IMM : -set_raddr_seg (seg,rdma_wr (wr)->remote_addr, -rdma_wr (wr)->rkey); - seg +=sizeof (struct mlx5_wqe_raddr_seg ); - size +=sizeof (struct mlx5_wqe_raddr_seg ) /16 ; -break ; - -default : -break ; - } +handle_qpt_uc (wr, &seg, &size);break ; -case IB_QPT_SMI :if (unlikely (!mdev->port_caps[qp->port -1 ].has_smi)) {mlx5_ib_warn (dev,"Send SMP MADs is not allowed\n" );
@@ -5510,49 +5605,16 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, }/* fall through */ case MLX5_IB_QPT_HW_GSI : -set_datagram_seg (seg, wr); - seg +=sizeof (struct mlx5_wqe_datagram_seg ); - size +=sizeof (struct mlx5_wqe_datagram_seg ) /16 ; -handle_post_send_edge (&qp->sq, &seg, size, &cur_edge); - +handle_qpt_hw_gsi (qp, wr, &seg, &size, &cur_edge);break ;case IB_QPT_UD : -set_datagram_seg (seg, wr); - seg +=sizeof (struct mlx5_wqe_datagram_seg ); - size +=sizeof (struct mlx5_wqe_datagram_seg ) /16 ; -handle_post_send_edge (&qp->sq, &seg, size, &cur_edge); - -/* handle qp that supports ud offload */ -if (qp->flags &IB_QP_CREATE_IPOIB_UD_LSO ) { -struct mlx5_wqe_eth_pad *pad; - - pad = seg; -memset (pad,0 ,sizeof (struct mlx5_wqe_eth_pad )); - seg +=sizeof (struct mlx5_wqe_eth_pad ); - size +=sizeof (struct mlx5_wqe_eth_pad ) /16 ; -set_eth_seg (wr, qp, &seg, &size, &cur_edge); -handle_post_send_edge (&qp->sq, &seg, size, - &cur_edge); - } +handle_qpt_ud (qp, wr, &seg, &size, &cur_edge);break ;case MLX5_IB_QPT_REG_UMR : -if (wr->opcode !=MLX5_IB_WR_UMR ) { - err = -EINVAL ; -mlx5_ib_warn (dev,"bad opcode\n" ); - goto out; - } - qp->sq.wr_data[idx] =MLX5_IB_WR_UMR ; - ctrl->imm =cpu_to_be32 (umr_wr (wr)->mkey); - err =set_reg_umr_segment (dev, seg, wr, !!(MLX5_CAP_GEN (mdev, atomic))); + err =handle_qpt_reg_umr (dev, qp, wr, &ctrl, &seg, + &size, &cur_edge, idx);if (unlikely (err)) goto out; - seg +=sizeof (struct mlx5_wqe_umr_ctrl_seg ); - size +=sizeof (struct mlx5_wqe_umr_ctrl_seg ) /16 ; -handle_post_send_edge (&qp->sq, &seg, size, &cur_edge); -set_reg_mkey_segment (seg, wr); - seg +=sizeof (struct mlx5_mkey_seg ); - size +=sizeof (struct mlx5_mkey_seg ) /16 ; -handle_post_send_edge (&qp->sq, &seg, size, &cur_edge);break ;default :
Signature · 1 line
2.26.2
PATCH 3/3 RDMA/mlx5: Move all WR logic from qp.c to separate file
In reply to Leon RomanovskyFrom: Leon Romanovsky <[email protected]>
Split qp.c by removing all WR logic to separate file.
--- drivers/infiniband/hw/mlx5/Makefile | 3 +- drivers/infiniband/hw/mlx5/main.c | 5 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 4 - drivers/infiniband/hw/mlx5/qp.c | 1550 +------------------------- drivers/infiniband/hw/mlx5/wr.c | 1504 +++++++++++++++++++++++++ drivers/infiniband/hw/mlx5/wr.h | 76 ++ 6 files changed, 1589 insertions(+), 1553 deletions(-) create mode 100644 drivers/infiniband/hw/mlx5/wr.c create mode 100644 drivers/infiniband/hw/mlx5/wr.h
drivers/infiniband/hw/mlx5/Makefile +2 -1
@@ -16,7 +16,8 @@ mlx5_ib-y := ah.o \ qpc.o \ restrack.o \ srq.o \ - srq_cmd.o + srq_cmd.o \ + wr.o mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o mlx5_ib-$(CONFIG_MLX5_ESWITCH) += ib_rep.o
drivers/infiniband/hw/mlx5/main.c +3 -2
@@ -61,6 +61,7 @@#include "cmd.h" #include "srq.h" #include "qp.h" +#include "wr.h" #include <linux/mlx5/fs_helpers.h> #include <linux/mlx5/accel.h> #include <rdma/uverbs_std_types.h>
@@ -6628,8 +6629,8 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .modify_qp = mlx5_ib_modify_qp, .modify_srq = mlx5_ib_modify_srq, .poll_cq = mlx5_ib_poll_cq, - .post_recv = mlx5_ib_post_recv, - .post_send = mlx5_ib_post_send, + .post_recv = mlx5_ib_post_recv_nodrain, + .post_send = mlx5_ib_post_send_nodrain, .post_srq_recv = mlx5_ib_post_srq_recv, .process_mad = mlx5_ib_process_mad, .query_ah = mlx5_ib_query_ah,
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -4
@@ -1178,10 +1178,6 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attrint mlx5_ib_destroy_qp (struct ib_qp *qp,struct ib_udata *udata);void mlx5_ib_drain_sq (struct ib_qp *qp);void mlx5_ib_drain_rq (struct ib_qp *qp); -int mlx5_ib_post_send (struct ib_qp *ibqp,const struct ib_send_wr *wr, -const struct ib_send_wr **bad_wr); -int mlx5_ib_post_recv (struct ib_qp *ibqp,const struct ib_recv_wr *wr, -const struct ib_recv_wr **bad_wr);int mlx5_ib_read_wqe_sq (struct mlx5_ib_qp *qp,int wqe_index,void *buffer,size_t buflen,size_t *bc); int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
drivers/infiniband/hw/mlx5/qp.c +4 -1546
1,598 lines not on this page · raw message
drivers/infiniband/hw/mlx5/wr.c new file +1504 -0
1,366 lines not on this page · raw message
drivers/infiniband/hw/mlx5/wr.h new file +76 -0
76 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
2.26.2
Part of the diff is not shown here. View the raw message.
Jason Gunthorpe
AppliedQuoted 7 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Hi,
This short series continue to cleanup
qp.cfile, which grew to be completely unmaintainable.
Quoted 8 lines
Thanks
Leon Romanovsky (2):
RDMA/mlx5: Updatemlx5_ibto use new cmd interface RDMA/mlx5: Move all WR logic fromqp.cto separate fileMax Gurtovoy (1):
RDMA/mlx5: Refactormlx5_post_send()to improve readability
Applied to for-next, thanks
Jason