Skip to contentSkip to conversation

Various cleanups

PATCH Applied Leon Romanovsky 2 participants 7 messages Feb 3, 2021 – Feb 5, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover Various cleanups

From: Leon Romanovsky <[email protected]>

Changelog:
v1:
* Rebase
v0: https://lore.kernel.org/linux-rdma/[email protected]

Parav Pandit (5):
IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev IB/mlx5: Avoid calling query device for reading pkey table length IB/mlx5: Improve query port for representor port RDMA/core: Introduce and use API to read port immutable data IB/mlx5: Use rdma_for_each_port for port iteration

 drivers/infiniband/core/device.c     |  14 +++
 drivers/infiniband/hw/mlx5/mad.c     |  10 +--
 drivers/infiniband/hw/mlx5/main.c    | 127 ++++++---------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |   9 +-
 drivers/infiniband/hw/mlx5/qp.c      |  25 ++----
 drivers/infiniband/hw/mlx5/wr.c      |   2 +-
 include/linux/mlx5/driver.h          |   8 --
 include/rdma/ib_verbs.h              |   3 +
 8 files changed, 65 insertions(+), 133 deletions(-)

--
2.29.2

#1 lore raw

PATCH 1/5 IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev

From: Parav Pandit <[email protected]>

mlx5_port_caps are RDMA specific capabilities. These are not used by the mlx5_core_device at all. Move them to mlx5_ib_dev where it is used and reduce the scope of it to multiple drivers.

---
 drivers/infiniband/hw/mlx5/mad.c     |  8 ++++----
 drivers/infiniband/hw/mlx5/main.c    | 14 ++++++--------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  8 ++++++++
 drivers/infiniband/hw/mlx5/qp.c      |  6 +++---
 drivers/infiniband/hw/mlx5/wr.c      |  2 +-
 include/linux/mlx5/driver.h          |  8 --------
 6 files changed, 22 insertions(+), 24 deletions(-)
drivers/infiniband/hw/mlx5/mad.c +4 -4
index 53dec6063245..e9d0a5269582 100644
@@ -48,7 +48,7 @@ static bool can_do_mad_ifc(struct mlx5_ib_dev *dev, u8 port_num,
 	if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED &&
 	    in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
 		return true;
-	return dev->mdev->port_caps[port_num - 1].has_smi;
+	return dev->port_caps[port_num - 1].has_smi;
 }
 
 static int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey,
@@ -299,7 +299,7 @@ int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)
 
 	packet_error = be16_to_cpu(out_mad->status);
 
-	dev->mdev->port_caps[port - 1].ext_port_cap = (!err && !packet_error) ?
+	dev->port_caps[port - 1].ext_port_cap = (!err && !packet_error) ?
 		MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO : 0;
 
 out:
@@ -549,7 +549,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 	props->port_cap_flags	= be32_to_cpup((__be32 *)(out_mad->data + 20));
 	props->gid_tbl_len	= out_mad->data[50];
 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
-	props->pkey_tbl_len	= mdev->port_caps[port - 1].pkey_table_len;
+	props->pkey_tbl_len	= dev->port_caps[port - 1].pkey_table_len;
 	props->bad_pkey_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 46));
 	props->qkey_viol_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 48));
 	props->active_width	= out_mad->data[31] & 0xf;
@@ -589,7 +589,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 
 	/* If reported active speed is QDR, check if is FDR-10 */
 	if (props->active_speed == 4) {
-		if (mdev->port_caps[port - 1].ext_port_cap &
+		if (dev->port_caps[port - 1].ext_port_cap &
 		    MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {
 			init_query_mad(in_mad);
 			in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;
drivers/infiniband/hw/mlx5/main.c +6 -8
index 40fb86db3376..d2f185ae2b9c 100644
@@ -2946,8 +2946,8 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 	int err;
 	int port;
 
-	for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) {
-		dev->mdev->port_caps[port - 1].has_smi = false;
+	for (port = 1; port <= ARRAY_SIZE(dev->port_caps); port++) {
+		dev->port_caps[port - 1].has_smi = false;
 		if (MLX5_CAP_GEN(dev->mdev, port_type) ==
 		    MLX5_CAP_PORT_TYPE_IB) {
 			if (MLX5_CAP_GEN(dev->mdev, ib_virt)) {
@@ -2959,10 +2959,10 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 						    port, err);
 					return err;
 				}
-				dev->mdev->port_caps[port - 1].has_smi =
+				dev->port_caps[port - 1].has_smi =
 					vport_ctx.has_smi;
 			} else {
-				dev->mdev->port_caps[port - 1].has_smi = true;
+				dev->port_caps[port - 1].has_smi = true;
 			}
 		}
 	}
@@ -3004,10 +3004,8 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 		goto out;
 	}
 
-	dev->mdev->port_caps[port - 1].pkey_table_len =
-					dprops->max_pkeys;
-	dev->mdev->port_caps[port - 1].gid_table_len =
-					pprops->gid_tbl_len;
+	dev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys;
+	dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
 	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
 		    port, dprops->max_pkeys, pprops->gid_tbl_len);
 
drivers/infiniband/hw/mlx5/mlx5_ib.h +8 -0
index 2fd2927abe45..c2f91c15b973 100644
@@ -1036,6 +1036,13 @@ struct mlx5_var_table {
 	u64 num_var_hw_entries;
 };
 
+struct mlx5_port_caps {
+	int gid_table_len;
+	int pkey_table_len;
+	bool has_smi;
+	u8 ext_port_cap;
+};
+
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
@@ -1096,6 +1103,7 @@ struct mlx5_ib_dev {
 	struct mlx5_var_table var_table;
 
 	struct xarray sig_mrs;
+	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
drivers/infiniband/hw/mlx5/qp.c +3 -3
index 88be94215708..5274349dd998 100644
@@ -3177,10 +3177,10 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 
 	if (ah_flags & IB_AH_GRH) {
 		if (grh->sgid_index >=
-		    dev->mdev->port_caps[port - 1].gid_table_len) {
+		    dev->port_caps[port - 1].gid_table_len) {
 			pr_err("sgid_index (%u) too large. max is %d\n",
 			       grh->sgid_index,
-			       dev->mdev->port_caps[port - 1].gid_table_len);
+			       dev->port_caps[port - 1].gid_table_len);
 			return -EINVAL;
 		}
 	}
@@ -4311,7 +4311,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	if (attr_mask & IB_QP_PKEY_INDEX) {
 		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
 		if (attr->pkey_index >=
-		    dev->mdev->port_caps[port - 1].pkey_table_len) {
+		    dev->port_caps[port - 1].pkey_table_len) {
 			mlx5_ib_dbg(dev, "invalid pkey index %d\n",
 				    attr->pkey_index);
 			goto out;
drivers/infiniband/hw/mlx5/wr.c +1 -1
index d6038fb6c50c..cf2852cba45c 100644
@@ -1369,7 +1369,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 			handle_qpt_uc(wr, &seg, &size);
 			break;
 		case IB_QPT_SMI:
-			if (unlikely(!mdev->port_caps[qp->port - 1].has_smi)) {
+			if (unlikely(!dev->port_caps[qp->port - 1].has_smi)) {
 				mlx5_ib_warn(dev, "Send SMP MADs is not allowed\n");
 				err = -EPERM;
 				*bad_wr = wr;
include/linux/mlx5/driver.h +0 -8
index f93bfe7473aa..11558c2e99f0 100644
@@ -305,13 +305,6 @@ struct mlx5_cmd {
 	struct mlx5_cmd_stats *stats;
 };
 
-struct mlx5_port_caps {
-	int	gid_table_len;
-	int	pkey_table_len;
-	u8	ext_port_cap;
-	bool	has_smi;
-};
-
 struct mlx5_cmd_mailbox {
 	void	       *buf;
 	dma_addr_t	dma;
@@ -694,7 +687,6 @@ struct mlx5_core_dev {
 	u8			rev_id;
 	char			board_id[MLX5_BOARD_ID_LEN];
 	struct mlx5_cmd		cmd;
-	struct mlx5_port_caps	port_caps[MLX5_MAX_PORTS];
 	struct {
 		u32 hca_cur[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)];
 		u32 hca_max[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)];

--
2.29.2

#2 lore raw

PATCH 2/5 IB/mlx5: Avoid calling query device for reading pkey table length

In reply to Leon Romanovsky

From: Parav Pandit <[email protected]>

Pkey table length for all the ports of the device is the same. Currently get_ports_cap() reads and stores it for each port by querying the device which reads more than just pkey table length.

For representor device ports which can be in range of hundreds, it queries is for each such port and end up returning same value for all the ports.

When in representor mode, modify QP accesses pkey port caps for a port index that can be outside of the port_caps table.

Hence, simplify the logic to query the max pkey table length only once during device initialization sequence.

---
 drivers/infiniband/hw/mlx5/mad.c     |  2 +-
 drivers/infiniband/hw/mlx5/main.c    | 24 ++++++------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  2 +-
 drivers/infiniband/hw/mlx5/qp.c      | 17 ++++-------------
 4 files changed, 12 insertions(+), 33 deletions(-)
drivers/infiniband/hw/mlx5/mad.c +1 -1
index e9d0a5269582..cdb47a00e516 100644
@@ -549,7 +549,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 	props->port_cap_flags	= be32_to_cpup((__be32 *)(out_mad->data + 20));
 	props->gid_tbl_len	= out_mad->data[50];
 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
-	props->pkey_tbl_len	= dev->port_caps[port - 1].pkey_table_len;
+	props->pkey_tbl_len	= dev->pkey_table_len;
 	props->bad_pkey_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 46));
 	props->qkey_viol_cntr	= be16_to_cpup((__be16 *)(out_mad->data + 48));
 	props->active_width	= out_mad->data[31] & 0xf;
drivers/infiniband/hw/mlx5/main.c +6 -18
index d2f185ae2b9c..364dc3e6a2de 100644
@@ -816,9 +816,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (err)
 		return err;
 
-	err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
-	if (err)
-		return err;
+	props->max_pkeys = dev->pkey_table_len;
 
 	err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
 	if (err)
@@ -2979,7 +2977,6 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)
 
 static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 {
-	struct ib_device_attr *dprops = NULL;
 	struct ib_port_attr *pprops = NULL;
 	int err = -ENOMEM;
 
@@ -2987,16 +2984,6 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 	if (!pprops)
 		goto out;
 
-	dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
-	if (!dprops)
-		goto out;
-
-	err = mlx5_ib_query_device(&dev->ib_dev, dprops, NULL);
-	if (err) {
-		mlx5_ib_warn(dev, "query_device failed %d\n", err);
-		goto out;
-	}
-
 	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
 	if (err) {
 		mlx5_ib_warn(dev, "query_port %d failed %d\n",
@@ -3004,15 +2991,12 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 		goto out;
 	}
 
-	dev->port_caps[port - 1].pkey_table_len = dprops->max_pkeys;
 	dev->port_caps[port - 1].gid_table_len = pprops->gid_tbl_len;
 	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
-		    port, dprops->max_pkeys, pprops->gid_tbl_len);
+		    port, dev->pkey_table_len, pprops->gid_tbl_len);
 
 out:
 	kfree(pprops);
-	kfree(dprops);
-
 	return err;
 }
 
@@ -3979,6 +3963,10 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	if (err)
 		goto err_mp;
 
+	err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
+	if (err)
+		goto err_mp;
+
 	if (mlx5_use_mad_ifc(dev))
 		get_ext_port_caps(dev);
 
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -1
index c2f91c15b973..36a92f3c29e3 100644
@@ -1038,7 +1038,6 @@ struct mlx5_var_table {
 
 struct mlx5_port_caps {
 	int gid_table_len;
-	int pkey_table_len;
 	bool has_smi;
 	u8 ext_port_cap;
 };
@@ -1104,6 +1103,7 @@ struct mlx5_ib_dev {
 
 	struct xarray sig_mrs;
 	struct mlx5_port_caps port_caps[MLX5_MAX_PORTS];
+	u16 pkey_table_len;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
drivers/infiniband/hw/mlx5/qp.c +4 -13
index 5274349dd998..475470237e0b 100644
@@ -4231,7 +4231,6 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	enum ib_qp_type qp_type;
 	enum ib_qp_state cur_state, new_state;
 	int err = -EINVAL;
-	int port;
 
 	if (!mlx5_ib_modify_qp_allowed(dev, qp, ibqp->qp_type))
 		return -EOPNOTSUPP;
@@ -4276,10 +4275,6 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
 	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
 
-	if (!(cur_state == new_state && cur_state == IB_QPS_RESET)) {
-		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
-	}
-
 	if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 		if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
 			mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
@@ -4308,14 +4303,10 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		goto out;
 	}
 
-	if (attr_mask & IB_QP_PKEY_INDEX) {
-		port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
-		if (attr->pkey_index >=
-		    dev->port_caps[port - 1].pkey_table_len) {
-			mlx5_ib_dbg(dev, "invalid pkey index %d\n",
-				    attr->pkey_index);
-			goto out;
-		}
+	if ((attr_mask & IB_QP_PKEY_INDEX) &&
+	    attr->pkey_index >= dev->pkey_table_len) {
+		mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
+		goto out;
 	}
 
 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
Signature · 1 line
2.29.2
#3 lore raw

PATCH 3/5 IB/mlx5: Improve query port for representor port

In reply to Leon Romanovsky

From: Parav Pandit <[email protected]>

Improve query port functionality for representor port as below.

1. RoCE Qkey violation counters are not applicable for representor port.
2. Avoid setting gid_tbl_len twice for representor port.
3. Avoid setting ip_gids and IB_PORT_CM_SUP property for representor port as GID table is empty and CM support is not present in representor mode.

---
 drivers/infiniband/hw/mlx5/main.c | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)
drivers/infiniband/hw/mlx5/main.c +10 -21
index 364dc3e6a2de..176f2a866f12 100644
@@ -462,7 +462,6 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	struct net_device *ndev, *upper;
 	enum ib_mtu ndev_ib_mtu;
 	bool put_mdev = true;
-	u16 qkey_viol_cntr;
 	u32 eth_prot_oper;
 	u8 mdev_port_num;
 	bool ext;
@@ -500,20 +499,22 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
 				 &props->active_width, ext);
 
-	props->port_cap_flags |= IB_PORT_CM_SUP;
-	props->ip_gids = true;
+	if (!dev->is_rep && mlx5_is_roce_enabled(mdev)) {
+		u16 qkey_viol_cntr;
 
-	props->gid_tbl_len      = MLX5_CAP_ROCE(dev->mdev,
-						roce_address_table_size);
+		props->port_cap_flags |= IB_PORT_CM_SUP;
+		props->ip_gids = true;
+		props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
+						   roce_address_table_size);
+		mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
+		props->qkey_viol_cntr = qkey_viol_cntr;
+	}
 	props->max_mtu          = IB_MTU_4096;
 	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
 	props->pkey_tbl_len     = 1;
 	props->state            = IB_PORT_DOWN;
 	props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;
 
-	mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
-	props->qkey_viol_cntr = qkey_viol_cntr;
-
 	/* If this is a stub query for an unaffiliated port stop here */
 	if (!put_mdev)
 		goto out;
@@ -1383,19 +1384,7 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,
 				  struct ib_port_attr *props)
 {
-	int ret;
-
-	/* Only link layer == ethernet is valid for representors
-	 * and we always use port 1
-	 */
-	ret = mlx5_query_port_roce(ibdev, port, props);
-	if (ret || !props)
-		return ret;
-
-	/* We don't support GIDS */
-	props->gid_tbl_len = 0;
-
-	return ret;
+	return mlx5_query_port_roce(ibdev, port, props);
 }
 
 static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
Signature · 1 line
2.29.2
#4 lore raw

PATCH 4/5 RDMA/core: Introduce and use API to read port immutable data

In reply to Leon Romanovsky

From: Parav Pandit <[email protected]>

Currently mlx5 driver caches port GID table length for 2 ports. It is also cached by IB core as port immutable data.

When mlx5 representor ports are present, which are usually more than 2, invalid access to port_caps array can happen while validating the GID table length which is only for 2 ports.

To avoid this, take help of the IB cores port immutable data by exposing an API to read the port immutable fields.

Remove mlx5 driver's internal cache, thereby reduce code and data.

---
 drivers/infiniband/core/device.c     | 14 +++++++
 drivers/infiniband/hw/mlx5/main.c    | 55 +---------------------------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  1 -
 drivers/infiniband/hw/mlx5/qp.c      |  8 ++--
 include/rdma/ib_verbs.h              |  3 ++
 5 files changed, 23 insertions(+), 58 deletions(-)
drivers/infiniband/core/device.c +14 -0
index c895d7bfa512..051c018fb73c 100644
@@ -848,6 +848,20 @@ static int setup_port_data(struct ib_device *device)
 	return 0;
 }
 
+/**
+ * ib_port_immutable_read() - Read rdma port's immutable data
+ * @dev - IB device
+ * @port - port number whose immutable data to read. It starts with index 1 and
+ *         valid upto including rdma_end_port().
+ */
+const struct ib_port_immutable*
+ib_port_immutable_read(struct ib_device *dev, unsigned int port)
+{
+	WARN_ON(!rdma_is_port_valid(dev, port));
+	return &dev->port_data[port].immutable;
+}
+EXPORT_SYMBOL(ib_port_immutable_read);
+
 void ib_get_device_fw_str(struct ib_device *dev, char *str)
 {
 	if (dev->ops.get_dev_fw_str)
drivers/infiniband/hw/mlx5/main.c +1 -54

81 lines not on this page · raw message

drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -1
index 36a92f3c29e3..0f567d570230 100644
@@ -1037,7 +1037,6 @@ struct mlx5_var_table {
 };
 
 struct mlx5_port_caps {
-	int gid_table_len;
 	bool has_smi;
 	u8 ext_port_cap;
 };
drivers/infiniband/hw/mlx5/qp.c +5 -3
index 475470237e0b..358c44c5a8fc 100644
@@ -3176,11 +3176,13 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			 alt ? attr->alt_pkey_index : attr->pkey_index);
 
 	if (ah_flags & IB_AH_GRH) {
-		if (grh->sgid_index >=
-		    dev->port_caps[port - 1].gid_table_len) {
+		const struct ib_port_immutable *immutable;
+
+		immutable = ib_port_immutable_read(&dev->ib_dev, port);
+		if (grh->sgid_index >= immutable->gid_tbl_len) {
 			pr_err("sgid_index (%u) too large. max is %d\n",
 			       grh->sgid_index,
-			       dev->port_caps[port - 1].gid_table_len);
+			       immutable->gid_tbl_len);
 			return -EINVAL;
 		}
 	}
include/rdma/ib_verbs.h +3 -0
index 62e574c50555..ca28fca5736b 100644
@@ -4674,4 +4674,7 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
 
 	return (u32)(v & IB_GRH_FLOWLABEL_MASK);
 }
+
+const struct ib_port_immutable*
+ib_port_immutable_read(struct ib_device *dev, unsigned int port);
 #endif /* IB_VERBS_H */
Signature · 1 line
2.29.2
#5 lore raw

PATCH 5/5 IB/mlx5: Use rdma_for_each_port for port iteration

In reply to Leon Romanovsky

From: Parav Pandit <[email protected]>

Instead of open coding the loop for port iteration, use rdma_for_each_port macro provided by core.

To use such macro, early initialization of phys_port_cnt is needed. Hence, initialize such constant early enough in the init stage.

Whichever functions are called with port using rdma_for_each_port(), convert their port type from u8 to unsigned int to match the core API.

---
 drivers/infiniband/hw/mlx5/mad.c     |  2 +-
 drivers/infiniband/hw/mlx5/main.c    | 15 ++++++++-------
 drivers/infiniband/hw/mlx5/mlx5_ib.h |  2 +-
 3 files changed, 10 insertions(+), 9 deletions(-)
drivers/infiniband/hw/mlx5/mad.c +1 -1
index cdb47a00e516..652c6ccf1881 100644
@@ -279,7 +279,7 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
 }
 
-int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)
+int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port)
 {
 	struct ib_smp *in_mad  = NULL;
 	struct ib_smp *out_mad = NULL;
drivers/infiniband/hw/mlx5/main.c +8 -7
index 1e1e3edcb1d5..ce4a9eba53f9 100644
@@ -2958,9 +2958,9 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 
 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
 {
-	int port;
+	unsigned int port;
 
-	for (port = 1; port <= dev->num_ports; port++)
+	rdma_for_each_port (&dev->ib_dev, port)
 		mlx5_query_ext_port_caps(dev, port);
 }
 
@@ -3881,6 +3881,12 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	int err;
 	int i;
 
+	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
+	dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
+	dev->ib_dev.phys_port_cnt = dev->num_ports;
+	dev->ib_dev.dev.parent = mdev->device;
+	dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
+
 	for (i = 0; i < dev->num_ports; i++) {
 		spin_lock_init(&dev->port[i].mp.mpi_lock);
 		rwlock_init(&dev->port[i].roce.netdev_lock);
@@ -3906,12 +3912,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	if (mlx5_use_mad_ifc(dev))
 		get_ext_port_caps(dev);
 
-	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
-	dev->ib_dev.local_dma_lkey	= 0 /* not supported for now */;
-	dev->ib_dev.phys_port_cnt	= dev->num_ports;
 	dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);
-	dev->ib_dev.dev.parent		= mdev->device;
-	dev->ib_dev.lag_flags		= RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
 
 	err = init_srcu_struct(&dev->odp_srcu);
 	if (err)
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -1
index 0f567d570230..ab52083634e6 100644
@@ -1299,7 +1299,7 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 			size_t *out_mad_size, u16 *out_mad_pkey_index);
 int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
 int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
-int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
+int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, unsigned int port);
 int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
 					 __be64 *sys_image_guid);
 int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
Signature · 1 line
2.29.2
maintainer Applied
#6 lore raw
In reply to Leon Romanovsky
Quoted 7 lines · Leon Romanovsky

From: Leon Romanovsky <[email protected]>

Changelog:
v1:
* Rebase
v0: https://lore.kernel.org/linux-rdma/[email protected]

Quoted 6 lines

Parav Pandit (5):
IB/mlx5: Move mlx5_port_caps from mlx5_core_dev to mlx5_ib_dev IB/mlx5: Avoid calling query device for reading pkey table length IB/mlx5: Improve query port for representor port RDMA/core: Introduce and use API to read port immutable data IB/mlx5: Use rdma_for_each_port for port iteration

Applied to for-next, thanks

Jason