Skip to contentSkip to conversation

Misc patches for rtrs

PATCH Applied Guoqing Jiang 2 participants 10 messages Nov 17, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v2 cover Misc patches for rtrs

Hi,

This V2 version only fix some typos in headers
and add collect more tags from Jinpu, so all
the patches have been reviewed, thanks you!

Thanks,
Guoqing

Changes from RFC:

1. drop 4 patches (one of them breaks compatibility which can do later after we collect similar changes to update proto version).

2. collect tags from Haris, thanks!

3. address other comments.

Guoqing Jiang (8):
RDMA/rtrs-srv: Refactor rtrs_srv_rdma_cm_handler RDMA/rtrs-srv: Refactor the handling of failure case in map_cont_bufs RDMA/rtrs-srv: Correct the checking of ib_map_mr_sg RDMA/rtrs-clt: Correct the checking of ib_map_mr_sg RDMA/rtrs-srv: Remove outdated comments from create_con RDMA/rtrs: Clean up rtrs_rdma_dev_pd_ops
RDMA/rtrs-srv: Fix several issues in rtrs_srv_destroy_path_files RDMA/rtrs-srv: Remove kobject_del from
rtrs_srv_destroy_once_sysfs_root_folders

 drivers/infiniband/ulp/rtrs/rtrs-clt.c       |  6 +-
 drivers/infiniband/ulp/rtrs/rtrs-pri.h       |  3 -
 drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c | 13 ++--
 drivers/infiniband/ulp/rtrs/rtrs-srv.c       | 72 ++++++++------------
 drivers/infiniband/ulp/rtrs/rtrs.c           | 22 ++----
 5 files changed, 44 insertions(+), 72 deletions(-)
Signature · 1 line
2.31.1
#1 lore raw

PATCH v2 1/8 RDMA/rtrs-srv: Refactor rtrs_srv_rdma_cm_handler

The RDMA_CM_EVENT_CONNECT_REQUEST is quite different to other types, let's check it separately at the beginning of routine, then we can avoid the indentation accordingly.

---
 drivers/infiniband/ulp/rtrs/rtrs-srv.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-srv.c +8 -9
index 22d7ba05e9fe..5fe3699cb8ff 100644
@@ -1950,22 +1950,21 @@ static int rtrs_srv_rdma_cm_handler(struct rdma_cm_id *cm_id,
 {
 	struct rtrs_srv_path *srv_path = NULL;
 	struct rtrs_path *s = NULL;
+	struct rtrs_con *c = NULL;
 
-	if (ev->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
-		struct rtrs_con *c = cm_id->context;
-
-		s = c->path;
-		srv_path = to_srv_path(s);
-	}
-
-	switch (ev->event) {
-	case RDMA_CM_EVENT_CONNECT_REQUEST:
+	if (ev->event == RDMA_CM_EVENT_CONNECT_REQUEST)
 		/*
 		 * In case of error cma.c will destroy cm_id,
 		 * see cma_process_remove()
 		 */
 		return rtrs_rdma_connect(cm_id, ev->param.conn.private_data,
 					  ev->param.conn.private_data_len);
+
+	c = cm_id->context;
+	s = c->path;
+	srv_path = to_srv_path(s);
+
+	switch (ev->event) {
 	case RDMA_CM_EVENT_ESTABLISHED:
 		/* Nothing here */
 		break;
Signature · 1 line
2.31.1
#2 lore raw

PATCH v2 2/8 RDMA/rtrs-srv: Refactor the handling of failure case in map_cont_bufs

In reply to Guoqing Jiang

Let's call unmap_cont_bufs when failure happens, and also only update mrs_num after everything is settled which means we can remove 'mri'.

---
 drivers/infiniband/ulp/rtrs/rtrs-srv.c | 47 +++++++++++---------------
 1 file changed, 20 insertions(+), 27 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-srv.c +20 -27
index 5fe3699cb8ff..b877dd57b6b9 100644
@@ -561,9 +561,11 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path)
 {
 	struct rtrs_srv_sess *srv = srv_path->srv;
 	struct rtrs_path *ss = &srv_path->s;
-	int i, mri, err, mrs_num;
+	int i, err, mrs_num;
 	unsigned int chunk_bits;
 	int chunks_per_mr = 1;
+	struct ib_mr *mr;
+	struct sg_table *sgt;
 
 	/*
 	 * Here we map queue_depth chunks to MR.  Firstly we have to
@@ -586,16 +588,14 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path)
 	if (!srv_path->mrs)
 		return -ENOMEM;
 
-	srv_path->mrs_num = mrs_num;
-
-	for (mri = 0; mri < mrs_num; mri++) {
-		struct rtrs_srv_mr *srv_mr = &srv_path->mrs[mri];
-		struct sg_table *sgt = &srv_mr->sgt;
+	for (srv_path->mrs_num = 0; srv_path->mrs_num < mrs_num;
+	     srv_path->mrs_num++) {
+		struct rtrs_srv_mr *srv_mr = &srv_path->mrs[srv_path->mrs_num];
 		struct scatterlist *s;
-		struct ib_mr *mr;
 		int nr, nr_sgt, chunks;
 
-		chunks = chunks_per_mr * mri;
+		sgt = &srv_mr->sgt;
+		chunks = chunks_per_mr * srv_path->mrs_num;
 		if (!always_invalidate)
 			chunks_per_mr = min_t(int, chunks_per_mr,
 					      srv->queue_depth - chunks);
@@ -644,31 +644,24 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path)
 
 		ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey));
 		srv_mr->mr = mr;
-
-		continue;
-err:
-		while (mri--) {
-			srv_mr = &srv_path->mrs[mri];
-			sgt = &srv_mr->sgt;
-			mr = srv_mr->mr;
-			rtrs_iu_free(srv_mr->iu, srv_path->s.dev->ib_dev, 1);
-dereg_mr:
-			ib_dereg_mr(mr);
-unmap_sg:
-			ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
-					sgt->nents, DMA_BIDIRECTIONAL);
-free_sg:
-			sg_free_table(sgt);
-		}
-		kfree(srv_path->mrs);
-
-		return err;
 	}
 
 	chunk_bits = ilog2(srv->queue_depth - 1) + 1;
 	srv_path->mem_bits = (MAX_IMM_PAYL_BITS - chunk_bits);
 
 	return 0;
+
+dereg_mr:
+	ib_dereg_mr(mr);
+unmap_sg:
+	ib_dma_unmap_sg(srv_path->s.dev->ib_dev, sgt->sgl,
+			sgt->nents, DMA_BIDIRECTIONAL);
+free_sg:
+	sg_free_table(sgt);
+err:
+	unmap_cont_bufs(srv_path);
+
+	return err;
 }
 
 static void rtrs_srv_hb_err_handler(struct rtrs_con *c)
Signature · 1 line
2.31.1
#3 lore raw

PATCH v2 3/8 RDMA/rtrs-srv: Correct the checking of ib_map_mr_sg

In reply to Guoqing Jiang

We should check with nr_sgt, also the only successful case is that all sg elements are mapped, so make it explicitly.

---
 drivers/infiniband/ulp/rtrs/rtrs-srv.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/ulp/rtrs/rtrs-srv.c +1 -1
index b877dd57b6b9..581c850e71d6 100644
@@ -622,7 +622,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path)
 		}
 		nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt,
 				  NULL, max_chunk_size);
-		if (nr < 0 || nr < sgt->nents) {
+		if (nr != nr_sgt) {
 			err = nr < 0 ? nr : -EINVAL;
 			goto dereg_mr;
 		}
Signature · 1 line
2.31.1
#4 lore raw

PATCH v2 4/8 RDMA/rtrs-clt: Correct the checking of ib_map_mr_sg

In reply to Guoqing Jiang

We should check with count, also the only successful case is that all sg elements are mapped, so make it explicitly.

---
 drivers/infiniband/ulp/rtrs/rtrs-clt.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-clt.c +2 -4
index 8546b8816524..be7c8480f947 100644
@@ -1064,10 +1064,8 @@ static int rtrs_map_sg_fr(struct rtrs_clt_io_req *req, size_t count)
 
 	/* Align the MR to a 4K page size to match the block virt boundary */
 	nr = ib_map_mr_sg(req->mr, req->sglist, count, NULL, SZ_4K);
-	if (nr < 0)
-		return nr;
-	if (nr < req->sg_cnt)
-		return -EINVAL;
+	if (nr != count)
+		return nr < 0 ? nr : -EINVAL;
 	ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
 
 	return nr;
Signature · 1 line
2.31.1
#5 lore raw

PATCH v2 5/8 RDMA/rtrs-srv: Remove outdated comments from create_con

In reply to Guoqing Jiang

Remove the orphan comments.

---
 drivers/infiniband/ulp/rtrs/rtrs-srv.c | 6 ------
 1 file changed, 6 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-srv.c +0 -6
index 581c850e71d6..d1703e2c0b82 100644
@@ -1671,12 +1671,6 @@ static int create_con(struct rtrs_srv_path *srv_path,
 				      srv->queue_depth * (1 + 2) + 1);
 
 		max_recv_wr = srv->queue_depth + 1;
-		/*
-		 * If we have all receive requests posted and
-		 * all write requests posted and each read request
-		 * requires an invalidate request + drain
-		 * and qp gets into error state.
-		 */
 	}
 	cq_num = max_send_wr + max_recv_wr;
 	atomic_set(&con->c.sq_wr_avail, max_send_wr);
Signature · 1 line
2.31.1
#6 lore raw

PATCH v2 6/8 RDMA/rtrs: Clean up rtrs_rdma_dev_pd_ops

In reply to Guoqing Jiang

Let's remove them since the three members are not used.

---
 drivers/infiniband/ulp/rtrs/rtrs-pri.h |  3 ---
 drivers/infiniband/ulp/rtrs/rtrs.c     | 22 ++++------------------
 2 files changed, 4 insertions(+), 21 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-pri.h +0 -3
index a2420eecaf5a..ab25619261d2 100644
@@ -68,10 +68,7 @@ enum {
 struct rtrs_ib_dev;
 
 struct rtrs_rdma_dev_pd_ops {
-	struct rtrs_ib_dev *(*alloc)(void);
-	void (*free)(struct rtrs_ib_dev *dev);
 	int (*init)(struct rtrs_ib_dev *dev);
-	void (*deinit)(struct rtrs_ib_dev *dev);
 };
 
 struct rtrs_rdma_dev_pd {
drivers/infiniband/ulp/rtrs/rtrs.c +4 -18
index ed324b47d93a..4bf9d868cc52 100644
@@ -557,7 +557,6 @@ EXPORT_SYMBOL(rtrs_addr_to_sockaddr);
 void rtrs_rdma_dev_pd_init(enum ib_pd_flags pd_flags,
 			    struct rtrs_rdma_dev_pd *pool)
 {
-	WARN_ON(pool->ops && (!pool->ops->alloc ^ !pool->ops->free));
 	INIT_LIST_HEAD(&pool->list);
 	mutex_init(&pool->mutex);
 	pool->pd_flags = pd_flags;
@@ -583,15 +582,8 @@ static void dev_free(struct kref *ref)
 	list_del(&dev->entry);
 	mutex_unlock(&pool->mutex);
 
-	if (pool->ops && pool->ops->deinit)
-		pool->ops->deinit(dev);
-
 	ib_dealloc_pd(dev->ib_pd);
-
-	if (pool->ops && pool->ops->free)
-		pool->ops->free(dev);
-	else
-		kfree(dev);
+	kfree(dev);
 }
 
 int rtrs_ib_dev_put(struct rtrs_ib_dev *dev)
@@ -618,11 +610,8 @@ rtrs_ib_dev_find_or_add(struct ib_device *ib_dev,
 			goto out_unlock;
 	}
 	mutex_unlock(&pool->mutex);
-	if (pool->ops && pool->ops->alloc)
-		dev = pool->ops->alloc();
-	else
-		dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (IS_ERR_OR_NULL(dev))
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
 		goto out_err;
 
 	kref_init(&dev->ref);
@@ -644,10 +633,7 @@ rtrs_ib_dev_find_or_add(struct ib_device *ib_dev,
 out_free_pd:
 	ib_dealloc_pd(dev->ib_pd);
 out_free_dev:
-	if (pool->ops && pool->ops->free)
-		pool->ops->free(dev);
-	else
-		kfree(dev);
+	kfree(dev);
 out_err:
 	return NULL;
 }
Signature · 1 line
2.31.1
#7 lore raw

PATCH v2 7/8 RDMA/rtrs-srv: Fix several issues in rtrs_srv_destroy_path_files

In reply to Guoqing Jiang

There are several issues in the function which is supposed to be paired with rtrs_srv_create_path_files.

1. rtrs_srv_stats_attr_group is not removed though it is created in rtrs_srv_create_stats_files.

2. it makes more sense to check kobj_stats.state_in_sysfs before destroy kobj_stats instead of rely on kobj.state_in_sysfs.

3. kobject_init_and_add is used for both kobjs (srv_path->kobj and srv_path->stats->kobj_stats), however we missed to call kobject_del for srv_path->kobj which was called in free_path.

4. rtrs_srv_destroy_once_sysfs_root_folders is independent of either kobj or kobj_stats.

---
 drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +9 -3
index 2a3c9ac64a42..da8e205ce331 100644
@@ -304,12 +304,18 @@ int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path)
 
 void rtrs_srv_destroy_path_files(struct rtrs_srv_path *srv_path)
 {
-	if (srv_path->kobj.state_in_sysfs) {
+	if (srv_path->stats->kobj_stats.state_in_sysfs) {
+		sysfs_remove_group(&srv_path->stats->kobj_stats,
+				   &rtrs_srv_stats_attr_group);
 		kobject_del(&srv_path->stats->kobj_stats);
 		kobject_put(&srv_path->stats->kobj_stats);
+	}
+
+	if (srv_path->kobj.state_in_sysfs) {
 		sysfs_remove_group(&srv_path->kobj, &rtrs_srv_path_attr_group);
+		kobject_del(&srv_path->kobj);
 		kobject_put(&srv_path->kobj);
-
-		rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
 	}
+
+	rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
 }
Signature · 1 line
2.31.1
#8 lore raw

PATCH v2 8/8 RDMA/rtrs-srv: Remove kobject_del from rtrs_srv_destroy_once_sysfs_root_folders

In reply to Guoqing Jiang

The kobj_paths which is created dynamically by kobject_create_and_add, and per the comment above kobject_create_and_add, we only need to call kobject_put which is not same as other kobjs such as stats->kobj_stats and srv_path->kobj.

---
 drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c | 1 -
 1 file changed, 1 deletion(-)
drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c +0 -1
index da8e205ce331..c76ba29da1e2 100644
@@ -203,7 +203,6 @@ rtrs_srv_destroy_once_sysfs_root_folders(struct rtrs_srv_path *srv_path)
 
 	mutex_lock(&srv->paths_mutex);
 	if (!--srv->dev_ref) {
-		kobject_del(srv->kobj_paths);
 		kobject_put(srv->kobj_paths);
 		mutex_unlock(&srv->paths_mutex);
 		device_del(&srv->dev);
Signature · 1 line
2.31.1
#9 lore raw
In reply to Guoqing Jiang
Quoted 8 lines · Guoqing Jiang

This V2 version only fix some typos in headers
and add collect more tags from Jinpu, so all
the patches have been reviewed, thanks you!

Thanks,
Guoqing

[...]

Applied, thanks!

[1/8] RDMA/rtrs-srv: Refactor rtrs_srv_rdma_cm_handler

      https://git.kernel.org/rdma/rdma/c/d7115727e32e94
[2/8] RDMA/rtrs-srv: Refactor the handling of failure case in map_cont_bufs
      https://git.kernel.org/rdma/rdma/c/0f597ac618d04b
[3/8] RDMA/rtrs-srv: Correct the checking of ib_map_mr_sg
      https://git.kernel.org/rdma/rdma/c/102d2f70ec0999
[4/8] RDMA/rtrs-clt: Correct the checking of ib_map_mr_sg
      https://git.kernel.org/rdma/rdma/c/f5708e6699c230
[5/8] RDMA/rtrs-srv: Remove outdated comments from create_con
      https://git.kernel.org/rdma/rdma/c/a4399563356c86
[6/8] RDMA/rtrs: Clean up rtrs_rdma_dev_pd_ops
      https://git.kernel.org/rdma/rdma/c/7526198f271072
[7/8] RDMA/rtrs-srv: Fix several issues in rtrs_srv_destroy_path_files
      https://git.kernel.org/rdma/rdma/c/6af4609c18b3aa
[8/8] RDMA/rtrs-srv: Remove kobject_del from rtrs_srv_destroy_once_sysfs_root_folders
      https://git.kernel.org/rdma/rdma/c/34a046f08b62fb

Best regards,

Signature · 1 line
Leon Romanovsky <[email protected]>