adjusting mkey creations

PATCH Superseded Michael Baum 2 participants 7 messages Mar 18, 2021 – May 3, 2021 Network drivers

lore

Share & tools

lore

Michael Baum

author

2021-03-18 09:56 UTC #0 lore raw

cover adjusting mkey creations

Adjusting mkey creations to use Verbs instead of DevX API.

Michael Baum (2):
net/mlx5: workaround ASO memory region creation net/mlx5: workaround counter memory region creation

 drivers/common/mlx5/linux/mlx5_common_verbs.c |   1 -
 drivers/common/mlx5/windows/mlx5_common_os.c  |  23 +++---
 drivers/net/mlx5/linux/mlx5_os.c              |  10 ---
 drivers/net/mlx5/mlx5.c                       |  11 ++-
 drivers/net/mlx5/mlx5.h                       |  15 +---
 drivers/net/mlx5/mlx5_flow.c                  |  27 ++-----
 drivers/net/mlx5/mlx5_flow_age.c              | 106 +++++++++++---------------
 drivers/net/mlx5/windows/mlx5_os.c            |   9 ---
 8 files changed, 71 insertions(+), 131 deletions(-)

Signature · 1 line

1.8.3.1

Michael Baum

author

2021-03-18 09:56 UTC #1 lore raw

PATCH 1/2 net/mlx5: workaround ASO memory region creation

Due to kernel issue in direct MKEY creation using the DevX API for physical memory, this patch replaces the ASO MR creation to use Verbs API.

Fixes f935ed4b645a ("net/mlx5: support flow hit action for aging")

Cc [email protected]

Signed-off-by Michael Baum <[email protected]>

Acked-by Matan Azrad <[email protected]>

---
 drivers/common/mlx5/linux/mlx5_common_verbs.c |   1 -
 drivers/common/mlx5/windows/mlx5_common_os.c  |  23 +++---
 drivers/net/mlx5/mlx5.h                       |  10 +--
 drivers/net/mlx5/mlx5_flow_age.c              | 106 +++++++++++---------------
 4 files changed, 58 insertions(+), 82 deletions(-)

drivers/common/mlx5/linux/mlx5_common_verbs.c +0 -1

index 339535d..aa560f0 100644
@@ -37,7 +37,6 @@
 {
 	struct ibv_mr *ibv_mr;
 
-	memset(pmd_mr, 0, sizeof(*pmd_mr));
 	ibv_mr = mlx5_glue->reg_mr(pd, addr, length,
 				   IBV_ACCESS_LOCAL_WRITE |
 				   (haswell_broadwell_cpu ? 0 :

drivers/common/mlx5/windows/mlx5_common_os.c +13 ~~-10~~

index f2d781a..cebf42d 100644
@@ -155,23 +155,22 @@
 	struct mlx5_devx_mkey_attr mkey_attr;
 	struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd;
 	struct mlx5_hca_attr attr;
+	struct mlx5_devx_obj *mkey;
+	void *obj;
 
 	if (!pd || !addr) {
 		rte_errno = EINVAL;
 		return -1;
 	}
-	memset(pmd_mr, 0, sizeof(*pmd_mr));
 	if (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr))
 		return -1;
-	pmd_mr->addr = addr;
-	pmd_mr->len = length;
-	pmd_mr->obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, pmd_mr->addr,
-				       pmd_mr->len, IBV_ACCESS_LOCAL_WRITE);
-	if (!pmd_mr->obj)
+	obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, addr, length,
+			       IBV_ACCESS_LOCAL_WRITE);
+	if (!obj)
 		return -1;
 	mkey_attr.addr = (uintptr_t)addr;
 	mkey_attr.size = length;
-	mkey_attr.umem_id = ((struct mlx5_devx_umem *)(pmd_mr->obj))->umem_id;
+	mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id;
 	mkey_attr.pd = mlx5_pd->pdn;
 	mkey_attr.log_entity_size = 0;
 	mkey_attr.pg_access = 0;

@@ -183,11 +182,15 @@
 		mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write;
 		mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read;
 	}
-	pmd_mr->mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);
-	if (!pmd_mr->mkey) {
-		claim_zero(mlx5_os_umem_dereg(pmd_mr->obj));
+	mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);
+	if (!mkey) {
+		claim_zero(mlx5_os_umem_dereg(obj));
 		return -1;
 	}
+	pmd_mr->addr = addr;
+	pmd_mr->len = length;
+	pmd_mr->obj = obj;
+	pmd_mr->mkey = mkey;
 	pmd_mr->lkey = pmd_mr->mkey->id;
 	return 0;
 }

drivers/net/mlx5/mlx5.h +1 -9

index 14043b6..e2eb4db 100644
@@ -471,14 +471,6 @@ struct mlx5_aso_cq {
 	uint64_t errors;
 };
 
-struct mlx5_aso_devx_mr {
-	void *buf;
-	uint64_t length;
-	struct mlx5dv_devx_umem *umem;
-	struct mlx5_devx_obj *mkey;
-	bool is_indirect;
-};
-
 struct mlx5_aso_sq_elem {
 	struct mlx5_aso_age_pool *pool;
 	uint16_t burst_size;

@@ -489,7 +481,7 @@ struct mlx5_aso_sq {
 	struct mlx5_aso_cq cq;
 	struct mlx5_devx_sq sq_obj;
 	volatile uint64_t *uar_addr;
-	struct mlx5_aso_devx_mr mr;
+	struct mlx5_pmd_mr mr;
 	uint16_t pi;
 	uint32_t head;
 	uint32_t tail;

drivers/net/mlx5/mlx5_flow_age.c +44 ~~-62~~

220 lines not on this page · raw message

Signature · 1 line

1.8.3.1

Michael Baum

author

2021-03-18 09:56 UTC #2 lore raw

PATCH 2/2 net/mlx5: workaround counter memory region creation

In reply to Michael Baum

Due to kernel issue in direct MKEY creation using the DevX API for physical memory, this patch replaces the counter MR creation to use Verbs API.

Fixes 3aa279157fa0 ("net/mlx5: synchronize flow counter pool creation")

Cc [email protected]

Signed-off-by Michael Baum <[email protected]>

Acked-by Matan Azrad <[email protected]>

---
 drivers/net/mlx5/linux/mlx5_os.c   | 10 ----------
 drivers/net/mlx5/mlx5.c            | 11 +++++++----
 drivers/net/mlx5/mlx5.h            |  5 +----
 drivers/net/mlx5/mlx5_flow.c       | 27 +++++----------------------
 drivers/net/mlx5/windows/mlx5_os.c |  9 ---------
 5 files changed, 13 insertions(+), 49 deletions(-)

drivers/net/mlx5/linux/mlx5_os.c +0 ~~-10~~

index 5e3ae9f..5740214 100644
@@ -1163,16 +1163,6 @@
 			err = -err;
 			goto error;
 		}
-		/* Check relax ordering support. */
-		if (!haswell_broadwell_cpu) {
-			sh->cmng.relaxed_ordering_write =
-				config->hca_attr.relaxed_ordering_write;
-			sh->cmng.relaxed_ordering_read =
-				config->hca_attr.relaxed_ordering_read;
-		} else {
-			sh->cmng.relaxed_ordering_read = 0;
-			sh->cmng.relaxed_ordering_write = 0;
-		}
 		sh->rq_ts_format = config->hca_attr.rq_ts_format;
 		sh->sq_ts_format = config->hca_attr.sq_ts_format;
 		sh->qp_ts_format = config->hca_attr.qp_ts_format;

drivers/net/mlx5/mlx5.c +7 -4

index abd7ff7..fb58631 100644
@@ -469,17 +469,20 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 /**
  * Destroy all the resources allocated for a counter memory management.
  *
+ * @param[in] sh
+ *   Pointer to mlx5_dev_ctx_shared object to free.
  * @param[in] mng
  *   Pointer to the memory management structure.
  */
 static void
-mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng)
+mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh,
+				       struct mlx5_counter_stats_mem_mng *mng)
 {
 	uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data;
 
 	LIST_REMOVE(mng, next);
-	claim_zero(mlx5_devx_cmd_destroy(mng->dm));
-	claim_zero(mlx5_os_umem_dereg(mng->umem));
+	sh->share_cache.dereg_mr_cb(&mng->dm);
+	memset(&mng->dm, 0, sizeof(mng->dm));
 	mlx5_free(mem);
 }

@@ -533,7 +536,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 	}
 	mng = LIST_FIRST(&sh->cmng.mem_mngs);
 	while (mng) {
-		mlx5_flow_destroy_counter_stat_mem_mng(mng);
+		mlx5_flow_destroy_counter_stat_mem_mng(sh, mng);
 		mng = LIST_FIRST(&sh->cmng.mem_mngs);
 	}
 	memset(&sh->cmng, 0, sizeof(sh->cmng));

drivers/net/mlx5/mlx5.h +1 -4

index e2eb4db..8e8727a 100644
@@ -422,8 +422,7 @@ struct mlx5_flow_counter_pool {
 struct mlx5_counter_stats_mem_mng {
 	LIST_ENTRY(mlx5_counter_stats_mem_mng) next;
 	struct mlx5_counter_stats_raw *raws;
-	struct mlx5_devx_obj *dm;
-	void *umem;
+	struct mlx5_pmd_mr dm;
 };
 
 /* Raw memory structure for the counter statistics values of a pool. */

@@ -454,8 +453,6 @@ struct mlx5_flow_counter_mng {
 	uint8_t pending_queries;
 	uint16_t pool_index;
 	uint8_t query_thread_on;
-	bool relaxed_ordering_read;
-	bool relaxed_ordering_write;
 	bool counter_fallback; /* Use counter fallback management. */
 	LIST_HEAD(mem_mngs, mlx5_counter_stats_mem_mng) mem_mngs;
 	LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;

drivers/net/mlx5/mlx5_flow.c +5 ~~-22~~

index d46fc33..afa8ab4 100644
@@ -6717,7 +6717,6 @@ struct mlx5_meter_domains_infos *
 static int
 mlx5_flow_create_counter_stat_mem_mng(struct mlx5_dev_ctx_shared *sh)
 {
-	struct mlx5_devx_mkey_attr mkey_attr;
 	struct mlx5_counter_stats_mem_mng *mem_mng;
 	volatile struct flow_counter_stats *raw_data;
 	int raws_n = MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES;

@@ -6727,6 +6726,7 @@ struct mlx5_meter_domains_infos *
 			sizeof(struct mlx5_counter_stats_mem_mng);
 	size_t pgsize = rte_mem_page_size();
 	uint8_t *mem;
+	int ret;
 	int i;
 
 	if (pgsize == (size_t)-1) {

@@ -6741,26 +6741,9 @@ struct mlx5_meter_domains_infos *
 	}
 	mem_mng = (struct mlx5_counter_stats_mem_mng *)(mem + size) - 1;
 	size = sizeof(*raw_data) * MLX5_COUNTERS_PER_POOL * raws_n;
-	mem_mng->umem = mlx5_os_umem_reg(sh->ctx, mem, size,
-						 IBV_ACCESS_LOCAL_WRITE);
-	if (!mem_mng->umem) {
-		rte_errno = errno;
-		mlx5_free(mem);
-		return -rte_errno;
-	}
-	mkey_attr.addr = (uintptr_t)mem;
-	mkey_attr.size = size;
-	mkey_attr.umem_id = mlx5_os_get_umem_id(mem_mng->umem);
-	mkey_attr.pd = sh->pdn;
-	mkey_attr.log_entity_size = 0;
-	mkey_attr.pg_access = 0;
-	mkey_attr.klm_array = NULL;
-	mkey_attr.klm_num = 0;
-	mkey_attr.relaxed_ordering_write = sh->cmng.relaxed_ordering_write;
-	mkey_attr.relaxed_ordering_read = sh->cmng.relaxed_ordering_read;
-	mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr);
-	if (!mem_mng->dm) {
-		mlx5_os_umem_dereg(mem_mng->umem);
+	ret = sh->share_cache.reg_mr_cb(sh->pd, (void *)mem, size,
+					&mem_mng->dm);
+	if (ret) {
 		rte_errno = errno;
 		mlx5_free(mem);
 		return -rte_errno;

@@ -6879,7 +6862,7 @@ struct mlx5_meter_domains_infos *
 	ret = mlx5_devx_cmd_flow_counter_query(pool->min_dcs, 0,
 					       MLX5_COUNTERS_PER_POOL,
 					       NULL, NULL,
-					       pool->raw_hw->mem_mng->dm->id,
+					       pool->raw_hw->mem_mng->dm.lkey,
 					       (void *)(uintptr_t)
 					       pool->raw_hw->data,
 					       sh->devx_comp,

drivers/net/mlx5/windows/mlx5_os.c +0 -9

index 6f39276..3e72e36 100644
@@ -466,15 +466,6 @@
 			err = -err;
 			goto error;
 		}
-		/* Check relax ordering support. */
-		sh->cmng.relaxed_ordering_read = 0;
-		sh->cmng.relaxed_ordering_write = 0;
-		if (!haswell_broadwell_cpu) {
-			sh->cmng.relaxed_ordering_write =
-				config->hca_attr.relaxed_ordering_write;
-			sh->cmng.relaxed_ordering_read =
-				config->hca_attr.relaxed_ordering_read;
-		}
 	}
 	if (config->devx) {
 		uint32_t reg[MLX5_ST_SZ_DW(register_mtutc)];

Signature · 1 line

1.8.3.1

Raslan Darawsheh

2021-03-25 08:52 UTC #3 lore raw

In reply to Michael Baum

Hi,

Quoted 24 lines

-----Original Message-----
From: Michael Baum <[email protected]>
Sent: Thursday, March 18, 2021 11:56 AM
To: [email protected]
Cc: Matan Azrad <[email protected]>; Raslan Darawsheh <[email protected]>; Slava Ovsiienko <[email protected]>
Subject: [PATCH 0/2] adjusting mkey creations
Adjusting mkey creations to use Verbs instead of DevX API.
Michael Baum (2):
net/mlx5: workaround ASO memory region creation net/mlx5: workaround counter memory region creation
 drivers/common/mlx5/linux/mlx5_common_verbs.c |   1 -
 drivers/common/mlx5/windows/mlx5_common_os.c  |  23 +++---
 drivers/net/mlx5/linux/mlx5_os.c              |  10 ---
 drivers/net/mlx5/mlx5.c                       |  11 ++-
 drivers/net/mlx5/mlx5.h                       |  15 +---
 drivers/net/mlx5/mlx5_flow.c                  |  27 ++-----
 drivers/net/mlx5/mlx5_flow_age.c              | 106 +++++++++++---------------
 drivers/net/mlx5/windows/mlx5_os.c            |   9 ---
 8 files changed, 71 insertions(+), 131 deletions(-)

Quoted 2 lines

--
1.8.3.1

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh

Raslan Darawsheh

2021-03-29 08:14 UTC #4 lore raw

Hi,

Quoted 34 lines

-----Original Message-----
From: Raslan Darawsheh
Sent: Thursday, March 25, 2021 10:53 AM
To: Michael Baum <[email protected]>; [email protected]
Cc: Matan Azrad <[email protected]>; Slava Ovsiienko <[email protected]>
Subject: RE: [PATCH 0/2] adjusting mkey creations
Hi,
Quoted 24 lines
-----Original Message-----
From: Michael Baum <[email protected]>
Sent: Thursday, March 18, 2021 11:56 AM
To: [email protected]
Cc: Matan Azrad <[email protected]>; Raslan Darawsheh <[email protected]>; Slava Ovsiienko <[email protected]>
Subject: [PATCH 0/2] adjusting mkey creations
Adjusting mkey creations to use Verbs instead of DevX API.
Michael Baum (2):
net/mlx5: workaround ASO memory region creation net/mlx5: workaround counter memory region creation
 drivers/common/mlx5/linux/mlx5_common_verbs.c |   1 -
 drivers/common/mlx5/windows/mlx5_common_os.c  |  23 +++---
 drivers/net/mlx5/linux/mlx5_os.c              |  10 ---
 drivers/net/mlx5/mlx5.c                       |  11 ++-
 drivers/net/mlx5/mlx5.h                       |  15 +---
 drivers/net/mlx5/mlx5_flow.c                  |  27 ++-----
 drivers/net/mlx5/mlx5_flow_age.c              | 106 +++++++++++---------------
 drivers/net/mlx5/windows/mlx5_os.c            |   9 ---
 8 files changed, 71 insertions(+), 131 deletions(-)

Quoted 7 lines

Quoted 2 lines
--
1.8.3.1
Series applied to next-net-mlx,
Kindest regards,
Raslan Darawsheh

Due to some regression which was found in these patches series we'll remove them from next-net-mlx, to have another version which has the fix for this issue.

Kindest regards,
Raslan Darawsheh

Michael Baum

author

2021-04-26 12:48 UTC #5 lore raw

PATCH v2 1/1 net/mlx5: workaround ASO memory region creation

In reply to Michael Baum

Due to kernel issue in direct MKEY creation using the DevX API for physical memory, this patch replaces the ASO MR creation to use Verbs API.

Fixes f935ed4b645a ("net/mlx5: support flow hit action for aging")

Cc [email protected]

Signed-off-by Michael Baum <[email protected]>

Acked-by Matan Azrad <[email protected]>

---
v2:
The second patch in the series did not work due to a FW issue, this issue does not exist in this patch.

drivers/common/mlx5/linux/mlx5_common_verbs.c | 1 -

 drivers/common/mlx5/windows/mlx5_common_os.c  | 23 ++++---
 drivers/net/mlx5/mlx5.h                       | 10 +--
 drivers/net/mlx5/mlx5_flow_aso.c              | 92 +++++++++++----------------
 4 files changed, 52 insertions(+), 74 deletions(-)

drivers/common/mlx5/linux/mlx5_common_verbs.c +0 -1

index 339535d..aa560f0 100644
@@ -37,7 +37,6 @@
 {
 	struct ibv_mr *ibv_mr;
 
-	memset(pmd_mr, 0, sizeof(*pmd_mr));
 	ibv_mr = mlx5_glue->reg_mr(pd, addr, length,
 				   IBV_ACCESS_LOCAL_WRITE |
 				   (haswell_broadwell_cpu ? 0 :

drivers/common/mlx5/windows/mlx5_common_os.c +13 ~~-10~~

index f2d781a..cebf42d 100644
@@ -155,23 +155,22 @@
 	struct mlx5_devx_mkey_attr mkey_attr;
 	struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd;
 	struct mlx5_hca_attr attr;
+	struct mlx5_devx_obj *mkey;
+	void *obj;
 
 	if (!pd || !addr) {
 		rte_errno = EINVAL;
 		return -1;
 	}
-	memset(pmd_mr, 0, sizeof(*pmd_mr));
 	if (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr))
 		return -1;
-	pmd_mr->addr = addr;
-	pmd_mr->len = length;
-	pmd_mr->obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, pmd_mr->addr,
-				       pmd_mr->len, IBV_ACCESS_LOCAL_WRITE);
-	if (!pmd_mr->obj)
+	obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, addr, length,
+			       IBV_ACCESS_LOCAL_WRITE);
+	if (!obj)
 		return -1;
 	mkey_attr.addr = (uintptr_t)addr;
 	mkey_attr.size = length;
-	mkey_attr.umem_id = ((struct mlx5_devx_umem *)(pmd_mr->obj))->umem_id;
+	mkey_attr.umem_id = ((struct mlx5_devx_umem *)(obj))->umem_id;
 	mkey_attr.pd = mlx5_pd->pdn;
 	mkey_attr.log_entity_size = 0;
 	mkey_attr.pg_access = 0;

@@ -183,11 +182,15 @@
 		mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write;
 		mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read;
 	}
-	pmd_mr->mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);
-	if (!pmd_mr->mkey) {
-		claim_zero(mlx5_os_umem_dereg(pmd_mr->obj));
+	mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr);
+	if (!mkey) {
+		claim_zero(mlx5_os_umem_dereg(obj));
 		return -1;
 	}
+	pmd_mr->addr = addr;
+	pmd_mr->len = length;
+	pmd_mr->obj = obj;
+	pmd_mr->mkey = mkey;
 	pmd_mr->lkey = pmd_mr->mkey->id;
 	return 0;
 }

drivers/net/mlx5/mlx5.h +1 -9

index 378b68e..a29b8d6 100644
@@ -483,14 +483,6 @@ struct mlx5_aso_cq {
 	uint64_t errors;
 };
 
-struct mlx5_aso_devx_mr {
-	void *buf;
-	uint64_t length;
-	struct mlx5dv_devx_umem *umem;
-	struct mlx5_devx_obj *mkey;
-	bool is_indirect;
-};
-
 struct mlx5_aso_sq_elem {
 	union {
 		struct {

@@ -507,7 +499,7 @@ struct mlx5_aso_sq {
 	struct mlx5_aso_cq cq;
 	struct mlx5_devx_sq sq_obj;
 	volatile uint64_t *uar_addr;
-	struct mlx5_aso_devx_mr mr;
+	struct mlx5_pmd_mr mr;
 	uint16_t pi;
 	uint32_t head;
 	uint32_t tail;

drivers/net/mlx5/mlx5_flow_aso.c +38 ~~-54~~

185 lines not on this page · raw message

Signature · 1 line

1.8.3.1

Raslan Darawsheh

2021-05-03 10:12 UTC #6 lore raw

Re: [dpdk-dev] [PATCH v2] net/mlx5: workaround ASO memory region creation

Hi,

Quoted 23 lines

-----Original Message-----
From: Michael Baum <[email protected]>
Sent: Monday, April 26, 2021 3:48 PM
To: [email protected]
Cc: Matan Azrad <[email protected]>; Raslan Darawsheh <[email protected]>; Slava Ovsiienko <[email protected]>; [email protected]
Subject: [PATCH v2] net/mlx5: workaround ASO memory region creation
Due to kernel issue in direct MKEY creation using the DevX API for physical memory, this patch replaces the ASO MR creation to use Verbs API.
Fixes f935ed4b645a ("net/mlx5: support flow hit action for aging")
Cc [email protected]
Signed-off-by Michael Baum <[email protected]>
Acked-by Matan Azrad <[email protected]>
---
v2:
The second patch in the series did not work due to a FW issue, this issue does not exist in this patch.

Quoted 6 lines

drivers/common/mlx5/linux/mlx5_common_verbs.c | 1 -

 drivers/common/mlx5/windows/mlx5_common_os.c  | 23 ++++---
 drivers/net/mlx5/mlx5.h                       | 10 +--
 drivers/net/mlx5/mlx5_flow_aso.c              | 92 +++++++++++----------------
 4 files changed, 52 insertions(+), 74 deletions(-)

Patch applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh