Skip to contentSkip to conversation

ASO meter sharing support

PATCH In review Shun Hao 2 participants 6 messages Jul 2, 2021 – Jul 6, 2021 Network drivers

lore
Share & tools
lore

Shun Hao

author
#0 lore raw

cover ASO meter sharing support

This adds several fixes in ASO meter. With these fixes, meter created on one port can be shared by other ports.

Shun Hao (4):
net/mlx5: use meter profile lookup table
net/mlx5: fix meter policy ID table container
net/mlx5: fix meter flow direction check
net/mlx5: fix meter policy flow match item

 doc/guides/nics/mlx5.rst           |  8 +++
 drivers/net/mlx5/linux/mlx5_os.c   |  6 +-
 drivers/net/mlx5/mlx5.c            |  2 -
 drivers/net/mlx5/mlx5.h            |  7 ++-
 drivers/net/mlx5/mlx5_flow.c       | 83 ++++++++++++++++-----------
 drivers/net/mlx5/mlx5_flow_dv.c    | 58 ++++++++++++++++---
 drivers/net/mlx5/mlx5_flow_meter.c | 91 +++++++++++++++++-------------
 drivers/net/mlx5/windows/mlx5_os.c |  6 +-
 8 files changed, 172 insertions(+), 89 deletions(-)
Signature · 1 line
2.20.0
author
#1 lore raw

PATCH 1/4 net/mlx5: use meter profile lookup table

Currently a list is used to save all meter profile ids, which is not efficient when looking up profile from huge mount of profiles.

This changes to use a l3 table instead to save meter profile ids, so as to improve the lookup performance.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  6 +++-
 drivers/net/mlx5/mlx5.h            |  2 +-
 drivers/net/mlx5/mlx5_flow_meter.c | 53 ++++++++++++++++++++----------
 drivers/net/mlx5/windows/mlx5_os.c |  6 +++-
 4 files changed, 47 insertions(+), 20 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -1
index 92b3009786..b3f9e392ab 100644
@@ -1570,7 +1570,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
-	TAILQ_INIT(&priv->flow_meter_profiles);
+	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
+	if (!priv->mtr_profile_tbl)
+		goto error;
 	/* Hint libmlx5 to use PMD allocator for data plane resources */
 	mlx5_glue->dv_set_context_attr(sh->ctx,
 			MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
@@ -1713,6 +1715,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
 		if (eth_dev && priv->drop_queue.hrxq)
 			mlx5_drop_action_destroy(eth_dev);
+		if (priv->mtr_profile_tbl)
+			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		mlx5_cache_list_destroy(&priv->hrxqs);
drivers/net/mlx5/mlx5.h +1 -1
index 1b2dc8f815..0226327bc3 100644
@@ -1373,8 +1373,8 @@ struct mlx5_priv {
 	/* Hash table of Rx metadata register copy table. */
 	uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */
 	uint8_t mtr_color_reg; /* Meter color match REG_C. */
-	struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */
 	struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */
+	struct mlx5_l3t_tbl *mtr_profile_tbl; /* Meter index lookup table. */
 	struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */
 	uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */
 	uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
drivers/net/mlx5/mlx5_flow_meter.c +36 -17

92 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +5 -1
index 3fe3f55f49..e30b682822 100644
@@ -566,7 +566,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
-	TAILQ_INIT(&priv->flow_meter_profiles);
+	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
+	if (!priv->mtr_profile_tbl)
+		goto error;
 	/* Bring Ethernet device up. */
 	DRV_LOG(DEBUG, "port %u forcing Ethernet interface up.",
 		eth_dev->data->port_id);
@@ -644,6 +646,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	return eth_dev;
 error:
 	if (priv) {
+		if (priv->mtr_profile_tbl)
+			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		mlx5_free(priv);
Signature · 1 line
2.20.0
author
#2 lore raw

PATCH 2/4 net/mlx5: fix meter policy ID table container

In reply to Shun Hao

The meter policy handlers are managed by user IDs and the driver used l3 table in order to map the user ID to the internal driver handler of the policy.

The l3 table was wrongly saved in the shared device structure which manages all the switch domain ports what made the user IDs shared between different ethdev ports.

Move the policy l3 table to be per port by saving it in the port private structure.

Fixes afb4aa4f122 ("net/mlx5: support meter policy operations")
---
 drivers/net/mlx5/mlx5.c            |  2 --
 drivers/net/mlx5/mlx5.h            |  3 +--
 drivers/net/mlx5/mlx5_flow_meter.c | 38 ++++++++++++------------------
 3 files changed, 16 insertions(+), 27 deletions(-)
drivers/net/mlx5/mlx5.c +0 -2
index cf1815cb74..5f10a45c02 100644
@@ -600,8 +600,6 @@ mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh)
 		if (sh->meter_aso_en) {
 			rte_spinlock_init(&sh->mtrmng->pools_mng.mtrsl);
 			LIST_INIT(&sh->mtrmng->pools_mng.meters);
-			sh->mtrmng->policy_idx_tbl =
-				mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 		}
 		sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 	}
drivers/net/mlx5/mlx5.h +1 -2
index 0226327bc3..6eae7b6fd7 100644
@@ -870,8 +870,6 @@ struct mlx5_flow_mtr_mng {
 	/* Default policy id. */
 	uint32_t def_policy_ref_cnt;
 	/** def_policy meter use count. */
-	struct mlx5_l3t_tbl *policy_idx_tbl;
-	/* Policy index lookup table. */
 	struct mlx5_flow_tbl_resource *drop_tbl[MLX5_MTR_DOMAIN_MAX];
 	/* Meter drop table. */
 	struct mlx5_flow_dv_matcher *
@@ -1375,6 +1373,7 @@ struct mlx5_priv {
 	uint8_t mtr_color_reg; /* Meter color match REG_C. */
 	struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */
 	struct mlx5_l3t_tbl *mtr_profile_tbl; /* Meter index lookup table. */
+	struct mlx5_l3t_tbl *policy_idx_tbl; /* Policy index lookup table. */
 	struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */
 	uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */
 	uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
drivers/net/mlx5/mlx5_flow_meter.c +15 -23
index 87c5782956..6f962a8d52 100644
@@ -512,11 +512,9 @@ mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 	struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 	union mlx5_l3t_data data;
 
-	if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
-		!priv->sh->mtrmng->policy_idx_tbl)
+	if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 		return NULL;
-	if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
-				policy_id, &data) ||
+	if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 				!data.dword)
 		return NULL;
 	if (policy_idx)
@@ -524,8 +522,7 @@ mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 	sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 					data.dword);
 	/* Remove reference taken by the mlx5_l3t_get_entry. */
-	mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
-				policy_id);
+	mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 	if (sub_policy)
 		if (sub_policy->main_policy_id)
 			return sub_policy->main_policy;
@@ -605,9 +602,8 @@ __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 			}
 		}
 	}
-	if (priv->sh->mtrmng->policy_idx_tbl && clear_l3t) {
-		if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
-					policy_id)) {
+	if (priv->policy_idx_tbl && clear_l3t) {
+		if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 			rte_spinlock_unlock(&mtr_policy->sl);
 			return -rte_mtr_error_set(error, ENOTSUP,
 				RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
@@ -770,14 +766,12 @@ mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 			goto policy_add_err;
 	}
 	data.dword = policy_idx;
-	if (!priv->sh->mtrmng->policy_idx_tbl) {
-		priv->sh->mtrmng->policy_idx_tbl =
-				mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
-		if (!priv->sh->mtrmng->policy_idx_tbl)
+	if (!priv->policy_idx_tbl) {
+		priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
+		if (!priv->policy_idx_tbl)
 			goto policy_add_err;
 	}
-	if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
-				policy_id, &data))
+	if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 		goto policy_add_err;
 	return 0;
 policy_add_err:
@@ -1840,9 +1834,8 @@ mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
 
 	if (!priv->mtr_en)
 		return;
-	if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
-		MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
-					i, entry) {
+	if (priv->policy_idx_tbl) {
+		MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
 			policy_idx = *(uint32_t *)entry;
 			sub_policy = mlx5_ipool_get
 				(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
@@ -1908,9 +1901,8 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
 				NULL, "MTR object meter profile invalid.");
 		}
 	}
-	if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
-		MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
-					i, entry) {
+	if (priv->policy_idx_tbl) {
+		MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
 			policy_idx = *(uint32_t *)entry;
 			sub_policy = mlx5_ipool_get
 				(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
@@ -1931,8 +1923,8 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
 						"meter policy invalid.");
 			mlx5_free(sub_policy->main_policy);
 		}
-		mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
-		priv->sh->mtrmng->policy_idx_tbl = NULL;
+		mlx5_l3t_destroy(priv->policy_idx_tbl);
+		priv->policy_idx_tbl = NULL;
 	}
 	if (priv->mtr_profile_tbl) {
 		MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
Signature · 1 line
2.20.0
author
#3 lore raw

PATCH 3/4 net/mlx5: fix meter flow direction check

In reply to Shun Hao

When preparing prefix flow using ASO meter, if it's tx flow, need to make meter action the first one.

Currently the check of flow direction in switch domain is incorrect that it checks the flow dev port only.

This adds the fix for the check that if there's port_id match item in flow, use that port_id as src port to determine flow direction.

Fixes c99b4f8bc2f1 ("net/mlx5: support ASO meter action")
---
 drivers/net/mlx5/mlx5_flow.c | 83 ++++++++++++++++++++++--------------
 1 file changed, 50 insertions(+), 33 deletions(-)
drivers/net/mlx5/mlx5_flow.c +50 -33
index 3b7c94d92f..c27f6197a0 100644
@@ -4740,12 +4740,12 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
 	struct mlx5_rte_flow_item_tag *tag_item_spec;
 	struct mlx5_rte_flow_item_tag *tag_item_mask;
 	uint32_t tag_id = 0;
-	bool copy_vlan = false;
+	struct rte_flow_item *vlan_item_dst = NULL;
+	const struct rte_flow_item *vlan_item_src = NULL;
 	struct rte_flow_action *hw_mtr_action;
 	struct rte_flow_action *action_pre_head = NULL;
-	bool mtr_first = priv->sh->meter_aso_en &&
-			(attr->egress ||
-			(attr->transfer && priv->representor_id != UINT16_MAX));
+	int32_t flow_src_port = priv->representor_id;
+	bool mtr_first;
 	uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
 	uint8_t mtr_reg_bits = priv->mtr_reg_share ?
 				MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
@@ -4754,6 +4754,42 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
 	uint8_t flow_id_bits = 0;
 	int shift;
 
+	/* Prepare the suffix subflow items. */
+	tag_item = sfx_items++;
+	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
+		struct mlx5_priv *port_priv;
+		const struct rte_flow_item_port_id *pid_v;
+		int item_type = items->type;
+
+		switch (item_type) {
+		case RTE_FLOW_ITEM_TYPE_PORT_ID:
+			pid_v = items->spec;
+			MLX5_ASSERT(pid_v);
+			port_priv = mlx5_port_to_eswitch_info(pid_v->id, false);
+			if (!port_priv)
+				return rte_flow_error_set(error,
+						rte_errno,
+						RTE_FLOW_ERROR_TYPE_ITEM_SPEC,
+						pid_v,
+						"Failed to get port info.");
+			flow_src_port = port_priv->representor_id;
+			memcpy(sfx_items, items, sizeof(*sfx_items));
+			sfx_items++;
+			break;
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+			/* Determine if copy vlan item below. */
+			vlan_item_src = items;
+			vlan_item_dst = sfx_items++;
+			vlan_item_dst->type = RTE_FLOW_ITEM_TYPE_VOID;
+			break;
+		default:
+			break;
+		}
+	}
+	sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
+	sfx_items++;
+	mtr_first = priv->sh->meter_aso_en &&
+		(attr->egress || (attr->transfer && flow_src_port != 0xffff));
 	/* For ASO meter, meter must be before tag in TX direction. */
 	if (mtr_first) {
 		action_pre_head = actions_pre++;
@@ -4790,7 +4826,16 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
 			break;
 		case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
 		case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
-			copy_vlan = true;
+			if (vlan_item_dst && vlan_item_src) {
+				memcpy(vlan_item_dst, vlan_item_src,
+					sizeof(*vlan_item_dst));
+				/*
+				 * Convert to internal match item, it is used
+				 * for vlan push and set vid.
+				 */
+				vlan_item_dst->type = (enum rte_flow_item_type)
+						MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
+			}
 			break;
 		default:
 			break;
@@ -4862,34 +4907,6 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
 		if (flow_id_bits > priv->sh->mtrmng->max_mtr_flow_bits)
 			priv->sh->mtrmng->max_mtr_flow_bits = flow_id_bits;
 	}
-	/* Prepare the suffix subflow items. */
-	tag_item = sfx_items++;
-	for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
-		int item_type = items->type;
-
-		switch (item_type) {
-		case RTE_FLOW_ITEM_TYPE_PORT_ID:
-			memcpy(sfx_items, items, sizeof(*sfx_items));
-			sfx_items++;
-			break;
-		case RTE_FLOW_ITEM_TYPE_VLAN:
-			if (copy_vlan) {
-				memcpy(sfx_items, items, sizeof(*sfx_items));
-				/*
-				 * Convert to internal match item, it is used
-				 * for vlan push and set vid.
-				 */
-				sfx_items->type = (enum rte_flow_item_type)
-						  MLX5_RTE_FLOW_ITEM_TYPE_VLAN;
-				sfx_items++;
-			}
-			break;
-		default:
-			break;
-		}
-	}
-	sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
-	sfx_items++;
 	/* Build tag actions and items for meter_id/meter flow_id. */
 	set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
 	tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
Signature · 1 line
2.20.0
author
#4 lore raw

PATCH 4/4 net/mlx5: fix meter policy flow match item

In reply to Shun Hao

Currently when creating meter policy, a src port_id match item will always be added in switch domain. So if one meter is used by another port, it will not work correctly.

This issue is solved:
1. If policy fate action is port_id, add the src port_id match item, and the meter cannot be shared by another port.
2. If policy fate action isn't port_id, don't add the src port_id match, meter can be shared by another port.

This fix enables one meter being shared by different ports. User can create a meter flow using a port_id match item to make this meter shared by other port.

Fixes afb4aa4f122 ("net/mlx5: support meter policy operations")
---
 doc/guides/nics/mlx5.rst        |  8 +++++
 drivers/net/mlx5/mlx5.h         |  2 ++
 drivers/net/mlx5/mlx5_flow_dv.c | 58 ++++++++++++++++++++++++++++-----
 3 files changed, 59 insertions(+), 9 deletions(-)
doc/guides/nics/mlx5.rst +8 -0
index eb44a070b1..57e7d17b87 100644
@@ -1912,3 +1912,11 @@ all flows with assistance of external tools.
    .. code-block:: console
 
        mlx_steering_dump.py -f <output_file> -flowptr <flow_ptr>
+
+How to share a meter between ports in the same switch domain
+------------------------------------------------------------
+
+This section demonstrates how to use the shared meter. A meter M can be created
+on port X and to be shared with a port Y on the same switch domain by the next way:
+
+        flow create X ingress transfer pattern eth / port_id id is Y / end actions meter mtr_id M / end
drivers/net/mlx5/mlx5.h +2 -0
index 6eae7b6fd7..0f4b239142 100644
@@ -682,6 +682,8 @@ struct mlx5_meter_policy_action_container {
 
 /* Flow meter policy parameter structure. */
 struct mlx5_flow_meter_policy {
+	struct rte_eth_dev *dev;
+	/* The port dev on which policy is created. */
 	uint32_t is_rss:1;
 	/* Is RSS policy table. */
 	uint32_t ingress:1;
drivers/net/mlx5/mlx5_flow_dv.c +49 -9

160 lines not on this page · raw message

Signature · 1 line
2.20.0
#5 lore raw
In reply to Shun Hao

Hi,

Quoted 30 lines

-----Original Message-----
From: Shun Hao <[email protected]>
Sent: Friday, July 2, 2021 12:15 PM
To: Ori Kam <[email protected]>; Slava Ovsiienko <[email protected]>; Matan Azrad <[email protected]>; Shahaf Shuler <[email protected]>
Cc: [email protected]; NBU-Contact-Thomas Monjalon <[email protected]>; Raslan Darawsheh <[email protected]>; Roni Bar Yanai <[email protected]>
Subject: [PATCH v1 0/4] ASO meter sharing support

This adds several fixes in ASO meter. With these fixes, meter created on one port can be shared by other ports.

Shun Hao (4):
net/mlx5: use meter profile lookup table
net/mlx5: fix meter policy ID table container
net/mlx5: fix meter flow direction check
net/mlx5: fix meter policy flow match item

 doc/guides/nics/mlx5.rst           |  8 +++
 drivers/net/mlx5/linux/mlx5_os.c   |  6 +-
 drivers/net/mlx5/mlx5.c            |  2 -
 drivers/net/mlx5/mlx5.h            |  7 ++-
 drivers/net/mlx5/mlx5_flow.c       | 83 ++++++++++++++++-----------
 drivers/net/mlx5/mlx5_flow_dv.c    | 58 ++++++++++++++++---
 drivers/net/mlx5/mlx5_flow_meter.c | 91 +++++++++++++++++-------------
drivers/net/mlx5/windows/mlx5_os.c |  6 +-
 8 files changed, 172 insertions(+), 89 deletions(-)
Quoted 2 lines

--
2.20.0

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh