Skip to contentSkip to conversation

net/mlx5: support match with switch manager

PATCH Superseded Suanming Mou 3 participants 8 messages May 31, 2024 – Jun 19, 2024

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 net/mlx5: support match with switch manager

Currently, in switch mode, mlx5 PMD only supports match with dedicate vport. There is a usercase which user may want to offload the rules only to match with all the pkt sent by application not from vport.

Since the port_id info of pkt sent by application is switch manager, and kernel driver has exposed the switch manager register value, this commit adds the support of register matching for switch manager.

---
 doc/guides/rel_notes/release_24_07.rst |  1 +
 drivers/common/mlx5/linux/meson.build  |  2 ++
 drivers/net/mlx5/linux/mlx5_os.c       | 12 ++++++++++++
 drivers/net/mlx5/mlx5.h                | 14 ++++++++++++++
 drivers/net/mlx5/mlx5_flow.h           |  6 ------
 drivers/net/mlx5/mlx5_flow_dv.c        | 24 +++++++++++++++++++-----
 6 files changed, 48 insertions(+), 11 deletions(-)
doc/guides/rel_notes/release_24_07.rst +1 -0
index 46efc04eac..389178acb0 100644
@@ -84,6 +84,7 @@ New Features
 * **Updated NVIDIA mlx5 driver.**
 
   * Added match with Tx queue.
+  * Added match with switch manager.
 
 
 Removed Items
drivers/common/mlx5/linux/meson.build +2 -0
index cdee40c553..82e8046e0c 100644
@@ -201,6 +201,8 @@ has_sym_args = [
             'mlx5dv_create_steering_anchor'],
         [ 'HAVE_IBV_FORK_UNNEEDED', 'infiniband/verbs.h',
             'ibv_is_fork_initialized'],
+        [ 'HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0', 'infiniband/mlx5dv.h',
+            'MLX5DV_CONTEXT_MASK_REG_C0' ],
 ]
 if  libmtcr_ul_found
     has_sym_args += [
drivers/net/mlx5/linux/mlx5_os.c +12 -0
index bb566ea236..24ea58fd01 100644
@@ -160,6 +160,9 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
 #endif
 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
+#endif
+#ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_REG_C0;
 #endif
 	err = mlx5_glue->dv_query_device(cdev->ctx, &dv_attr);
 	if (err) {
@@ -374,6 +377,15 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
 					hca_attr->scatter_fcs_w_decap_disable;
 	sh->dev_cap.rq_delay_drop_en = hca_attr->rq_delay_drop;
 	mlx5_rt_timestamp_config(sh, hca_attr);
+#ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_REG_C0) {
+		sh->dev_cap.esw_info.regc_value = dv_attr.reg_c0.value;
+		sh->dev_cap.esw_info.regc_mask = dv_attr.reg_c0.mask;
+	}
+#else
+	sh->dev_cap.esw_info.regc_value = 0;
+	sh->dev_cap.esw_info.regc_mask = 0;
+#endif
 	return 0;
 }
 
drivers/net/mlx5/mlx5.h +14 -0
index 5b23043b8b..13718273dc 100644
@@ -141,6 +141,19 @@ struct mlx5_flow_cb_ctx {
 	void *data2;
 };
 
+struct flow_hw_port_info {
+	uint32_t regc_mask;
+	uint32_t regc_value;
+	uint32_t is_wire:1;
+	uint32_t direction:2;
+};
+
+enum mlx5_vport_direction {
+	MLX5_VPORT_DIRECTION_ANY = 0,
+	MLX5_VPORT_DIRECTION_NORTH,
+	MLX5_VPORT_DIRECTION_SOUTH,
+};
+
 /* Device capabilities structure which isn't changed in any stage. */
 struct mlx5_dev_cap {
 	int max_cq; /* Maximum number of supported CQs */
@@ -184,6 +197,7 @@ struct mlx5_dev_cap {
 		/* Log min WQE size, (size of single stride)*(num of strides).*/
 	} mprq; /* Capability for Multi-Packet RQ. */
 	char fw_ver[64]; /* Firmware version of this device. */
+	struct flow_hw_port_info esw_info; /* E-switch manager reg_c0. */
 };
 
 #define MLX5_MPESW_PORT_INVALID (-1)
drivers/net/mlx5/mlx5_flow.h +0 -6
index 9a359da042..0a6d33f8ee 100644
@@ -1938,12 +1938,6 @@ struct mlx5_hl_data {
 	uint32_t dw_mask;
 };
 
-struct flow_hw_port_info {
-	uint32_t regc_mask;
-	uint32_t regc_value;
-	uint32_t is_wire:1;
-};
-
 extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
 
 /*
drivers/net/mlx5/mlx5_flow_dv.c +19 -5
index a834b3e2e0..5477eba4a0 100644
@@ -10837,17 +10837,31 @@ flow_dv_translate_item_represented_port(struct rte_eth_dev *dev, void *key,
 	const struct rte_flow_item_ethdev *pid_m = item ? item->mask : NULL;
 	const struct rte_flow_item_ethdev *pid_v = item ? item->spec : NULL;
 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
-	struct mlx5_priv *priv;
+	struct mlx5_priv *priv = dev->data->dev_private;
 	uint16_t mask, id;
 	uint32_t vport_meta;
+	bool vport_match = false;
 
 	MLX5_ASSERT(wks);
+#ifndef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	if (priv->sh->config.dv_flow_en == 2)
+		vport_match = true;
+#endif
 	if (!pid_m && !pid_v)
 		return 0;
 	if (pid_v && pid_v->port_id == UINT16_MAX) {
-		flow_dv_translate_item_source_vport(key,
-			key_type & MLX5_SET_MATCHER_V ?
-			mlx5_flow_get_esw_manager_vport_id(dev) : 0xffff);
+		if (priv->sh->config.dv_flow_en != 2 || vport_match) {
+			flow_dv_translate_item_source_vport
+				(key, key_type & MLX5_SET_MATCHER_V ?
+				 mlx5_flow_get_esw_manager_vport_id(dev) : 0xffff);
+		} else {
+			if (key_type & MLX5_SET_MATCHER_M)
+				vport_meta = priv->sh->dev_cap.esw_info.regc_mask;
+			else
+				vport_meta = priv->sh->dev_cap.esw_info.regc_value;
+			flow_dv_translate_item_meta_vport(key, vport_meta,
+							  priv->sh->dev_cap.esw_info.regc_mask);
+		}
 		return 0;
 	}
 	mask = pid_m ? pid_m->port_id : UINT16_MAX;
@@ -10868,7 +10882,7 @@ flow_dv_translate_item_represented_port(struct rte_eth_dev *dev, void *key,
 	 * Kernel can use either misc.source_port or half of C0 metadata
 	 * register.
 	 */
-	if (priv->vport_meta_mask) {
+	if (priv->vport_meta_mask && !vport_match) {
 		/*
 		 * Provide the hint for SW steering library
 		 * to insert the flow into ingress domain and
Signature · 1 line
2.34.1
#1 lore raw

PATCH 2/2 net/mlx5/hws: add match with switch manager

This commit adds HWS layer match with switch manager code.

---
 drivers/net/mlx5/hws/mlx5dr_definer.c |  3 ++-
 drivers/net/mlx5/hws/mlx5dr_definer.h |  5 ++++-
 drivers/net/mlx5/hws/mlx5dr_rule.c    |  2 +-
 drivers/net/mlx5/mlx5_flow.h          | 30 +++++++++++++++++++++++++--
 4 files changed, 35 insertions(+), 5 deletions(-)
drivers/net/mlx5/hws/mlx5dr_definer.c +2 -1
index bc128c7b99..4be9012e25 100644
@@ -721,7 +721,7 @@ mlx5dr_definer_vport_set(struct mlx5dr_definer_fc *fc,
 	const struct flow_hw_port_info *port_info;
 	uint32_t regc_value;
 
-	port_info = flow_hw_conv_port_id(v->port_id);
+	port_info = flow_hw_conv_port_id(fc->dr_ctx, v->port_id);
 	if (unlikely(!port_info))
 		regc_value = BAD_PORT;
 	else
@@ -1548,6 +1548,7 @@ mlx5dr_definer_conv_item_port(struct mlx5dr_definer_conv_data *cd,
 		DR_CALC_SET_HDR(fc, registers, register_c_0);
 		fc->bit_off = __builtin_ctz(caps->wire_regc_mask);
 		fc->bit_mask = caps->wire_regc_mask >> fc->bit_off;
+		fc->dr_ctx = cd->ctx;
 	} else {
 		DR_LOG(ERR, "Pord ID item mask must specify ID mask");
 		rte_errno = EINVAL;
drivers/net/mlx5/hws/mlx5dr_definer.h +4 -1
index 463e22732e..b583f78943 100644
@@ -223,9 +223,12 @@ enum mlx5dr_definer_type {
 struct mlx5dr_definer_fc {
 	uint8_t item_idx;
 	uint8_t is_range;
-	uint16_t extra_data;
 	uint8_t compare_idx;
 	bool compare_set_base;
+	union {
+		uint32_t extra_data;
+		void *dr_ctx;
+	};
 	uint32_t byte_off;
 	int bit_off;
 	uint32_t bit_mask;
drivers/net/mlx5/hws/mlx5dr_rule.c +1 -1
index 171a0bff38..ea30c37270 100644
@@ -28,7 +28,7 @@ static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
 
 	if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
 		v = items[mt->vport_item_id].spec;
-		vport = flow_hw_conv_port_id(v->port_id);
+		vport = flow_hw_conv_port_id(matcher->tbl->ctx, v->port_id);
 		if (unlikely(!vport)) {
 			DR_LOG(ERR, "Fail to map port ID %d, ignoring", v->port_id);
 			return;
drivers/net/mlx5/mlx5_flow.h +28 -2
index 0a6d33f8ee..9619985450 100644
@@ -2003,15 +2003,41 @@ flow_hw_get_port_id(void *dr_ctx)
 	return UINT16_MAX;
 }
 
+/*
+ * Get given eswitch manager id.
+ * Used in HWS match with port creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_esw_mgr_id(void *dr_ctx)
+{
+#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+	uint16_t port_id;
+
+	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+		struct mlx5_priv *priv;
+
+		priv = rte_eth_devices[port_id].data->dev_private;
+		if (priv->dr_ctx == dr_ctx)
+			return &priv->sh->dev_cap.esw_info;
+	}
+#else
+	RTE_SET_USED(dr_ctx);
+#endif
+	return NULL;
+}
+
 /*
  * Get metadata match tag and mask for given rte_eth_dev port.
  * Used in HWS rule creation.
  */
 static __rte_always_inline const struct flow_hw_port_info *
-flow_hw_conv_port_id(const uint16_t port_id)
+flow_hw_conv_port_id(void *ctx, const uint16_t port_id)
 {
 	struct flow_hw_port_info *port_info;
 
+	if (port_id == UINT16_MAX && ctx)
+		return flow_hw_get_esw_mgr_id(ctx);
+
 	if (port_id >= RTE_MAX_ETHPORTS)
 		return NULL;
 	port_info = &mlx5_flow_hw_port_infos[port_id];
@@ -2037,7 +2063,7 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
 			struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
 
 			if (port_ibctx->device == ibdev)
-				return flow_hw_conv_port_id(port_id);
+				return flow_hw_conv_port_id(priv->dr_ctx, port_id);
 		}
 	}
 	return NULL;
Signature · 1 line
2.34.1
#2 lore raw
In reply to Suanming Mou
Quoted 10 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Friday, May 31, 2024 05:52
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH 1/2] net/mlx5: support match with switch manager

Currently, in switch mode, mlx5 PMD only supports match with dedicate vport.

Typo: s/dedicate/dedicated/

Quoted 2 lines

There is a usercase which user may want to offload the rules only to match with all the pkt sent by application not from vport.

Typo: s/usercase/use case/

Quoted 3 lines

Since the port_id info of pkt sent by application is switch manager, and kernel driver has exposed the switch manager register value, this commit adds the support of register matching for switch manager.

Let's expand pkt to packet and I think it would be better to use E-Switch Manager, instead of switch manager, since it's an already established name in the doca.

doc/guides/rel_notes/release_24_07.rst New Features · line 87

Quoted 5 lines
doc/guides/rel_notes/release_24_07.rst +1 -0
@@ -84,6 +84,7 @@ New Features
 * **Updated NVIDIA mlx5 driver.**
 
   * Added match with Tx queue.
+  * Added match with switch manager.

s/switch manager/E-Switch manager/

Could you also please expand in mlx5 docs what the feature allows to achieve and how to use it?

And I think that both patches in these series can be squashed into one. What do you think?

Best regards,
Dariusz Sosnowski

#3 lore raw

Hi,

Quoted 18 lines

-----Original Message-----
From: Dariusz Sosnowski <[email protected]>
Sent: Wednesday, June 5, 2024 4:34 PM
To: Suanming Mou <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: RE: [PATCH 1/2] net/mlx5: support match with switch manager

Quoted 9 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Friday, May 31, 2024 05:52
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH 1/2] net/mlx5: support match with switch manager

Quoted 3 lines
Quoted 1 line

Currently, in switch mode, mlx5 PMD only supports match with dedicate vport.

Typo: s/dedicate/dedicated/

OK.

Quoted 5 lines
Quoted 3 lines

There is a usercase which user may want to offload the rules only to match with all the pkt sent by application not from vport.

Typo: s/usercase/use case/

OK.

Quoted 7 lines
Quoted 4 lines

Since the port_id info of pkt sent by application is switch manager, and kernel driver has exposed the switch manager register value, this commit adds the support of register matching for switch manager.

Let's expand pkt to packet and I think it would be better to use E-Switch Manager, instead of switch manager, since it's an already established name in the doca.

Good suggestion.

Quoted 9 lines

doc/guides/rel_notes/release_24_07.rst New Features · line 87

Quoted 6 lines
doc/guides/rel_notes/release_24_07.rst +1 -0
@@ -84,6 +84,7 @@ New Features
 * **Updated NVIDIA mlx5 driver.**
 
   * Added match with Tx queue.
+  * Added match with switch manager.

s/switch manager/E-Switch manager/

Quoted 5 lines

Could you also please expand in mlx5 docs what the feature allows to achieve and how to use it?

And I think that both patches in these series can be squashed into one. What do you think?

Agree.

Quoted 3 lines

Best regards,
Dariusz Sosnowski

#4 lore raw

PATCH v2 1/1 net/mlx5: support match with E-Switch manager

In reply to Suanming Mou

Currently, in switch mode, mlx5 PMD only supports match with dedicated vport. There is a use case which user may want to offload the rules only to match with all the packet sent by application not from vport.

Since the port_id info of packet sent by application is E-Switch manager, and kernel driver has exposed the E-Switch manager register value, this commit adds the support of register matching for E-Switch manager.

---

v2:
- align to single patch.
- update doc.

---
 doc/guides/nics/mlx5.rst               |  6 +++++
 doc/guides/rel_notes/release_24_07.rst |  1 +
 drivers/common/mlx5/linux/meson.build  |  2 ++
 drivers/net/mlx5/hws/mlx5dr_definer.c  |  3 ++-
 drivers/net/mlx5/hws/mlx5dr_definer.h  |  5 +++-
 drivers/net/mlx5/hws/mlx5dr_rule.c     |  2 +-
 drivers/net/mlx5/linux/mlx5_os.c       | 12 +++++++++
 drivers/net/mlx5/mlx5.h                | 14 ++++++++++
 drivers/net/mlx5/mlx5_flow.h           | 36 ++++++++++++++++++++------
 drivers/net/mlx5/mlx5_flow_dv.c        | 24 +++++++++++++----
 10 files changed, 89 insertions(+), 16 deletions(-)
doc/guides/nics/mlx5.rst +6 -0
index 5cd41d3c7f..8afb32ed35 100644
@@ -170,6 +170,7 @@ Features
 - Matching on represented port.
 - Matching on aggregated affinity.
 - Matching on external Tx queue.
+- Matching on E-Switch manager.
 
 
 Limitations
@@ -910,6 +911,10 @@ Limitations
   The flow engine of a process cannot move from active to standby mode
   if preceding active application rules are still present and vice versa.
 
+- In switch mode, flow rule matching ``RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT``
+  item with port_id ``UINT16_MAX`` means matching with packets sent by E-Switch
+  manager from software. Need MLNX_OFED 24.04+.
+
 
 Statistics
 ----------
@@ -1585,6 +1590,7 @@ entities on the HCA.
 With this configuration, mlx5 PMD supports:
 
 - matching traffic coming from physical port, PF, VF or SF using REPRESENTED_PORT items;
+- matching traffic coming from E-Switch manager using REPRESENTED_PORT item with port_id UINT16_MAX;
 - forwarding traffic to physical port, PF, VF or SF using REPRESENTED_PORT actions;
 
 Requirements
doc/guides/rel_notes/release_24_07.rst +1 -0
index 3a3257fcd5..7688ed2764 100644
@@ -85,6 +85,7 @@ New Features
 
   * Added match with Tx queue.
   * Added match with external Tx queue.
+  * Added match with E-Switch manager.
 
 
 Removed Items
drivers/common/mlx5/linux/meson.build +2 -0
index cdee40c553..82e8046e0c 100644
@@ -201,6 +201,8 @@ has_sym_args = [
             'mlx5dv_create_steering_anchor'],
         [ 'HAVE_IBV_FORK_UNNEEDED', 'infiniband/verbs.h',
             'ibv_is_fork_initialized'],
+        [ 'HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0', 'infiniband/mlx5dv.h',
+            'MLX5DV_CONTEXT_MASK_REG_C0' ],
 ]
 if  libmtcr_ul_found
     has_sym_args += [
drivers/net/mlx5/hws/mlx5dr_definer.c +2 -1
index bc128c7b99..4be9012e25 100644
@@ -721,7 +721,7 @@ mlx5dr_definer_vport_set(struct mlx5dr_definer_fc *fc,
 	const struct flow_hw_port_info *port_info;
 	uint32_t regc_value;
 
-	port_info = flow_hw_conv_port_id(v->port_id);
+	port_info = flow_hw_conv_port_id(fc->dr_ctx, v->port_id);
 	if (unlikely(!port_info))
 		regc_value = BAD_PORT;
 	else
@@ -1548,6 +1548,7 @@ mlx5dr_definer_conv_item_port(struct mlx5dr_definer_conv_data *cd,
 		DR_CALC_SET_HDR(fc, registers, register_c_0);
 		fc->bit_off = __builtin_ctz(caps->wire_regc_mask);
 		fc->bit_mask = caps->wire_regc_mask >> fc->bit_off;
+		fc->dr_ctx = cd->ctx;
 	} else {
 		DR_LOG(ERR, "Pord ID item mask must specify ID mask");
 		rte_errno = EINVAL;
drivers/net/mlx5/hws/mlx5dr_definer.h +4 -1
index 463e22732e..b583f78943 100644
@@ -223,9 +223,12 @@ enum mlx5dr_definer_type {
 struct mlx5dr_definer_fc {
 	uint8_t item_idx;
 	uint8_t is_range;
-	uint16_t extra_data;
 	uint8_t compare_idx;
 	bool compare_set_base;
+	union {
+		uint32_t extra_data;
+		void *dr_ctx;
+	};
 	uint32_t byte_off;
 	int bit_off;
 	uint32_t bit_mask;
drivers/net/mlx5/hws/mlx5dr_rule.c +1 -1
index 171a0bff38..ea30c37270 100644
@@ -28,7 +28,7 @@ static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
 
 	if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
 		v = items[mt->vport_item_id].spec;
-		vport = flow_hw_conv_port_id(v->port_id);
+		vport = flow_hw_conv_port_id(matcher->tbl->ctx, v->port_id);
 		if (unlikely(!vport)) {
 			DR_LOG(ERR, "Fail to map port ID %d, ignoring", v->port_id);
 			return;
drivers/net/mlx5/linux/mlx5_os.c +12 -0
index f887501a9b..62ea151b6f 100644
@@ -160,6 +160,9 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
 #endif
 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
+#endif
+#ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_REG_C0;
 #endif
 	err = mlx5_glue->dv_query_device(cdev->ctx, &dv_attr);
 	if (err) {
@@ -374,6 +377,15 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
 					hca_attr->scatter_fcs_w_decap_disable;
 	sh->dev_cap.rq_delay_drop_en = hca_attr->rq_delay_drop;
 	mlx5_rt_timestamp_config(sh, hca_attr);
+#ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_REG_C0) {
+		sh->dev_cap.esw_info.regc_value = dv_attr.reg_c0.value;
+		sh->dev_cap.esw_info.regc_mask = dv_attr.reg_c0.mask;
+	}
+#else
+	sh->dev_cap.esw_info.regc_value = 0;
+	sh->dev_cap.esw_info.regc_mask = 0;
+#endif
 	return 0;
 }
 
drivers/net/mlx5/mlx5.h +14 -0
index 91ceceb34a..11b955ad12 100644
@@ -141,6 +141,19 @@ struct mlx5_flow_cb_ctx {
 	void *data2;
 };
 
+struct flow_hw_port_info {
+	uint32_t regc_mask;
+	uint32_t regc_value;
+	uint32_t is_wire:1;
+	uint32_t direction:2;
+};
+
+enum mlx5_vport_direction {
+	MLX5_VPORT_DIRECTION_ANY = 0,
+	MLX5_VPORT_DIRECTION_NORTH,
+	MLX5_VPORT_DIRECTION_SOUTH,
+};
+
 /* Device capabilities structure which isn't changed in any stage. */
 struct mlx5_dev_cap {
 	int max_cq; /* Maximum number of supported CQs */
@@ -184,6 +197,7 @@ struct mlx5_dev_cap {
 		/* Log min WQE size, (size of single stride)*(num of strides).*/
 	} mprq; /* Capability for Multi-Packet RQ. */
 	char fw_ver[64]; /* Firmware version of this device. */
+	struct flow_hw_port_info esw_info; /* E-switch manager reg_c0. */
 };
 
 #define MLX5_MPESW_PORT_INVALID (-1)
drivers/net/mlx5/mlx5_flow.h +28 -8
index ba75b99139..36ff567c61 100644
@@ -1938,12 +1938,6 @@ struct mlx5_hl_data {
 	uint32_t dw_mask;
 };
 
-struct flow_hw_port_info {
-	uint32_t regc_mask;
-	uint32_t regc_value;
-	uint32_t is_wire:1;
-};
-
 extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
 
 /*
@@ -2009,15 +2003,41 @@ flow_hw_get_port_id(void *dr_ctx)
 	return UINT16_MAX;
 }
 
+/*
+ * Get given eswitch manager id.
+ * Used in HWS match with port creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_esw_mgr_id(void *dr_ctx)
+{
+#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+	uint16_t port_id;
+
+	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+		struct mlx5_priv *priv;
+
+		priv = rte_eth_devices[port_id].data->dev_private;
+		if (priv->dr_ctx == dr_ctx)
+			return &priv->sh->dev_cap.esw_info;
+	}
+#else
+	RTE_SET_USED(dr_ctx);
+#endif
+	return NULL;
+}
+
 /*
  * Get metadata match tag and mask for given rte_eth_dev port.
  * Used in HWS rule creation.
  */
 static __rte_always_inline const struct flow_hw_port_info *
-flow_hw_conv_port_id(const uint16_t port_id)
+flow_hw_conv_port_id(void *ctx, const uint16_t port_id)
 {
 	struct flow_hw_port_info *port_info;
 
+	if (port_id == UINT16_MAX && ctx)
+		return flow_hw_get_esw_mgr_id(ctx);
+
 	if (port_id >= RTE_MAX_ETHPORTS)
 		return NULL;
 	port_info = &mlx5_flow_hw_port_infos[port_id];
@@ -2043,7 +2063,7 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
 			struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
 
 			if (port_ibctx->device == ibdev)
-				return flow_hw_conv_port_id(port_id);
+				return flow_hw_conv_port_id(priv->dr_ctx, port_id);
 		}
 	}
 	return NULL;
drivers/net/mlx5/mlx5_flow_dv.c +19 -5
index 14cdd4468d..c30fbefe96 100644
@@ -10839,17 +10839,31 @@ flow_dv_translate_item_represented_port(struct rte_eth_dev *dev, void *key,
 	const struct rte_flow_item_ethdev *pid_m = item ? item->mask : NULL;
 	const struct rte_flow_item_ethdev *pid_v = item ? item->spec : NULL;
 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
-	struct mlx5_priv *priv;
+	struct mlx5_priv *priv = dev->data->dev_private;
 	uint16_t mask, id;
 	uint32_t vport_meta;
+	bool vport_match = false;
 
 	MLX5_ASSERT(wks);
+#ifndef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	if (priv->sh->config.dv_flow_en == 2)
+		vport_match = true;
+#endif
 	if (!pid_m && !pid_v)
 		return 0;
 	if (pid_v && pid_v->port_id == UINT16_MAX) {
-		flow_dv_translate_item_source_vport(key,
-			key_type & MLX5_SET_MATCHER_V ?
-			mlx5_flow_get_esw_manager_vport_id(dev) : 0xffff);
+		if (priv->sh->config.dv_flow_en != 2 || vport_match) {
+			flow_dv_translate_item_source_vport
+				(key, key_type & MLX5_SET_MATCHER_V ?
+				 mlx5_flow_get_esw_manager_vport_id(dev) : 0xffff);
+		} else {
+			if (key_type & MLX5_SET_MATCHER_M)
+				vport_meta = priv->sh->dev_cap.esw_info.regc_mask;
+			else
+				vport_meta = priv->sh->dev_cap.esw_info.regc_value;
+			flow_dv_translate_item_meta_vport(key, vport_meta,
+							  priv->sh->dev_cap.esw_info.regc_mask);
+		}
 		return 0;
 	}
 	mask = pid_m ? pid_m->port_id : UINT16_MAX;
@@ -10870,7 +10884,7 @@ flow_dv_translate_item_represented_port(struct rte_eth_dev *dev, void *key,
 	 * Kernel can use either misc.source_port or half of C0 metadata
 	 * register.
 	 */
-	if (priv->vport_meta_mask) {
+	if (priv->vport_meta_mask && !vport_match) {
 		/*
 		 * Provide the hint for SW steering library
 		 * to insert the flow into ingress domain and
Signature · 1 line
2.34.1
Acked-by on v2 1/1
#5 lore raw

RE: [PATCH v2] net/mlx5: support match with E-Switch manager

Quoted 13 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Wednesday, June 5, 2024 11:37
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH v2] net/mlx5: support match with E-Switch manager

Currently, in switch mode, mlx5 PMD only supports match with dedicated vport. There is a use case which user may want to offload the rules only to match with all the packet sent by application not from vport.

Quoted 5 lines

Since the port_id info of packet sent by application is E-Switch manager, and kernel driver has exposed the E-Switch manager register value, this commit adds the support of register matching for E-Switch manager.

Best regards,
Dariusz Sosnowski

#6 lore raw

Re: [PATCH v2] net/mlx5: support match with E-Switch manager

In reply to Suanming Mou

Hi,

From: Suanming Mou <[email protected]>
Sent: Wednesday, June 5, 2024 12:37 PM
To: Dariusz Sosnowski; Slava Ovsiienko; Ori Kam; Matan Azrad
Cc: [email protected]; Raslan Darawsheh
Subject: [PATCH v2] net/mlx5: support match with E-Switch manager

Currently, in switch mode, mlx5 PMD only supports match with dedicated vport. There is a use case which user may want to offload the rules only to match with all the packet sent by application not from vport.

Since the port_id info of packet sent by application is E-Switch manager, and kernel driver has exposed the E-Switch manager register value, this commit adds the support of register matching for E-Switch manager.

Patch applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh

#7 lore raw

PATCH v3 1/1 net/mlx5: support match with E-Switch manager

In reply to Suanming Mou

Currently, in switch mode, mlx5 PMD only supports match with dedicated vport. There is a use case which user may want to offload the rules only to match with all the packet sent by application not from vport.

Since the port_id info of packet sent by application is E-Switch manager, and kernel driver has exposed the E-Switch manager register value, this commit adds the support of register matching for E-Switch manager.

---

v3:
- update dependency.

v2:
- align to single patch.
- update doc.

---
 doc/guides/nics/mlx5.rst               |  7 +++++
 doc/guides/rel_notes/release_24_07.rst |  1 +
 drivers/common/mlx5/linux/meson.build  |  2 ++
 drivers/net/mlx5/hws/mlx5dr_definer.c  |  3 ++-
 drivers/net/mlx5/hws/mlx5dr_definer.h  |  5 +++-
 drivers/net/mlx5/hws/mlx5dr_rule.c     |  2 +-
 drivers/net/mlx5/linux/mlx5_os.c       | 12 +++++++++
 drivers/net/mlx5/mlx5.h                | 14 ++++++++++
 drivers/net/mlx5/mlx5_flow.h           | 36 ++++++++++++++++++++------
 drivers/net/mlx5/mlx5_flow_dv.c        | 24 +++++++++++++----
 10 files changed, 90 insertions(+), 16 deletions(-)
doc/guides/nics/mlx5.rst +7 -0
index 5cd41d3c7f..8e8a19f64d 100644
@@ -170,6 +170,7 @@ Features
 - Matching on represented port.
 - Matching on aggregated affinity.
 - Matching on external Tx queue.
+- Matching on E-Switch manager.
 
 
 Limitations
@@ -910,6 +911,11 @@ Limitations
   The flow engine of a process cannot move from active to standby mode
   if preceding active application rules are still present and vice versa.
 
+- In switch mode, flow rule matching ``RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT``
+  item with port_id ``UINT16_MAX`` means matching with packets sent by E-Switch
+  manager from software.
+  Need upstream Linux 6.7+ and rdma-core 50.0+ or MLNX_OFED 24.04+.
+
 
 Statistics
 ----------
@@ -1585,6 +1591,7 @@ entities on the HCA.
 With this configuration, mlx5 PMD supports:
 
 - matching traffic coming from physical port, PF, VF or SF using REPRESENTED_PORT items;
+- matching traffic coming from E-Switch manager using REPRESENTED_PORT item with port_id UINT16_MAX;
 - forwarding traffic to physical port, PF, VF or SF using REPRESENTED_PORT actions;
 
 Requirements
doc/guides/rel_notes/release_24_07.rst +1 -0
index 3a3257fcd5..7688ed2764 100644
@@ -85,6 +85,7 @@ New Features
 
   * Added match with Tx queue.
   * Added match with external Tx queue.
+  * Added match with E-Switch manager.
 
 
 Removed Items
drivers/common/mlx5/linux/meson.build +2 -0
index cdee40c553..82e8046e0c 100644
@@ -201,6 +201,8 @@ has_sym_args = [
             'mlx5dv_create_steering_anchor'],
         [ 'HAVE_IBV_FORK_UNNEEDED', 'infiniband/verbs.h',
             'ibv_is_fork_initialized'],
+        [ 'HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0', 'infiniband/mlx5dv.h',
+            'MLX5DV_CONTEXT_MASK_REG_C0' ],
 ]
 if  libmtcr_ul_found
     has_sym_args += [
drivers/net/mlx5/hws/mlx5dr_definer.c +2 -1
index 67cfba1c10..9ebda9267d 100644
@@ -723,7 +723,7 @@ mlx5dr_definer_vport_set(struct mlx5dr_definer_fc *fc,
 	const struct flow_hw_port_info *port_info;
 	uint32_t regc_value;
 
-	port_info = flow_hw_conv_port_id(v->port_id);
+	port_info = flow_hw_conv_port_id(fc->dr_ctx, v->port_id);
 	if (unlikely(!port_info))
 		regc_value = BAD_PORT;
 	else
@@ -1550,6 +1550,7 @@ mlx5dr_definer_conv_item_port(struct mlx5dr_definer_conv_data *cd,
 		DR_CALC_SET_HDR(fc, registers, register_c_0);
 		fc->bit_off = __builtin_ctz(caps->wire_regc_mask);
 		fc->bit_mask = caps->wire_regc_mask >> fc->bit_off;
+		fc->dr_ctx = cd->ctx;
 	} else {
 		DR_LOG(ERR, "Pord ID item mask must specify ID mask");
 		rte_errno = EINVAL;
drivers/net/mlx5/hws/mlx5dr_definer.h +4 -1
index 463e22732e..b583f78943 100644
@@ -223,9 +223,12 @@ enum mlx5dr_definer_type {
 struct mlx5dr_definer_fc {
 	uint8_t item_idx;
 	uint8_t is_range;
-	uint16_t extra_data;
 	uint8_t compare_idx;
 	bool compare_set_base;
+	union {
+		uint32_t extra_data;
+		void *dr_ctx;
+	};
 	uint32_t byte_off;
 	int bit_off;
 	uint32_t bit_mask;
drivers/net/mlx5/hws/mlx5dr_rule.c +1 -1
index 74abb38163..06d8e66f63 100644
@@ -28,7 +28,7 @@ static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
 
 	if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
 		v = items[mt->vport_item_id].spec;
-		vport = flow_hw_conv_port_id(v->port_id);
+		vport = flow_hw_conv_port_id(matcher->tbl->ctx, v->port_id);
 		if (unlikely(!vport)) {
 			DR_LOG(ERR, "Fail to map port ID %d, ignoring", v->port_id);
 			return;
drivers/net/mlx5/linux/mlx5_os.c +12 -0
index 2c3d5bd6cd..1753acd050 100644
@@ -160,6 +160,9 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
 #endif
 #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
 	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_STRIDING_RQ;
+#endif
+#ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_REG_C0;
 #endif
 	err = mlx5_glue->dv_query_device(cdev->ctx, &dv_attr);
 	if (err) {
@@ -374,6 +377,15 @@ mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh)
 					hca_attr->scatter_fcs_w_decap_disable;
 	sh->dev_cap.rq_delay_drop_en = hca_attr->rq_delay_drop;
 	mlx5_rt_timestamp_config(sh, hca_attr);
+#ifdef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_REG_C0) {
+		sh->dev_cap.esw_info.regc_value = dv_attr.reg_c0.value;
+		sh->dev_cap.esw_info.regc_mask = dv_attr.reg_c0.mask;
+	}
+#else
+	sh->dev_cap.esw_info.regc_value = 0;
+	sh->dev_cap.esw_info.regc_mask = 0;
+#endif
 	return 0;
 }
 
drivers/net/mlx5/mlx5.h +14 -0
index 6af367b36b..c9a3837bd2 100644
@@ -141,6 +141,19 @@ struct mlx5_flow_cb_ctx {
 	void *data2;
 };
 
+struct flow_hw_port_info {
+	uint32_t regc_mask;
+	uint32_t regc_value;
+	uint32_t is_wire:1;
+	uint32_t direction:2;
+};
+
+enum mlx5_vport_direction {
+	MLX5_VPORT_DIRECTION_ANY = 0,
+	MLX5_VPORT_DIRECTION_NORTH,
+	MLX5_VPORT_DIRECTION_SOUTH,
+};
+
 /* Device capabilities structure which isn't changed in any stage. */
 struct mlx5_dev_cap {
 	int max_cq; /* Maximum number of supported CQs */
@@ -184,6 +197,7 @@ struct mlx5_dev_cap {
 		/* Log min WQE size, (size of single stride)*(num of strides).*/
 	} mprq; /* Capability for Multi-Packet RQ. */
 	char fw_ver[64]; /* Firmware version of this device. */
+	struct flow_hw_port_info esw_info; /* E-switch manager reg_c0. */
 };
 
 #define MLX5_MPESW_PORT_INVALID (-1)
drivers/net/mlx5/mlx5_flow.h +28 -8
index 8ab92dc4a6..92e2ecedb3 100644
@@ -1977,12 +1977,6 @@ struct mlx5_hl_data {
 	uint32_t dw_mask;
 };
 
-struct flow_hw_port_info {
-	uint32_t regc_mask;
-	uint32_t regc_value;
-	uint32_t is_wire:1;
-};
-
 extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
 
 /*
@@ -2048,15 +2042,41 @@ flow_hw_get_port_id(void *dr_ctx)
 	return UINT16_MAX;
 }
 
+/*
+ * Get given eswitch manager id.
+ * Used in HWS match with port creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_esw_mgr_id(void *dr_ctx)
+{
+#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+	uint16_t port_id;
+
+	MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+		struct mlx5_priv *priv;
+
+		priv = rte_eth_devices[port_id].data->dev_private;
+		if (priv->dr_ctx == dr_ctx)
+			return &priv->sh->dev_cap.esw_info;
+	}
+#else
+	RTE_SET_USED(dr_ctx);
+#endif
+	return NULL;
+}
+
 /*
  * Get metadata match tag and mask for given rte_eth_dev port.
  * Used in HWS rule creation.
  */
 static __rte_always_inline const struct flow_hw_port_info *
-flow_hw_conv_port_id(const uint16_t port_id)
+flow_hw_conv_port_id(void *ctx, const uint16_t port_id)
 {
 	struct flow_hw_port_info *port_info;
 
+	if (port_id == UINT16_MAX && ctx)
+		return flow_hw_get_esw_mgr_id(ctx);
+
 	if (port_id >= RTE_MAX_ETHPORTS)
 		return NULL;
 	port_info = &mlx5_flow_hw_port_infos[port_id];
@@ -2082,7 +2102,7 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
 			struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
 
 			if (port_ibctx->device == ibdev)
-				return flow_hw_conv_port_id(port_id);
+				return flow_hw_conv_port_id(priv->dr_ctx, port_id);
 		}
 	}
 	return NULL;
drivers/net/mlx5/mlx5_flow_dv.c +19 -5
index 4436b6f4c9..27f15817b4 100644
@@ -10947,17 +10947,31 @@ flow_dv_translate_item_represented_port(struct rte_eth_dev *dev, void *key,
 	const struct rte_flow_item_ethdev *pid_m = item ? item->mask : NULL;
 	const struct rte_flow_item_ethdev *pid_v = item ? item->spec : NULL;
 	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
-	struct mlx5_priv *priv;
+	struct mlx5_priv *priv = dev->data->dev_private;
 	uint16_t mask, id;
 	uint32_t vport_meta;
+	bool vport_match = false;
 
 	MLX5_ASSERT(wks);
+#ifndef HAVE_IBV_DEVICE_ATTR_ESW_MGR_REG_C0
+	if (priv->sh->config.dv_flow_en == 2)
+		vport_match = true;
+#endif
 	if (!pid_m && !pid_v)
 		return 0;
 	if (pid_v && pid_v->port_id == UINT16_MAX) {
-		flow_dv_translate_item_source_vport(key,
-			key_type & MLX5_SET_MATCHER_V ?
-			mlx5_flow_get_esw_manager_vport_id(dev) : 0xffff);
+		if (priv->sh->config.dv_flow_en != 2 || vport_match) {
+			flow_dv_translate_item_source_vport
+				(key, key_type & MLX5_SET_MATCHER_V ?
+				 mlx5_flow_get_esw_manager_vport_id(dev) : 0xffff);
+		} else {
+			if (key_type & MLX5_SET_MATCHER_M)
+				vport_meta = priv->sh->dev_cap.esw_info.regc_mask;
+			else
+				vport_meta = priv->sh->dev_cap.esw_info.regc_value;
+			flow_dv_translate_item_meta_vport(key, vport_meta,
+							  priv->sh->dev_cap.esw_info.regc_mask);
+		}
 		return 0;
 	}
 	mask = pid_m ? pid_m->port_id : UINT16_MAX;
@@ -10978,7 +10992,7 @@ flow_dv_translate_item_represented_port(struct rte_eth_dev *dev, void *key,
 	 * Kernel can use either misc.source_port or half of C0 metadata
 	 * register.
 	 */
-	if (priv->vport_meta_mask) {
+	if (priv->vport_meta_mask && !vport_match) {
 		/*
 		 * Provide the hint for SW steering library
 		 * to insert the flow into ingress domain and
Signature · 1 line
2.34.1