Skip to contentSkip to conversation

net/mlx5: support multiple-thread flow operations

PATCH Superseded Suanming Mou 2 participants 193 messages Oct 6, 2020 – Oct 28, 2020 Network drivers Page 2 of 2

lore
Share & tools
lore
#110 lore raw

PATCH v4 23/34 net/mlx5: make Rx queue thread safe

In reply to Suanming Mou

This commit applies the cache linked list to Rx queue to make it thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c   |   5 +
 drivers/net/mlx5/mlx5.c            |   1 +
 drivers/net/mlx5/mlx5.h            |  28 +++-
 drivers/net/mlx5/mlx5_flow.h       |  16 ---
 drivers/net/mlx5/mlx5_flow_dv.c    |  74 ++++------
 drivers/net/mlx5/mlx5_flow_verbs.c |  21 +--
 drivers/net/mlx5/mlx5_rxq.c        | 281 ++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_rxtx.h       |  20 +--
 8 files changed, 228 insertions(+), 218 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -0
index d017c23..10fc7c5 100644
@@ -1468,6 +1468,10 @@
 			err = ENOTSUP;
 			goto error;
 	}
+	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+			     mlx5_hrxq_create_cb,
+			     mlx5_hrxq_match_cb,
+			     mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
@@ -1520,6 +1524,7 @@
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+		mlx5_cache_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -0
index 1d25a8e..862bd40 100644
@@ -1286,6 +1286,7 @@ struct mlx5_dev_ctx_shared *
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
+	mlx5_cache_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +26 -2
index 9f96bd0..7804d3a 100644
@@ -65,6 +65,13 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* Hash and cache list callback context. */
+struct mlx5_flow_cb_ctx {
+	struct rte_eth_dev *dev;
+	struct rte_flow_error *error;
+	void *data;
+};
+
 /* Device attributes used in mlx5 PMD */
 struct mlx5_dev_attr {
 	uint64_t	device_cap_flags_ex;
@@ -684,6 +691,22 @@ struct mlx5_proc_priv {
 /* MTR list. */
 TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter);
 
+/* RSS description. */
+struct mlx5_flow_rss_desc {
+	uint32_t level;
+	uint32_t queue_num; /**< Number of entries in @p queue. */
+	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
+	uint64_t hash_fields; /* Verbs Hash fields. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint32_t key_len; /**< RSS hash key len. */
+	uint32_t tunnel; /**< Queue in tunnel. */
+	union {
+		uint16_t *queue; /**< Destination queues. */
+		const uint16_t *const_q; /**< Const pointer convert. */
+	};
+	bool standalone; /**< Queue is standalone or not. */
+};
+
 #define MLX5_PROC_PRIV(port_id) \
 	((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)
 
@@ -723,7 +746,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	ILIST_ENTRY(uint32_t)next; /* Index to the next element. */
+	struct mlx5_cache_entry entry; /* Cache entry. */
 	uint32_t refcnt; /* Reference counter. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
@@ -737,6 +760,7 @@ struct mlx5_hrxq {
 #endif
 	uint64_t hash_fields; /* Verbs Hash fields. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
+	uint32_t idx; /* Hash Rx queue index. */
 	uint8_t rss_key[]; /* Hash key. */
 };
 
@@ -853,7 +877,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	uint32_t hrxqs; /* Verbs Hash Rx queues. */
+	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +0 -16
index d0a1000..84988c6 100644
@@ -384,13 +384,6 @@ enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_MAX,
 };
 
-/* Hash list callback context */
-struct mlx5_flow_cb_ctx {
-	struct rte_eth_dev *dev;
-	struct rte_flow_error *error;
-	void *data;
-};
-
 /* Matcher PRM representation */
 struct mlx5_flow_dv_match_params {
 	size_t size;
@@ -609,15 +602,6 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
-/* RSS description. */
-struct mlx5_flow_rss_desc {
-	uint32_t level;
-	uint32_t queue_num; /**< Number of entries in @p queue. */
-	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
-	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
-	uint16_t *queue; /**< Destination queues. */
-};
-
 /* PMD flow priority for tunnel */
 #define MLX5_TUNNEL_PRIO_GET(rss_desc) \
 	((rss_desc)->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4)
drivers/net/mlx5/mlx5_flow_dv.c +27 -47

131 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_verbs.c +6 -15
index 3c04727..28fd120 100644
@@ -1968,21 +1968,12 @@
 				&wks->rss_desc[!!wks->flow_nested_idx];
 
 			MLX5_ASSERT(rss_desc->queue_num);
-			hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
-						 MLX5_RSS_HASH_KEY_LEN,
-						 dev_flow->hash_fields,
-						 rss_desc->queue,
-						 rss_desc->queue_num);
-			if (!hrxq_idx)
-				hrxq_idx = mlx5_hrxq_new
-						(dev, rss_desc->key,
-						 MLX5_RSS_HASH_KEY_LEN,
-						 dev_flow->hash_fields,
-						 rss_desc->queue,
-						 rss_desc->queue_num,
-						 !!(handle->layers &
-						 MLX5_FLOW_LAYER_TUNNEL),
-						 false);
+			rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
+			rss_desc->hash_fields = dev_flow->hash_fields;
+			rss_desc->tunnel = !!(handle->layers &
+					      MLX5_FLOW_LAYER_TUNNEL);
+			rss_desc->standalone = false;
+			hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
 			hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
 					      hrxq_idx);
 			if (!hrxq) {
drivers/net/mlx5/mlx5_rxq.c +152 -129

342 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxtx.h +11 -9
index 8fe0473..c361bbd 100644
@@ -349,17 +349,19 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_obj *ind_tbl,
 			       bool standalone);
-uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev,
-		       const uint8_t *rss_key, uint32_t rss_key_len,
-		       uint64_t hash_fields,
-		       const uint16_t *queues, uint32_t queues_n,
-		       int tunnel, bool standalone);
+struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
+		       struct mlx5_cache_entry *entry,
+		       void *cb_ctx);
+void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
+			 struct mlx5_cache_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
-		       const uint8_t *rss_key, uint32_t rss_key_len,
-		       uint64_t hash_fields,
-		       const uint16_t *queues, uint32_t queues_n);
+		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
-int mlx5_hrxq_verify(struct rte_eth_dev *dev);
+uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev);
+
+
 enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx);
 const struct rte_eth_hairpin_conf *mlx5_rxq_get_hairpin_conf
 	(struct rte_eth_dev *dev, uint16_t idx);
Signature · 1 line
1.8.3.1
#111 lore raw

PATCH v4 24/34 net/mlx5: make matcher list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this path converts matcher list to use thread safe cache list API.

---
 drivers/net/mlx5/mlx5.h         |   3 +
 drivers/net/mlx5/mlx5_flow.h    |  15 ++-
 drivers/net/mlx5/mlx5_flow_dv.c | 215 +++++++++++++++++++++-------------------
 3 files changed, 129 insertions(+), 104 deletions(-)
drivers/net/mlx5/mlx5.h +3 -0
index 7804d3a..eadb797 100644
@@ -32,6 +32,9 @@
 #include "mlx5_os.h"
 #include "mlx5_autoconf.h"
 
+
+#define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
+
 enum mlx5_ipool_index {
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
drivers/net/mlx5/mlx5_flow.h +11 -4
index 84988c6..6c0815a 100644
@@ -394,11 +394,9 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	LIST_ENTRY(mlx5_flow_dv_matcher) next;
-	/**< Pointer to the next element. */
+	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
-	uint32_t refcnt; /**< Reference counter. */
 	void *matcher_object; /**< Pointer to DV matcher */
 	uint16_t crc; /**< CRC of key. */
 	uint16_t priority; /**< Priority of matcher. */
@@ -532,7 +530,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
+	struct mlx5_cache_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -542,6 +540,7 @@ struct mlx5_flow_tbl_data_entry {
 	uint32_t group_id;
 	bool external;
 	bool tunnel_offload; /* Tunnel offlod table or not. */
+	bool is_egress; /**< Egress table. */
 };
 
 /* Sub rdma-core actions list. */
@@ -1430,4 +1429,12 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 				uint64_t key, void *cb_ctx);
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
+
+int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry, void *ctx);
+struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry, void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +115 -100

346 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#112 lore raw

PATCH v4 25/34 net/mlx5: make port ID action cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch convert port id action cache list to thread safe cache list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 ++
 drivers/net/mlx5/mlx5.h          |   2 +-
 drivers/net/mlx5/mlx5_flow.h     |  15 ++--
 drivers/net/mlx5/mlx5_flow_dv.c  | 143 +++++++++++++++++++++------------------
 4 files changed, 94 insertions(+), 73 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index 10fc7c5..3d2bf57 100644
@@ -236,6 +236,12 @@
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	/* Init port id action cache list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+			     flow_dv_port_id_create_cb,
+			     flow_dv_port_id_match_cb,
+			     flow_dv_port_id_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -431,6 +437,7 @@
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
+	mlx5_cache_list_destroy(&sh->port_id_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index eadb797..1cd42eb 100644
@@ -659,7 +659,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	uint32_t port_id_action_list; /* List of port ID actions. */
+	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	uint32_t push_vlan_action_list; /* List of push VLAN actions. */
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
drivers/net/mlx5/mlx5_flow.h +10 -5
index 6c0815a..0e4d47f 100644
@@ -485,12 +485,10 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	ILIST_ENTRY(uint32_t)next;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *action;
-	/**< Action object. */
+	struct mlx5_cache_entry entry;
+	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
+	uint32_t idx; /**< Indexed pool memory index. */
 };
 
 /* Push VLAN action resource structure */
@@ -1437,4 +1435,11 @@ struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
 			       struct mlx5_cache_entry *entry);
 
+int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +76 -67

170 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#113 lore raw

PATCH v4 26/34 net/mlx5: make push VLAN action cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch converts push VLAN action cache list to thread safe cache list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 ++
 drivers/net/mlx5/mlx5.h          |   2 +-
 drivers/net/mlx5/mlx5_flow.h     |  13 +++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 159 +++++++++++++++++++++------------------
 4 files changed, 102 insertions(+), 79 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index 3d2bf57..9ab3ba9 100644
@@ -242,6 +242,12 @@
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
+	/* Init push vlan action cache list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+			     flow_dv_push_vlan_create_cb,
+			     flow_dv_push_vlan_match_cb,
+			     flow_dv_push_vlan_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -438,6 +444,7 @@
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_cache_list_destroy(&sh->port_id_action_list);
+	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index 1cd42eb..9efa354 100644
@@ -660,7 +660,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	uint32_t push_vlan_action_list; /* List of push VLAN actions. */
+	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
drivers/net/mlx5/mlx5_flow.h +10 -3
index 0e4d47f..500a580 100644
@@ -493,12 +493,11 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	ILIST_ENTRY(uint32_t)next;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
+	uint32_t idx; /**< Indexed pool memory index. */
 };
 
 /* Metadata register copy table entry. */
@@ -1442,4 +1441,12 @@ struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
 			       struct mlx5_cache_entry *entry);
 
+int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +84 -75

189 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#114 lore raw

PATCH v4 27/34 net/mlx5: simplify sample attributes

In reply to Suanming Mou

Currently, the sample action resource already has ft_type to indicate the action domain attribute, the extra flow attributes parameter can be optimized.

This commit uses action resource ty_type as domain attribute instead of the flow attribute.

---
 drivers/net/mlx5/mlx5_flow_dv.c | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_flow_dv.c +14 -15
index 4996e24..4462653 100644
@@ -8603,8 +8603,6 @@ struct mlx5_hlist_entry *
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] attr
- *   Attributes of flow that includes this item.
  * @param[in] resource
  *   Pointer to sample resource.
  * @parm[in, out] dev_flow
@@ -8619,7 +8617,6 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 const struct rte_flow_attr *attr,
 			 struct mlx5_flow_dv_sample_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 void **sample_dv_actions,
@@ -8633,6 +8630,8 @@ struct mlx5_hlist_entry *
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
 	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint8_t is_egress = 0;
+	uint8_t is_transfer = 0;
 
 	/* Lookup a matching resource from cache. */
 	ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
@@ -8665,8 +8664,12 @@ struct mlx5_hlist_entry *
 					  "cannot allocate resource memory");
 	*cache_resource = *resource;
 	/* Create normal path table level */
+	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+		is_transfer = 1;
+	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
-					attr->egress, attr->transfer,
+					is_egress, is_transfer,
 					dev_flow->external, NULL, 0, 0, error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENOMEM,
@@ -8749,8 +8752,6 @@ struct mlx5_hlist_entry *
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] attr
- *   Attributes of flow that includes this item.
  * @param[in] resource
  *   Pointer to destination array resource.
  * @parm[in, out] dev_flow
@@ -8763,7 +8764,6 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 const struct rte_flow_attr *attr,
 			 struct mlx5_flow_dv_dest_array_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
@@ -8807,9 +8807,9 @@ struct mlx5_hlist_entry *
 					  NULL,
 					  "cannot allocate resource memory");
 	*cache_resource = *resource;
-	if (attr->transfer)
+	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (attr->ingress)
+	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
@@ -9098,6 +9098,8 @@ struct mlx5_hlist_entry *
 		res->set_action = action_ctx.set_action;
 	} else if (attr->ingress) {
 		res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
+	} else {
+		res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
 	}
 	return 0;
 }
@@ -9109,8 +9111,6 @@ struct mlx5_hlist_entry *
  *   Pointer to rte_eth_dev structure.
  * @param[in, out] dev_flow
  *   Pointer to the mlx5_flow.
- * @param[in] attr
- *   Pointer to the flow attributes.
  * @param[in] num_of_dest
  *   The num of destination.
  * @param[in, out] res
@@ -9130,7 +9130,6 @@ struct mlx5_hlist_entry *
 static int
 flow_dv_create_action_sample(struct rte_eth_dev *dev,
 			     struct mlx5_flow *dev_flow,
-			     const struct rte_flow_attr *attr,
 			     uint32_t num_of_dest,
 			     struct mlx5_flow_dv_sample_resource *res,
 			     struct mlx5_flow_dv_dest_array_resource *mdest_res,
@@ -9190,14 +9189,14 @@ struct mlx5_hlist_entry *
 		memcpy(&mdest_res->sample_act[0], &res->sample_act,
 				sizeof(struct mlx5_flow_sub_actions_list));
 		mdest_res->num_of_dest = num_of_dest;
-		if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
+		if (flow_dv_dest_array_resource_register(dev, mdest_res,
 							 dev_flow, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL, "can't create sample "
 						  "action");
 	} else {
-		if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
+		if (flow_dv_sample_resource_register(dev, res, dev_flow,
 						     sample_actions, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -9837,7 +9836,7 @@ struct mlx5_hlist_entry *
 			}
 			if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
 				ret = flow_dv_create_action_sample(dev,
-							  dev_flow, attr,
+							  dev_flow,
 							  num_of_dest,
 							  &sample_res,
 							  &mdest_res,
Signature · 1 line
1.8.3.1
#115 lore raw

PATCH v4 28/34 net/mlx5: fix sample register error flow

In reply to Suanming Mou

Currently, sample flow need to prepare and register the sub-actions before sample action is created.

Once the same sample action exists, the sub-actions registered by the second flow should be released, or these sub-actions will be leaked. Since the exist sample action only release these same sub-actions when the sample action itself releases.

When same sample action exists, call the sub-action release function for the later flow to release the redundant prepared sub-actions.

Fixes 0756228b2704 ("net/mlx5: update translate function for sample action")
---
 drivers/net/mlx5/mlx5_flow_dv.c | 117 ++++++++++++++++++++--------------------
 1 file changed, 60 insertions(+), 57 deletions(-)
drivers/net/mlx5/mlx5_flow_dv.c +60 -57
index 4462653..1866a21 100644
@@ -8599,6 +8599,42 @@ struct mlx5_hlist_entry *
 }
 
 /**
+ * Release sample sub action resource.
+ *
+ * @param[in, out] dev
+ *   Pointer to rte_eth_dev structure.
+ * @param[in] act_res
+ *   Pointer to sample sub action resource.
+ */
+static void
+flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
+				   struct mlx5_flow_sub_actions_idx *act_res)
+{
+	if (act_res->rix_hrxq) {
+		mlx5_hrxq_release(dev, act_res->rix_hrxq);
+		act_res->rix_hrxq = 0;
+	}
+	if (act_res->rix_encap_decap) {
+		flow_dv_encap_decap_resource_release(dev,
+						     act_res->rix_encap_decap);
+		act_res->rix_encap_decap = 0;
+	}
+	if (act_res->rix_port_id_action) {
+		flow_dv_port_id_action_resource_release(dev,
+						act_res->rix_port_id_action);
+		act_res->rix_port_id_action = 0;
+	}
+	if (act_res->rix_tag) {
+		flow_dv_tag_release(dev, act_res->rix_tag);
+		act_res->rix_tag = 0;
+	}
+	if (act_res->cnt) {
+		flow_dv_counter_release(dev, act_res->cnt);
+		act_res->cnt = 0;
+	}
+}
+
+/**
  * Find existing sample resource or create and register a new one.
  *
  * @param[in, out] dev
@@ -8651,6 +8687,12 @@ struct mlx5_hlist_entry *
 					   __ATOMIC_RELAXED);
 			dev_flow->handle->dvh.rix_sample = idx;
 			dev_flow->dv.sample_res = cache_resource;
+			/*
+			 * Existing smaple action should release the prepared
+			 * sub-actions reference counter.
+			 */
+			flow_dv_sample_sub_actions_release(dev,
+							&resource->sample_idx);
 			return 0;
 		}
 	}
@@ -8719,25 +8761,13 @@ struct mlx5_hlist_entry *
 		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
 	return 0;
 error:
-	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
-		if (cache_resource->default_miss)
-			claim_zero(mlx5_glue->destroy_flow_action
+	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB &&
+	    cache_resource->default_miss)
+		claim_zero(mlx5_glue->destroy_flow_action
 				(cache_resource->default_miss));
-	} else {
-		if (cache_resource->sample_idx.rix_hrxq &&
-		    !mlx5_hrxq_release(dev,
-				cache_resource->sample_idx.rix_hrxq))
-			cache_resource->sample_idx.rix_hrxq = 0;
-		if (cache_resource->sample_idx.rix_tag &&
-		    !flow_dv_tag_release(dev,
-				cache_resource->sample_idx.rix_tag))
-			cache_resource->sample_idx.rix_tag = 0;
-		if (cache_resource->sample_idx.cnt) {
-			flow_dv_counter_release(dev,
-				cache_resource->sample_idx.cnt);
-			cache_resource->sample_idx.cnt = 0;
-		}
-	}
+	else
+		flow_dv_sample_sub_actions_release(dev,
+						   &cache_resource->sample_idx);
 	if (cache_resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
@@ -8795,6 +8825,13 @@ struct mlx5_hlist_entry *
 					   __ATOMIC_RELAXED);
 			dev_flow->handle->dvh.rix_dest_array = idx;
 			dev_flow->dv.dest_array_res = cache_resource;
+			/*
+			 * Existing smaple action should release the prepared
+			 * sub-actions reference counter.
+			 */
+			for (idx = 0; idx < resource->num_of_dest; idx++)
+				flow_dv_sample_sub_actions_release(dev,
+						&resource->sample_idx[idx]);
 			return 0;
 		}
 	}
@@ -10691,21 +10728,8 @@ struct mlx5_hlist_entry *
 		if (cache_resource->normal_path_tbl)
 			flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
-	}
-	if (cache_resource->sample_idx.rix_hrxq &&
-		!mlx5_hrxq_release(dev,
-			cache_resource->sample_idx.rix_hrxq))
-		cache_resource->sample_idx.rix_hrxq = 0;
-	if (cache_resource->sample_idx.rix_tag &&
-		!flow_dv_tag_release(dev,
-			cache_resource->sample_idx.rix_tag))
-		cache_resource->sample_idx.rix_tag = 0;
-	if (cache_resource->sample_idx.cnt) {
-		flow_dv_counter_release(dev,
-			cache_resource->sample_idx.cnt);
-		cache_resource->sample_idx.cnt = 0;
-	}
-	if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
+		flow_dv_sample_sub_actions_release(dev,
+					&cache_resource->sample_idx);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
 			     &priv->sh->sample_action_list, idx,
 			     cache_resource, next);
@@ -10734,7 +10758,6 @@ struct mlx5_hlist_entry *
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_sub_actions_idx *mdest_act_res;
 	uint32_t idx = handle->dvh.rix_dest_array;
 	uint32_t i = 0;
 
@@ -10751,29 +10774,9 @@ struct mlx5_hlist_entry *
 		if (cache_resource->action)
 			claim_zero(mlx5_glue->destroy_flow_action
 						(cache_resource->action));
-		for (; i < cache_resource->num_of_dest; i++) {
-			mdest_act_res = &cache_resource->sample_idx[i];
-			if (mdest_act_res->rix_hrxq) {
-				mlx5_hrxq_release(dev,
-					mdest_act_res->rix_hrxq);
-				mdest_act_res->rix_hrxq = 0;
-			}
-			if (mdest_act_res->rix_encap_decap) {
-				flow_dv_encap_decap_resource_release(dev,
-					mdest_act_res->rix_encap_decap);
-				mdest_act_res->rix_encap_decap = 0;
-			}
-			if (mdest_act_res->rix_port_id_action) {
-				flow_dv_port_id_action_resource_release(dev,
-					mdest_act_res->rix_port_id_action);
-				mdest_act_res->rix_port_id_action = 0;
-			}
-			if (mdest_act_res->rix_tag) {
-				flow_dv_tag_release(dev,
-					mdest_act_res->rix_tag);
-				mdest_act_res->rix_tag = 0;
-			}
-		}
+		for (; i < cache_resource->num_of_dest; i++)
+			flow_dv_sample_sub_actions_release(dev,
+					&cache_resource->sample_idx[i]);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 			     &priv->sh->dest_array_list, idx,
 			     cache_resource, next);
Signature · 1 line
1.8.3.1
#116 lore raw

PATCH v4 29/34 net/mlx5: make sample and mirror action thread safe

In reply to Suanming Mou

This commit uses cache list to make sample and mirror action thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c |  14 ++
 drivers/net/mlx5/mlx5.h          |   5 +-
 drivers/net/mlx5/mlx5_flow.h     |  28 ++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 396 ++++++++++++++++++++++-----------------
 4 files changed, 260 insertions(+), 183 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +14 -0
index 9ab3ba9..5856981 100644
@@ -248,6 +248,20 @@
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
+	/* Init sample action cache list. */
+	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->sample_action_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_sample_create_cb,
+			     flow_dv_sample_match_cb,
+			     flow_dv_sample_remove_cb);
+	/* Init dest array action cache list. */
+	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->dest_array_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_dest_array_create_cb,
+			     flow_dv_dest_array_match_cb,
+			     flow_dv_dest_array_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
drivers/net/mlx5/mlx5.h +3 -2
index 9efa354..ac7a026 100644
@@ -661,8 +661,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	uint32_t sample_action_list; /* List of sample actions. */
-	uint32_t dest_array_list; /* List of destination array actions. */
+	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
+	struct mlx5_cache_list dest_array_list;
+	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
drivers/net/mlx5/mlx5_flow.h +23 -5
index 500a580..4a8c2bf 100644
@@ -562,9 +562,12 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *verbs_action; /**< Verbs sample action object. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	union {
+		void *verbs_action; /**< Verbs sample action object. */
+		void **sub_actions; /**< Sample sub-action array. */
+	};
+	uint32_t idx; /** Sample object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint32_t ft_id; /** Flow Table Level */
 	uint32_t ratio;   /** Sample Ratio */
@@ -581,8 +584,8 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
 	void *action; /**< Pointer to the rdma core action. */
@@ -1449,4 +1452,19 @@ struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
 void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
 				 struct mlx5_cache_entry *entry);
 
+int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
+			    struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_sample_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
+			      struct mlx5_cache_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
+				struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_dest_array_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
+				  struct mlx5_cache_entry *entry);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +220 -176

510 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#117 lore raw

PATCH v4 30/34 net/mlx5: make tunnel offloading table thread safe

In reply to Suanming Mou

To support multi-thread flow insertion, this patch updates tunnel offloading hash table to use thread safe hash list.

---
 drivers/net/mlx5/mlx5_flow.c    | 92 ++++++++++++++++++++++++++---------------
 drivers/net/mlx5/mlx5_flow_dv.c | 11 +----
 2 files changed, 61 insertions(+), 42 deletions(-)
drivers/net/mlx5/mlx5_flow.c +59 -33
index 37e7fc3..1e82030 100644
@@ -7399,13 +7399,56 @@ struct mlx5_meter_domains_infos *
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
+static void
+mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+			tunnel_flow_tbl_to_id(tte->flow_table));
+	mlx5_free(tte);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
+				   uint64_t key __rte_unused,
+				   void *ctx __rte_unused)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct tunnel_tbl_entry *tte;
+
+	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
+			  sizeof(*tte), 0,
+			  SOCKET_ID_ANY);
+	if (!tte)
+		goto err;
+	mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+			  &tte->flow_table);
+	if (tte->flow_table >= MLX5_MAX_TABLES) {
+		DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
+			tte->flow_table);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+				tte->flow_table);
+		goto err;
+	} else if (!tte->flow_table) {
+		goto err;
+	}
+	tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
+	return &tte->hash;
+err:
+	if (tte)
+		mlx5_free(tte);
+	return NULL;
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hlist_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
@@ -7416,40 +7459,17 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_hlist *group_hash;
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_lookup(group_hash, key.val, NULL);
-	if (!he) {
-		tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
-				  sizeof(*tte), 0,
-				  SOCKET_ID_ANY);
-		if (!tte)
-			goto err;
-		tte->hash.key = key.val;
-		mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-				  &tte->flow_table);
-		if (tte->flow_table >= MLX5_MAX_TABLES) {
-			DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
-				tte->flow_table);
-			mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-					tte->flow_table);
-			goto err;
-		} else if (!tte->flow_table) {
-			goto err;
-		}
-		tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
-		mlx5_hlist_insert(group_hash, &tte->hash);
-	} else {
-		tte = container_of(he, typeof(*tte), hash);
-	}
+	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	if (!he)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "tunnel group index not supported");
+	tte = container_of(he, typeof(*tte), hash);
 	*table = tte->flow_table;
 	DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
 		dev->data->port_id, key.tunnel_id, group, *table);
 	return 0;
-
-err:
-	if (tte)
-		mlx5_free(tte);
-	return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				  NULL, "tunnel group index not supported");
 }
 
 static int
@@ -7972,13 +7992,16 @@ struct mlx5_meter_domains_infos *
 		return NULL;
 	}
 	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
-					   NULL, NULL, NULL);
+					   mlx5_flow_tunnel_grp2tbl_create_cb,
+					   NULL,
+					   mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(priv->sh->ipool
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
 		mlx5_free(tunnel);
 		return NULL;
 	}
+	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8052,11 +8075,14 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
-					 0, NULL, NULL, NULL);
+					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 NULL,
+					 mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
+	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
drivers/net/mlx5/mlx5_flow_dv.c +2 -9
index 39bd6b2..5db2e04 100644
@@ -8100,15 +8100,8 @@ struct mlx5_flow_tbl_resource *
 					tbl_data->tunnel->groups :
 					thub->groups;
 		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
-		if (he) {
-			struct tunnel_tbl_entry *tte;
-			tte = container_of(he, typeof(*tte), hash);
-			MLX5_ASSERT(tte->flow_table == table_id);
-			mlx5_hlist_remove(tunnel_grp_hash, he);
-			mlx5_free(tte);
-		}
-		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-				tunnel_flow_tbl_to_id(table_id));
+		if (he)
+			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
 			"Table_id %#x tunnel %u group %u released.",
 			table_id,
Signature · 1 line
1.8.3.1
#118 lore raw

PATCH v4 31/34 net/mlx5: remove unused hash list operations

In reply to Suanming Mou

In previous commits the hash list objects have been converted to new thread safe hash list. The legacy hash list code can be removed now.

---
 drivers/net/mlx5/mlx5_utils.c | 38 -------------------------
 drivers/net/mlx5/mlx5_utils.h | 66 -------------------------------------------
 2 files changed, 104 deletions(-)
drivers/net/mlx5/mlx5_utils.c +0 -38
index df15f63..13590dd 100644
@@ -170,44 +170,6 @@ struct mlx5_hlist_entry*
 	return entry;
 }
 
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup_ex(struct mlx5_hlist *h, uint64_t key,
-		     mlx5_hlist_match_callback_fn cb, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h && cb && ctx);
-	idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	first = &h->heads[idx];
-	LIST_FOREACH(node, first, next) {
-		if (!cb(node, ctx))
-			return node;
-	}
-	return NULL;
-}
-
-int
-mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry,
-		     mlx5_hlist_match_callback_fn cb, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h && entry && cb && ctx);
-	idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
-	first = &h->heads[idx];
-	/* No need to reuse the lookup function. */
-	LIST_FOREACH(node, first, next) {
-		if (!cb(node, ctx))
-			return -EEXIST;
-	}
-	LIST_INSERT_HEAD(first, entry, next);
-	return 0;
-}
-
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
 {
drivers/net/mlx5/mlx5_utils.h +0 -66

84 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#119 lore raw

PATCH v4 32/34 net/mlx5: make tunnel hub list thread safe

In reply to Suanming Mou

This commit uses spinlock to protect the tunnel hub list in multiple thread.

---
 drivers/net/mlx5/mlx5_flow.c | 20 +++++++++++++++++---
 drivers/net/mlx5/mlx5_flow.h |  1 +
 2 files changed, 18 insertions(+), 3 deletions(-)
drivers/net/mlx5/mlx5_flow.c +17 -3
index 1e82030..a6e60af 100644
@@ -669,10 +669,14 @@ enum mlx5_expansion {
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
-		if (&tun->item == pmd_items)
+		if (&tun->item == pmd_items) {
+			LIST_REMOVE(tun, chain);
 			break;
+		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (!tun || num_items != 1)
 		return rte_flow_error_set(err, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
@@ -690,10 +694,14 @@ enum mlx5_expansion {
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
-		if (&tun->action == pmd_actions)
+		if (&tun->action == pmd_actions) {
+			LIST_REMOVE(tun, chain);
 			break;
+		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (!tun || num_actions != 1)
 		return rte_flow_error_set(err, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
@@ -5880,8 +5888,12 @@ struct rte_flow *
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
+
+		rte_spinlock_lock(&mlx5_tunnel_hub(dev)->sl);
 		tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
 		RTE_VERIFY(tunnel);
+		LIST_REMOVE(tunnel, chain);
+		rte_spinlock_unlock(&mlx5_tunnel_hub(dev)->sl);
 		if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
@@ -7940,7 +7952,6 @@ struct mlx5_meter_domains_infos *
 	DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
 		dev->data->port_id, tunnel->tunnel_id);
 	RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED));
-	LIST_REMOVE(tunnel, chain);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID],
 			tunnel->tunnel_id);
 	mlx5_hlist_destroy(tunnel->groups);
@@ -8029,6 +8040,7 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
 		if (!memcmp(app_tunnel, &tun->app_tunnel,
 			    sizeof(*app_tunnel))) {
@@ -8046,6 +8058,7 @@ struct mlx5_meter_domains_infos *
 			ret = -ENOMEM;
 		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (tun)
 		__atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED);
 
@@ -8074,6 +8087,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	if (!thub)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
+	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
 					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
 					 NULL,
drivers/net/mlx5/mlx5_flow.h +1 -0
index 4a8c2bf..8ef2a85 100644
@@ -945,6 +945,7 @@ struct mlx5_flow_tunnel {
 /** PMD tunnel related context */
 struct mlx5_flow_tunnel_hub {
 	LIST_HEAD(, mlx5_flow_tunnel) tunnels;
+	rte_spinlock_t sl;			/* Tunnel list spinlock. */
 	struct mlx5_hlist *groups;		/** non tunnel groups */
 };
 
Signature · 1 line
1.8.3.1
#120 lore raw

PATCH v4 33/34 net/mlx5: make shared action list thread safe

In reply to Suanming Mou

This commit uses spinlock to protect the shared action list in multiple thread.

---
 drivers/net/mlx5/linux/mlx5_os.c | 1 +
 drivers/net/mlx5/mlx5.h          | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c  | 5 +++++
 3 files changed, 7 insertions(+)
drivers/net/mlx5/linux/mlx5_os.c +1 -0
index 5856981..8612cab 100644
@@ -1534,6 +1534,7 @@
 		}
 		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
+	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
 	return eth_dev;
 error:
drivers/net/mlx5/mlx5.h +1 -0
index ac7a026..be21a9a 100644
@@ -912,6 +912,7 @@ struct mlx5_priv {
 	uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
 	struct mlx5_mp_id mp_id; /* ID of a multi-process process */
 	LIST_HEAD(fdir, mlx5_fdir_flow) fdir_flows; /* fdir flows. */
+	rte_spinlock_t shared_act_sl; /* Shared actions spinlock. */
 	LIST_HEAD(shared_action, rte_flow_shared_action) shared_actions;
 	/* shared actions */
 };
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index 5db2e04..e4325c1 100644
@@ -11189,7 +11189,9 @@ struct mlx5_cache_entry *
 	if (shared_action) {
 		__atomic_add_fetch(&shared_action->refcnt, 1,
 				   __ATOMIC_RELAXED);
+		rte_spinlock_lock(&priv->shared_act_sl);
 		LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next);
+		rte_spinlock_unlock(&priv->shared_act_sl);
 	}
 	return shared_action;
 }
@@ -11216,6 +11218,7 @@ struct mlx5_cache_entry *
 			 struct rte_flow_shared_action *action,
 			 struct rte_flow_error *error)
 {
+	struct mlx5_priv *priv = dev->data->dev_private;
 	int ret;
 
 	switch (action->type) {
@@ -11230,7 +11233,9 @@ struct mlx5_cache_entry *
 	}
 	if (ret)
 		return ret;
+	rte_spinlock_lock(&priv->shared_act_sl);
 	LIST_REMOVE(action, next);
+	rte_spinlock_unlock(&priv->shared_act_sl);
 	rte_free(action);
 	return 0;
 }
Signature · 1 line
1.8.3.1
#121 lore raw

PATCH v4 34/34 net/mlx5: remove shared context lock

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch removes shared data lock since all resources should support concurrent protection.

---
 doc/guides/nics/mlx5.rst               |   1 +
 doc/guides/rel_notes/release_20_11.rst |   1 +
 drivers/net/mlx5/linux/mlx5_os.c       |   4 +-
 drivers/net/mlx5/mlx5.h                |   1 -
 drivers/net/mlx5/mlx5_flow_dv.c        | 209 ++++-----------------------------
 5 files changed, 27 insertions(+), 189 deletions(-)
doc/guides/nics/mlx5.rst +1 -0
index e5e55fc..8c59cc6 100644
@@ -96,6 +96,7 @@ Features
 - Per packet no-inline hint flag to disable packet data copying into Tx descriptors.
 - Hardware LRO.
 - Hairpin.
+- Multiple-thread flow insertion.
 
 Limitations
 -----------
doc/guides/rel_notes/release_20_11.rst +1 -0
index e284771..0742fc8 100644
@@ -358,6 +358,7 @@ New Features
   * Added support for QinQ packets matching.
   * Added support for the new vlan fields ``has_vlan`` in the eth item and
     ``has_more_vlan`` in the vlan item.
+  * Added support for PMD level multiple-thread flow insertion.
 
 * **Updated vhost sample application.**
 
drivers/net/mlx5/linux/mlx5_os.c +2 -2
index 8612cab..d4f2194 100644
@@ -321,7 +321,6 @@
 		err = errno;
 		goto error;
 	}
-	pthread_mutex_init(&sh->dv_mutex, NULL);
 	sh->tx_domain = domain;
 #ifdef HAVE_MLX5DV_DR_ESWITCH
 	if (priv->config.dv_esw_en) {
@@ -435,7 +434,6 @@
 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 		sh->pop_vlan_action = NULL;
 	}
-	pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
 	if (sh->default_miss_action)
 		mlx5_glue->destroy_flow_action
@@ -1536,6 +1534,8 @@
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
+	if (priv->config.dv_flow_en)
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
 	return eth_dev;
 error:
 	if (priv) {
drivers/net/mlx5/mlx5.h +0 -1
index be21a9a..9a458cd 100644
@@ -639,7 +639,6 @@ struct mlx5_dev_ctx_shared {
 	/* Packet pacing related structure. */
 	struct mlx5_dev_txpp txpp;
 	/* Shared DV/DR flow data section. */
-	pthread_mutex_t dv_mutex; /* DV context mutex. */
 	uint32_t dv_meta_mask; /* flow META metadata supported mask. */
 	uint32_t dv_mark_mask; /* flow MARK metadata supported mask. */
 	uint32_t dv_regc0_mask; /* available bits of metatada reg_c[0]. */
drivers/net/mlx5/mlx5_flow_dv.c +23 -186

296 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#122 lore raw

v5 cover net/mlx5: support multiple-thread flow operations

In reply to Suanming Mou

This patch set contains multiple-thread flow operations support for the flow objects.

Suanming Mou (19):
net/mlx5: use thread safe index pool for flow objects net/mlx5: make meter action thread safe net/mlx5: make VLAN network interface thread safe net/mlx5: create global jump action
net/mlx5: create global default miss action net/mlx5: create global drop action
net/mlx5: add flow table tunnel offload attribute net/mlx5: fix redundant Direct Verbs resources allocate net/mlx5: remove unused mreg copy code net/mlx5: make header reformat action thread safe net/mlx5: optimize shared RSS list operation net/mlx5: make Rx queue thread safe
net/mlx5: simplify sample attributes
net/mlx5: fix sample register error flow net/mlx5: make sample and mirror action thread safe net/mlx5: make tunnel offloading table thread safe net/mlx5: remove unused hash list operations net/mlx5: make tunnel hub list thread safe net/mlx5: make shared action list thread safe

Xueming Li (15):
net/mlx5: use thread specific flow workspace net/mlx5: reuse flow Id as hairpin Id net/mlx5: indexed pool supports zero size entry net/mlx5: use indexed pool as ID generator net/mlx5: make rte flow list thread safe net/mlx5: support concurrent access for hash list net/mlx5: make flow table cache thread safe net/mlx5: make flow tag list thread safe net/mlx5: make flow modify action list thread safe net/mlx5: make metadata copy flow list thread safe net/mlx5: introduce thread safe linked list cache net/mlx5: make matcher list thread safe net/mlx5: make port ID action cache thread safe net/mlx5: make push VLAN action cache thread safe net/mlx5: remove shared context lock
---

v5:
- fix table refcnt leak issue.

v4:
- rebase the code on top of the latest code to fix the conflict.

v3:
- rebase the code on top of the new added features.
- New feature code adaption for multiple-thread support.

v2:
- fix review comments.
- remove unused mreg copy code.
- drop the sh lock removing patch as new feature still not fully updated.

---

 doc/guides/nics/mlx5.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst |    1 +
 drivers/common/mlx5/linux/mlx5_nl.h    |    1 +
 drivers/net/mlx5/linux/mlx5_os.c       |  119 +-
 drivers/net/mlx5/linux/mlx5_vlan_os.c  |    5 +
 drivers/net/mlx5/mlx5.c                |  278 +---
 drivers/net/mlx5/mlx5.h                |   69 +-
 drivers/net/mlx5/mlx5_flow.c           |  756 +++++------
 drivers/net/mlx5/mlx5_flow.h           |  164 ++-
 drivers/net/mlx5/mlx5_flow_dv.c        | 2259 ++++++++++++++------------------
 drivers/net/mlx5/mlx5_flow_meter.c     |   72 +-
 drivers/net/mlx5/mlx5_flow_verbs.c     |   86 +-
 drivers/net/mlx5/mlx5_rxq.c            |  302 +++--
 drivers/net/mlx5/mlx5_rxtx.h           |   23 +-
 drivers/net/mlx5/mlx5_utils.c          |  345 ++++-
 drivers/net/mlx5/mlx5_utils.h          |  320 ++++-
 16 files changed, 2464 insertions(+), 2337 deletions(-)
Signature · 1 line
1.8.3.1
#123 lore raw

PATCH v5 1/34 net/mlx5: use thread safe index pool for flow objects

As mlx5 PMD is changed to be thread safe, all the flow-related sub-objects inside the PMD should be thread safe. This commit changes the index memory pools' lock configuration to be enabled. That makes the index pool be thread safe.

---
 drivers/net/mlx5/mlx5.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5.c +11 -11
index 27c9c2a..dc5b7a4 100644
@@ -191,7 +191,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -202,7 +202,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -213,7 +213,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -224,7 +224,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -235,7 +235,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -246,7 +246,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -257,7 +257,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -269,7 +269,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -280,7 +280,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -291,7 +291,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -306,7 +306,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
Signature · 1 line
1.8.3.1
#124 lore raw

PATCH v5 2/34 net/mlx5: use thread specific flow workspace

In reply to Suanming Mou

From: Xueming Li <[email protected]>

As part of multi-thread flow support, this patch moves flow intermediate data to thread specific, makes them a flow workspace. The workspace is allocated per thread, destroyed along with thread life-cycle.

---
 drivers/net/mlx5/linux/mlx5_os.c   |   5 --
 drivers/net/mlx5/mlx5.c            |   2 -
 drivers/net/mlx5/mlx5.h            |   6 --
 drivers/net/mlx5/mlx5_flow.c       | 168 ++++++++++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_flow.h       |  15 +++-
 drivers/net/mlx5/mlx5_flow_dv.c    |  40 +++++----
 drivers/net/mlx5/mlx5_flow_verbs.c |  24 +++---
 7 files changed, 171 insertions(+), 89 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +0 -5
index c890998..4ba6d8e 100644
@@ -1449,11 +1449,6 @@
 			err = ENOTSUP;
 			goto error;
 	}
-	/*
-	 * Allocate the buffer for flow creating, just once.
-	 * The allocation must be done before any flow creating.
-	 */
-	mlx5_flow_alloc_intermediate(eth_dev);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.c +0 -2
index dc5b7a4..bb494c1 100644
@@ -1400,8 +1400,6 @@ struct mlx5_dev_ctx_shared *
 	mlx5_flow_list_flush(dev, &priv->flows, true);
 	mlx5_shared_action_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
-	/* Free the intermediate buffers for flow creation. */
-	mlx5_flow_free_intermediate(dev);
 	/* Prevent crashes when queues are still in use. */
 	dev->rx_pkt_burst = removed_rx_burst;
 	dev->tx_pkt_burst = removed_tx_burst;
drivers/net/mlx5/mlx5.h +0 -6
index 8de5842..5400b95 100644
@@ -854,10 +854,6 @@ struct mlx5_priv {
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	uint32_t flows; /* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
-	void *inter_flows; /* Intermediate resources for flow creation. */
-	void *rss_desc; /* Intermediate rss description resources. */
-	int flow_idx; /* Intermediate device flow index. */
-	int flow_nested_idx; /* Intermediate device flow index, nested. */
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
@@ -1104,8 +1100,6 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 void mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list);
 int mlx5_flow_start_default(struct rte_eth_dev *dev);
 void mlx5_flow_stop_default(struct rte_eth_dev *dev);
-void mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev);
-void mlx5_flow_free_intermediate(struct rte_eth_dev *dev);
 int mlx5_flow_verify(struct rte_eth_dev *dev);
 int mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, uint32_t queue);
 int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow.c +120 -48

235 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +14 -1
index 06db440..668fcc3 100644
@@ -75,6 +75,9 @@ enum mlx5_feature_name {
 	MLX5_MTR_SFX,
 };
 
+/* Default queue number. */
+#define MLX5_RSSQ_DEFAULT_NUM 16
+
 #define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
@@ -603,7 +606,7 @@ struct mlx5_flow_rss_desc {
 	uint32_t queue_num; /**< Number of entries in @p queue. */
 	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
-	uint16_t queue[]; /**< Destination queues to redirect traffic to. */
+	uint16_t *queue; /**< Destination queues. */
 };
 
 /* PMD flow priority for tunnel */
@@ -1102,6 +1105,15 @@ struct rte_flow_shared_action {
 	};
 };
 
+/* Thread specific flow workspace intermediate data. */
+struct mlx5_flow_workspace {
+	struct mlx5_flow flows[MLX5_NUM_MAX_DEV_FLOWS];
+	struct mlx5_flow_rss_desc rss_desc[2];
+	uint32_t rssq_num[2]; /* Allocated queue num in rss_desc. */
+	int flow_idx; /* Intermediate device flow index. */
+	int flow_nested_idx; /* Intermediate device flow index, nested. */
+};
+
 typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
 				    const struct rte_flow_attr *attr,
 				    const struct rte_flow_item items[],
@@ -1204,6 +1216,7 @@ struct mlx5_flow_driver_ops {
 
 /* mlx5_flow.c */
 
+struct mlx5_flow_workspace *mlx5_flow_get_thread_workspace(void);
 struct mlx5_flow_id_pool *mlx5_flow_id_pool_alloc(uint32_t max_id);
 void mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool);
 uint32_t mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id);
drivers/net/mlx5/mlx5_flow_dv.c +23 -17
index 4a35010..f4f8e15 100644
@@ -6253,9 +6253,11 @@ struct field_modify_info modify_tcp[] = {
 	struct mlx5_flow *dev_flow;
 	struct mlx5_flow_handle *dev_handle;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
+	MLX5_ASSERT(wks);
 	/* In case of corrupting the memory. */
-	if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+	if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
 		rte_flow_error_set(error, ENOSPC,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "not free temporary device flow");
@@ -6269,8 +6271,8 @@ struct field_modify_info modify_tcp[] = {
 				   "not enough memory to create flow handle");
 		return NULL;
 	}
-	/* No multi-thread supporting. */
-	dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+	MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
+	dev_flow = &wks->flows[wks->flow_idx++];
 	dev_flow->handle = dev_handle;
 	dev_flow->handle_idx = handle_idx;
 	/*
@@ -8942,11 +8944,12 @@ struct field_modify_info modify_tcp[] = {
 	const struct rte_flow_action_queue *queue;
 	struct mlx5_flow_sub_actions_list *sample_act;
 	struct mlx5_flow_sub_actions_idx *sample_idx;
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	uint64_t action_flags = 0;
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	sample_act = &res->sample_act;
 	sample_idx = &res->sample_idx;
 	sample_action = (const struct rte_flow_action_sample *)action->conf;
@@ -9148,18 +9151,18 @@ struct field_modify_info modify_tcp[] = {
 			     uint64_t action_flags,
 			     struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/* update normal path action resource into last index of array */
 	uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
 	struct mlx5_flow_sub_actions_list *sample_act =
 					&mdest_res->sample_act[dest_index];
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	uint32_t normal_idx = 0;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx;
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	if (num_of_dest > 1) {
 		if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
 			/* Handle QP action for mirroring */
@@ -9249,9 +9252,8 @@ struct field_modify_info modify_tcp[] = {
 	struct mlx5_dev_config *dev_conf = &priv->config;
 	struct rte_flow *flow = dev_flow->flow;
 	struct mlx5_flow_handle *handle = dev_flow->handle;
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	uint64_t item_flags = 0;
 	uint64_t last_item = 0;
 	uint64_t action_flags = 0;
@@ -9297,6 +9299,8 @@ struct field_modify_info modify_tcp[] = {
 		.fdb_def_rule = !!priv->fdb_def_rule,
 	};
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
 	memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
 	mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
@@ -10280,6 +10284,7 @@ struct field_modify_info modify_tcp[] = {
 			   struct mlx5_hrxq **hrxq)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 	uint32_t hrxq_idx;
 
 	if (flow->shared_rss) {
@@ -10295,8 +10300,7 @@ struct field_modify_info modify_tcp[] = {
 		}
 	} else {
 		struct mlx5_flow_rss_desc *rss_desc =
-				&((struct mlx5_flow_rss_desc *)priv->rss_desc)
-				[!!priv->flow_nested_idx];
+				&wks->rss_desc[!!wks->flow_nested_idx];
 
 		MLX5_ASSERT(rss_desc->queue_num);
 		hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
@@ -10347,9 +10351,11 @@ struct field_modify_info modify_tcp[] = {
 	int n;
 	int err;
 	int idx;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
-	for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
-		dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
+	MLX5_ASSERT(wks);
+	for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) {
+		dev_flow = &wks->flows[idx];
 		dv = &dev_flow->dv;
 		dh = dev_flow->handle;
 		dv_h = &dh->dvh;
drivers/net/mlx5/mlx5_flow_verbs.c +14 -10
index d04c37f..0ec9acd 100644
@@ -1626,7 +1626,9 @@
 	struct mlx5_flow *dev_flow;
 	struct mlx5_flow_handle *dev_handle;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
+	MLX5_ASSERT(wks);
 	size += flow_verbs_get_actions_size(actions);
 	size += flow_verbs_get_items_size(items);
 	if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
@@ -1636,7 +1638,7 @@
 		return NULL;
 	}
 	/* In case of corrupting the memory. */
-	if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+	if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
 		rte_flow_error_set(error, ENOSPC,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "not free temporary device flow");
@@ -1650,8 +1652,8 @@
 				   "not enough memory to create flow handle");
 		return NULL;
 	}
-	/* No multi-thread supporting. */
-	dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+	MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
+	dev_flow = &wks->flows[wks->flow_idx++];
 	dev_flow->handle = dev_handle;
 	dev_flow->handle_idx = handle_idx;
 	/* Memcpy is used, only size needs to be cleared to 0. */
@@ -1695,11 +1697,12 @@
 	uint64_t priority = attr->priority;
 	uint32_t subpriority = 0;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	char errstr[32];
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	if (priority == MLX5_FLOW_PRIO_RSVD)
 		priority = priv->config.flow_prio - 1;
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
@@ -1956,9 +1959,11 @@
 	uint32_t dev_handles;
 	int err;
 	int idx;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
-	for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
-		dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
+	MLX5_ASSERT(wks);
+	for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) {
+		dev_flow = &wks->flows[idx];
 		handle = dev_flow->handle;
 		if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
 			hrxq = mlx5_drop_action_create(dev);
@@ -1972,8 +1977,7 @@
 		} else {
 			uint32_t hrxq_idx;
 			struct mlx5_flow_rss_desc *rss_desc =
-				&((struct mlx5_flow_rss_desc *)priv->rss_desc)
-				[!!priv->flow_nested_idx];
+				&wks->rss_desc[!!wks->flow_nested_idx];
 
 			MLX5_ASSERT(rss_desc->queue_num);
 			hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
Signature · 1 line
1.8.3.1
#125 lore raw

PATCH v5 3/34 net/mlx5: reuse flow Id as hairpin Id

In reply to Suanming Mou

From: Xueming Li <[email protected]>

Hairpin flow matching required a unique flow ID for matching. This patch reuses flow ID as hairpin flow ID, this will save some code to generate a separate hairpin ID, also saves flow memory by removing hairpin ID.

---
 drivers/net/mlx5/mlx5.c      | 11 -----------
 drivers/net/mlx5/mlx5.h      |  1 -
 drivers/net/mlx5/mlx5_flow.c | 30 +++++++++---------------------
 drivers/net/mlx5/mlx5_flow.h |  6 +-----
 4 files changed, 10 insertions(+), 38 deletions(-)
drivers/net/mlx5/mlx5.c +0 -11
index bb494c1..eca97b0 100644
@@ -979,13 +979,6 @@ struct mlx5_dev_ctx_shared *
 		MLX5_ASSERT(sh->devx_rx_uar);
 		MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar));
 	}
-	sh->flow_id_pool = mlx5_flow_id_pool_alloc
-					((1 << HAIRPIN_FLOW_ID_BITS) - 1);
-	if (!sh->flow_id_pool) {
-		DRV_LOG(ERR, "can't create flow id pool");
-		err = ENOMEM;
-		goto error;
-	}
 #ifndef RTE_ARCH_64
 	/* Initialize UAR access locks for 32bit implementations. */
 	rte_spinlock_init(&sh->uar_lock_cq);
@@ -1047,8 +1040,6 @@ struct mlx5_dev_ctx_shared *
 		claim_zero(mlx5_glue->dealloc_pd(sh->pd));
 	if (sh->ctx)
 		claim_zero(mlx5_glue->close_device(sh->ctx));
-	if (sh->flow_id_pool)
-		mlx5_flow_id_pool_release(sh->flow_id_pool);
 	mlx5_free(sh);
 	MLX5_ASSERT(err > 0);
 	rte_errno = err;
@@ -1119,8 +1110,6 @@ struct mlx5_dev_ctx_shared *
 		mlx5_glue->devx_free_uar(sh->devx_rx_uar);
 	if (sh->ctx)
 		claim_zero(mlx5_glue->close_device(sh->ctx));
-	if (sh->flow_id_pool)
-		mlx5_flow_id_pool_release(sh->flow_id_pool);
 	pthread_mutex_destroy(&sh->txpp.mutex);
 	mlx5_free(sh);
 	return;
drivers/net/mlx5/mlx5.h +0 -1
index 5400b95..8e00065 100644
@@ -668,7 +668,6 @@ struct mlx5_dev_ctx_shared {
 	void *devx_comp; /* DEVX async comp obj. */
 	struct mlx5_devx_obj *tis; /* TIS object. */
 	struct mlx5_devx_obj *td; /* Transport domain. */
-	struct mlx5_flow_id_pool *flow_id_pool; /* Flow ID pool. */
 	void *tx_uar; /* Tx/packet pacing shared UAR. */
 	struct mlx5_flex_parser_profiles fp[MLX5_FLEX_PARSER_MAX];
 	/* Flex parser profiles information. */
drivers/net/mlx5/mlx5_flow.c +9 -21
index e6a4571..4733885 100644
@@ -4154,9 +4154,8 @@ struct rte_flow_shared_action *
 		   struct rte_flow_action actions_rx[],
 		   struct rte_flow_action actions_tx[],
 		   struct rte_flow_item pattern_tx[],
-		   uint32_t *flow_id)
+		   uint32_t flow_id)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_action_raw_encap *raw_encap;
 	const struct rte_flow_action_raw_decap *raw_decap;
 	struct mlx5_rte_flow_action_set_tag *set_tag;
@@ -4166,7 +4165,6 @@ struct rte_flow_shared_action *
 	char *addr;
 	int encap = 0;
 
-	mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
@@ -4235,7 +4233,7 @@ struct rte_flow_shared_action *
 	set_tag = (void *)actions_rx;
 	set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
 	MLX5_ASSERT(set_tag->id > REG_NON);
-	set_tag->data = *flow_id;
+	set_tag->data = flow_id;
 	tag_action->conf = set_tag;
 	/* Create Tx item list. */
 	rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
@@ -4244,7 +4242,7 @@ struct rte_flow_shared_action *
 	item->type = (enum rte_flow_item_type)
 		     MLX5_RTE_FLOW_ITEM_TYPE_TAG;
 	tag_item = (void *)addr;
-	tag_item->data = *flow_id;
+	tag_item->data = flow_id;
 	tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
 	MLX5_ASSERT(set_tag->id > REG_NON);
 	item->spec = tag_item;
@@ -5630,7 +5628,6 @@ struct tunnel_default_miss_ctx {
 	uint32_t i;
 	uint32_t idx = 0;
 	int hairpin_flow;
-	uint32_t hairpin_id = 0;
 	struct rte_flow_attr attr_tx = { .priority = 0 };
 	struct rte_flow_attr attr_factor = {0};
 	const struct rte_flow_action *actions;
@@ -5659,6 +5656,11 @@ struct tunnel_default_miss_ctx {
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
+	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	if (!flow) {
+		rte_errno = ENOMEM;
+		goto error_before_hairpin_split;
+	}
 	if (hairpin_flow > 0) {
 		if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
 			rte_errno = EINVAL;
@@ -5666,17 +5668,10 @@ struct tunnel_default_miss_ctx {
 		}
 		flow_hairpin_split(dev, actions, actions_rx.actions,
 				   actions_hairpin_tx.actions, items_tx.items,
-				   &hairpin_id);
+				   idx);
 		p_actions_rx = actions_rx.actions;
 	}
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
-	if (!flow) {
-		rte_errno = ENOMEM;
-		goto error_before_flow;
-	}
 	flow->drv_type = flow_get_drv_type(dev, &attr_factor);
-	if (hairpin_id != 0)
-		flow->hairpin_flow_id = hairpin_id;
 	MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
 		    flow->drv_type < MLX5_FLOW_TYPE_MAX);
 	memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
@@ -5812,11 +5807,7 @@ struct tunnel_default_miss_ctx {
 	flow_drv_destroy(dev, flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
 	rte_errno = ret; /* Restore rte_errno. */
-error_before_flow:
 	ret = rte_errno;
-	if (hairpin_id)
-		mlx5_flow_id_release(priv->sh->flow_id_pool,
-				     hairpin_id);
 	rte_errno = ret;
 	wks->flow_idx = wks->flow_nested_idx;
 	if (wks->flow_nested_idx)
@@ -5974,9 +5965,6 @@ struct rte_flow *
 	 */
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
-	if (flow->hairpin_flow_id)
-		mlx5_flow_id_release(priv->sh->flow_id_pool,
-				     flow->hairpin_flow_id);
 	flow_drv_destroy(dev, flow);
 	if (list)
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
drivers/net/mlx5/mlx5_flow.h +1 -5
index 668fcc3..e6ec709 100644
@@ -922,8 +922,6 @@ struct mlx5_fdir_flow {
 	uint32_t rix_flow; /* Index to flow. */
 };
 
-#define HAIRPIN_FLOW_ID_BITS 28
-
 #define MLX5_MAX_TUNNELS 256
 #define MLX5_TNL_MISS_RULE_PRIORITY 3
 #define MLX5_TNL_MISS_FDB_JUMP_GRP  0x1234faac
@@ -1039,14 +1037,12 @@ struct rte_flow {
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t fdir:1; /**< Identifier of associated FDIR if any. */
 	uint32_t tunnel:1;
-	uint32_t hairpin_flow_id:HAIRPIN_FLOW_ID_BITS;
-	/**< The flow id used for hairpin. */
 	uint32_t copy_applied:1; /**< The MARK copy Flow os applied. */
+	uint32_t meter:16; /**< Holds flow meter id. */
 	uint32_t rix_mreg_copy;
 	/**< Index to metadata register copy table resource. */
 	uint32_t counter; /**< Holds flow counter. */
 	uint32_t tunnel_id;  /**< Tunnel id */
-	uint16_t meter; /**< Holds flow meter id. */
 } __rte_packed;
 
 /*
Signature · 1 line
1.8.3.1
#126 lore raw

PATCH v5 4/34 net/mlx5: indexed pool supports zero size entry

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To make indexed pool to be used as ID generator, this patch allows entry size to be zero.

---
 drivers/net/mlx5/mlx5_utils.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +2 -2
index 9a54fda..3a1f87a 100644
@@ -230,7 +230,7 @@ struct mlx5_indexed_pool *
 	struct mlx5_indexed_pool *pool;
 	uint32_t i;
 
-	if (!cfg || !cfg->size || (!cfg->malloc ^ !cfg->free) ||
+	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -391,7 +391,7 @@ struct mlx5_indexed_pool *
 {
 	void *entry = mlx5_ipool_malloc(pool, idx);
 
-	if (entry)
+	if (entry && pool->cfg.size)
 		memset(entry, 0, pool->cfg.size);
 	return entry;
 }
Signature · 1 line
1.8.3.1
#127 lore raw

PATCH v5 5/34 net/mlx5: use indexed pool as ID generator

In reply to Suanming Mou

From: Xueming Li <[email protected]>

The ID generation API used an integer pool to save released ID, To support multiple flow, it has to be enhanced to be thread safe.

Indexed pool could be used to generate unique ID by setting size of pool entry to zero. Since bitmap is used, an extra benefits is saving memory to about one bit per entry. Further more indexed pool could be thread safe by enabling lock.

This patch leverages indexed pool to generate ID, removes unused ID generating API.

---
 drivers/net/mlx5/linux/mlx5_os.c |  13 ----
 drivers/net/mlx5/mlx5.c          | 136 +++++----------------------------------
 drivers/net/mlx5/mlx5.h          |   4 +-
 drivers/net/mlx5/mlx5_flow.c     | 116 +++++++++++++++------------------
 drivers/net/mlx5/mlx5_flow.h     |   7 --
 drivers/net/mlx5/mlx5_flow_dv.c  |   4 +-
 drivers/net/mlx5/mlx5_utils.c    |   5 ++
 7 files changed, 78 insertions(+), 207 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +0 -13
index 4ba6d8e..0b59e74 100644
@@ -1395,17 +1395,6 @@
 		err = mlx5_alloc_shared_dr(priv);
 		if (err)
 			goto error;
-		/*
-		 * RSS id is shared with meter flow id. Meter flow id can only
-		 * use the 24 MSB of the register.
-		 */
-		priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >>
-				     MLX5_MTR_COLOR_BITS);
-		if (!priv->qrss_id_pool) {
-			DRV_LOG(ERR, "can't create flow id pool");
-			err = ENOMEM;
-			goto error;
-		}
 	}
 	if (config->devx && config->dv_flow_en && config->dest_tir) {
 		priv->obj_ops = devx_obj_ops;
@@ -1492,8 +1481,6 @@
 			close(priv->nl_socket_rdma);
 		if (priv->vmwa_context)
 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
-		if (priv->qrss_id_pool)
-			mlx5_flow_id_pool_release(priv->qrss_id_pool);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		mlx5_free(priv);
drivers/net/mlx5/mlx5.c +15 -121

148 lines not on this page · raw message

drivers/net/mlx5/mlx5.h +3 -1
index 8e00065..5bda233 100644
@@ -47,6 +47,9 @@ enum mlx5_ipool_index {
 	MLX5_IPOOL_HRXQ, /* Pool for hrxq resource. */
 	MLX5_IPOOL_MLX5_FLOW, /* Pool for mlx5 flow handle. */
 	MLX5_IPOOL_RTE_FLOW, /* Pool for rte_flow. */
+	MLX5_IPOOL_RSS_EXPANTION_FLOW_ID, /* Pool for Queue/RSS flow ID. */
+	MLX5_IPOOL_TUNNEL_ID, /* Pool for flow tunnel ID. */
+	MLX5_IPOOL_TNL_TBL_ID, /* Pool for tunnel table ID. */
 	MLX5_IPOOL_MAX,
 };
 
@@ -877,7 +880,6 @@ struct mlx5_priv {
 	int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
 	struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */
 	struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
-	struct mlx5_flow_id_pool *qrss_id_pool;
 	struct mlx5_hlist *mreg_cp_tbl;
 	/* Hash table of Rx metadata register copy table. */
 	uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */
drivers/net/mlx5/mlx5_flow.c +53 -63

249 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +0 -7
index e6ec709..629cdbe 100644
@@ -951,8 +951,6 @@ struct mlx5_flow_tunnel {
 /** PMD tunnel related context */
 struct mlx5_flow_tunnel_hub {
 	LIST_HEAD(, mlx5_flow_tunnel) tunnels;
-	struct mlx5_flow_id_pool *tunnel_ids;
-	struct mlx5_flow_id_pool *table_ids;
 	struct mlx5_hlist *groups;		/** non tunnel groups */
 };
 
@@ -1213,11 +1211,6 @@ struct mlx5_flow_driver_ops {
 /* mlx5_flow.c */
 
 struct mlx5_flow_workspace *mlx5_flow_get_thread_workspace(void);
-struct mlx5_flow_id_pool *mlx5_flow_id_pool_alloc(uint32_t max_id);
-void mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool);
-uint32_t mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id);
-uint32_t mlx5_flow_id_release(struct mlx5_flow_id_pool *pool,
-			      uint32_t id);
 __extension__
 struct flow_grp_info {
 	uint64_t external:1;
drivers/net/mlx5/mlx5_flow_dv.c +2 -2
index f4f8e15..bfcba45 100644
@@ -8132,8 +8132,8 @@ struct field_modify_info modify_tcp[] = {
 				mlx5_hlist_remove(tunnel_grp_hash, he);
 				mlx5_free(tte);
 			}
-			mlx5_flow_id_release(mlx5_tunnel_hub(dev)->table_ids,
-					     tunnel_flow_tbl_to_id(table_id));
+			mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+					tunnel_flow_tbl_to_id(table_id));
 			DRV_LOG(DEBUG,
 				"port %u release table_id %#x tunnel %u group %u",
 				dev->data->port_id, table_id,
drivers/net/mlx5/mlx5_utils.c +5 -0
index 3a1f87a..7a6b0c6 100644
@@ -362,6 +362,11 @@ struct mlx5_indexed_pool *
 	MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx));
 	rte_bitmap_clear(trunk->bmp, iidx);
 	p = &trunk->data[iidx * pool->cfg.size];
+	/*
+	 * The ipool index should grow continually from small to big,
+	 * some features as metering only accept limited bits of index.
+	 * Random index with MSB set may be rejected.
+	 */
 	iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx);
 	iidx += 1; /* non-zero index. */
 	trunk->free--;
Signature · 1 line
1.8.3.1
#128 lore raw

PATCH v5 6/34 net/mlx5: make rte flow list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow operations, this patch introduces list lock for the rte_flow list manages all the rte_flow handlers.

---
 drivers/net/mlx5/linux/mlx5_os.c |  1 +
 drivers/net/mlx5/mlx5.h          |  1 +
 drivers/net/mlx5/mlx5_flow.c     | 10 ++++++++--
 3 files changed, 10 insertions(+), 2 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -0
index 0b59e74..a579dde 100644
@@ -1358,6 +1358,7 @@
 				      MLX5_MAX_MAC_ADDRESSES);
 	priv->flows = 0;
 	priv->ctrl_flows = 0;
+	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
 	TAILQ_INIT(&priv->flow_meter_profiles);
 	/* Hint libmlx5 to use PMD allocator for data plane resources */
drivers/net/mlx5/mlx5.h +1 -0
index 5bda233..9ab2976 100644
@@ -856,6 +856,7 @@ struct mlx5_priv {
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	uint32_t flows; /* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
+	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
drivers/net/mlx5/mlx5_flow.c +8 -2
index ed2acd1..cc31801 100644
@@ -5774,9 +5774,12 @@ struct tunnel_default_miss_ctx {
 		if (ret < 0)
 			goto error;
 	}
-	if (list)
+	if (list) {
+		rte_spinlock_lock(&priv->flow_list_lock);
 		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
 			     flow, next);
+		rte_spinlock_unlock(&priv->flow_list_lock);
+	}
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	/* Nested flow creation index recovery. */
@@ -5957,9 +5960,12 @@ struct rte_flow *
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list)
+	if (list) {
+		rte_spinlock_lock(&priv->flow_list_lock);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
 			     flow_idx, flow, next);
+		rte_spinlock_unlock(&priv->flow_list_lock);
+	}
 	flow_mreg_del_copy_action(dev, flow);
 	if (flow->fdir) {
 		LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
Signature · 1 line
1.8.3.1
#129 lore raw

PATCH v5 7/34 net/mlx5: make meter action thread safe

In reply to Suanming Mou

This commit adds the spinlock for the meter action to make it be thread safe. Atomic reference counter in all is not enough as the meter action should be created synchronized with reference counter increment. With only atomic reference counter, even the counter is increased, the action may still not be created.

---
 drivers/net/mlx5/mlx5_flow.h       |  2 ++
 drivers/net/mlx5/mlx5_flow_meter.c | 72 ++++++++++++++++++++------------------
 2 files changed, 39 insertions(+), 35 deletions(-)
drivers/net/mlx5/mlx5_flow.h +2 -0
index 629cdbe..fde5007 100644
@@ -845,6 +845,8 @@ struct mlx5_flow_meter {
 	struct mlx5_flow_meter_profile *profile;
 	/**< Meter profile parameters. */
 
+	rte_spinlock_t sl; /**< Meter action spinlock. */
+
 	/** Policer actions (per meter output color). */
 	enum rte_mtr_policer_action action[RTE_COLORS];
 
drivers/net/mlx5/mlx5_flow_meter.c +37 -35

107 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#130 lore raw

PATCH v5 8/34 net/mlx5: make VLAN network interface thread safe

In reply to Suanming Mou

This commit protects the VLAN VM workaround area using a spinlock in multiple-thread flow insertion to make it thread safe.

---
 drivers/common/mlx5/linux/mlx5_nl.h   | 1 +
 drivers/net/mlx5/linux/mlx5_vlan_os.c | 5 +++++
 2 files changed, 6 insertions(+)
drivers/common/mlx5/linux/mlx5_nl.h +1 -0
index 53021e1..15129ff 100644
@@ -25,6 +25,7 @@ struct mlx5_nl_vlan_dev {
 struct mlx5_nl_vlan_vmwa_context {
 	int nl_socket;
 	uint32_t vf_ifindex;
+	rte_spinlock_t sl;
 	struct mlx5_nl_vlan_dev vlan_dev[4096];
 };
 
drivers/net/mlx5/linux/mlx5_vlan_os.c +5 -0
index 92fc17d..40e895e 100644
@@ -44,12 +44,14 @@
 	if (!vlan->created || !vmwa)
 		return;
 	vlan->created = 0;
+	rte_spinlock_lock(&vmwa->sl);
 	MLX5_ASSERT(vlan_dev[vlan->tag].refcnt);
 	if (--vlan_dev[vlan->tag].refcnt == 0 &&
 	    vlan_dev[vlan->tag].ifindex) {
 		mlx5_nl_vlan_vmwa_delete(vmwa, vlan_dev[vlan->tag].ifindex);
 		vlan_dev[vlan->tag].ifindex = 0;
 	}
+	rte_spinlock_unlock(&vmwa->sl);
 }
 
 /**
@@ -72,6 +74,7 @@
 	MLX5_ASSERT(priv->vmwa_context);
 	if (vlan->created || !vmwa)
 		return;
+	rte_spinlock_lock(&vmwa->sl);
 	if (vlan_dev[vlan->tag].refcnt == 0) {
 		MLX5_ASSERT(!vlan_dev[vlan->tag].ifindex);
 		vlan_dev[vlan->tag].ifindex =
@@ -82,6 +85,7 @@
 		vlan_dev[vlan->tag].refcnt++;
 		vlan->created = 1;
 	}
+	rte_spinlock_unlock(&vmwa->sl);
 }
 
 /*
@@ -131,6 +135,7 @@
 			" for VLAN workaround context");
 		return NULL;
 	}
+	rte_spinlock_init(&vmwa->sl);
 	vmwa->nl_socket = mlx5_nl_init(NETLINK_ROUTE);
 	if (vmwa->nl_socket < 0) {
 		DRV_LOG(WARNING,
Signature · 1 line
1.8.3.1
#131 lore raw

PATCH v5 9/34 net/mlx5: create global jump action

In reply to Suanming Mou

This commit changes the jump action in table to be created with table creation in advanced. In this case, the jump action is safe to be used in multiple thread. The jump action will be destroyed when table is not used anymore and released.

---
 drivers/net/mlx5/mlx5_flow.h    |  2 --
 drivers/net/mlx5/mlx5_flow_dv.c | 56 +++++++++++++----------------------------
 2 files changed, 17 insertions(+), 41 deletions(-)
drivers/net/mlx5/mlx5_flow.h +0 -2
index fde5007..cc64792 100644
@@ -484,8 +484,6 @@ struct mlx5_flow_dv_modify_hdr_resource {
 
 /* Jump action resource structure. */
 struct mlx5_flow_dv_jump_tbl_resource {
-	uint32_t refcnt; /**< Reference counter. */
-	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	void *action; /**< Pointer to the rdma core action. */
 };
 
drivers/net/mlx5/mlx5_flow_dv.c +17 -39

85 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#132 lore raw

PATCH v5 10/34 net/mlx5: create global default miss action

In reply to Suanming Mou

This commit creates the global default miss action instead of maintain it in flow insertion time. This makes the action to be thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c |  7 ++++
 drivers/net/mlx5/mlx5.h          |  9 +---
 drivers/net/mlx5/mlx5_flow_dv.c  | 90 +++-------------------------------------
 3 files changed, 13 insertions(+), 93 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index a579dde..ae735a3 100644
@@ -305,6 +305,10 @@
 	}
 	sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
 #endif /* HAVE_MLX5DV_DR */
+	sh->default_miss_action =
+			mlx5_glue->dr_create_flow_action_default_miss();
+	if (!sh->default_miss_action)
+		DRV_LOG(WARNING, "Default miss action is not supported.");
 	return 0;
 error:
 	/* Rollback the created objects. */
@@ -388,6 +392,9 @@
 	}
 	pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
+	if (sh->default_miss_action)
+		mlx5_glue->destroy_flow_action
+				(sh->default_miss_action);
 	if (sh->encaps_decaps) {
 		mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
 		sh->encaps_decaps = NULL;
drivers/net/mlx5/mlx5.h +1 -8
index 9ab2976..3ce890a 100644
@@ -461,12 +461,6 @@ struct mlx5_flow_counter_mng {
 	LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;
 };
 
-/* Default miss action resource structure. */
-struct mlx5_flow_default_miss_resource {
-	void *action; /* Pointer to the rdma-core action. */
-	uint32_t refcnt; /* Default miss action reference counter. */
-};
-
 #define MLX5_AGE_EVENT_NEW		1
 #define MLX5_AGE_TRIGGER		2
 #define MLX5_AGE_SET(age_info, BIT) \
@@ -660,8 +654,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
-	struct mlx5_flow_default_miss_resource default_miss;
-	/* Default miss action resource structure. */
+	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -85

136 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#133 lore raw

PATCH v5 11/34 net/mlx5: create global drop action

In reply to Suanming Mou

This commit creates the global drop action for flows instead of maintain it in flow insertion time. The uniqueu global drop action makes it thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  5 +++++
 drivers/net/mlx5/mlx5.c            |  2 ++
 drivers/net/mlx5/mlx5_flow_dv.c    | 38 +++++++----------------------------
 drivers/net/mlx5/mlx5_flow_verbs.c | 41 +++++++++++---------------------------
 4 files changed, 26 insertions(+), 60 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -0
index ae735a3..b12d1d5 100644
@@ -1424,6 +1424,9 @@
 	} else {
 		priv->obj_ops = ibv_obj_ops;
 	}
+	priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
+	if (!priv->drop_queue.hrxq)
+		goto error;
 	/* Supported Verbs flow priority number detection. */
 	err = mlx5_flow_discover_priorities(eth_dev);
 	if (err < 0) {
@@ -1489,6 +1492,8 @@
 			close(priv->nl_socket_rdma);
 		if (priv->vmwa_context)
 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
+		if (eth_dev && priv->drop_queue.hrxq)
+			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		mlx5_free(priv);
drivers/net/mlx5/mlx5.c +2 -0
index fa00e45..2ebc179 100644
@@ -1308,6 +1308,8 @@ struct mlx5_dev_ctx_shared *
 		priv->txqs = NULL;
 	}
 	mlx5_proc_priv_uninit(dev);
+	if (priv->drop_queue.hrxq)
+		mlx5_drop_action_destroy(dev);
 	if (priv->mreg_cp_tbl)
 		mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
 	mlx5_mprq_free_mp(dev);
drivers/net/mlx5/mlx5_flow_dv.c +7 -31
index 34f5608..4322c59 100644
@@ -10319,24 +10319,9 @@ struct field_modify_info modify_tcp[] = {
 			if (dv->transfer) {
 				dv->actions[n++] = priv->sh->esw_drop_action;
 			} else {
-				struct mlx5_hrxq *drop_hrxq;
-				drop_hrxq = mlx5_drop_action_create(dev);
-				if (!drop_hrxq) {
-					rte_flow_error_set
-						(error, errno,
-						 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-						 NULL,
-						 "cannot get drop hash queue");
-					goto error;
-				}
-				/*
-				 * Drop queues will be released by the specify
-				 * mlx5_drop_action_destroy() function. Assign
-				 * the special index to hrxq to mark the queue
-				 * has been allocated.
-				 */
-				dh->rix_hrxq = UINT32_MAX;
-				dv->actions[n++] = drop_hrxq->action;
+				MLX5_ASSERT(priv->drop_queue.hrxq);
+				dv->actions[n++] =
+						priv->drop_queue.hrxq->action;
 			}
 		} else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
 			   !dv_h->rix_sample && !dv_h->rix_dest_array) {
@@ -10390,14 +10375,9 @@ struct field_modify_info modify_tcp[] = {
 	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
 		       handle_idx, dh, next) {
 		/* hrxq is union, don't clear it if the flag is not set. */
-		if (dh->rix_hrxq) {
-			if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
-				mlx5_drop_action_destroy(dev);
-				dh->rix_hrxq = 0;
-			} else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
-				mlx5_hrxq_release(dev, dh->rix_hrxq);
-				dh->rix_hrxq = 0;
-			}
+		if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
+			mlx5_hrxq_release(dev, dh->rix_hrxq);
+			dh->rix_hrxq = 0;
 		}
 		if (dh->vf_vlan.tag && dh->vf_vlan.created)
 			mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
@@ -10643,9 +10623,6 @@ struct field_modify_info modify_tcp[] = {
 	if (!handle->rix_fate)
 		return;
 	switch (handle->fate_action) {
-	case MLX5_FLOW_FATE_DROP:
-		mlx5_drop_action_destroy(dev);
-		break;
 	case MLX5_FLOW_FATE_QUEUE:
 		mlx5_hrxq_release(dev, handle->rix_hrxq);
 		break;
@@ -10825,8 +10802,7 @@ struct field_modify_info modify_tcp[] = {
 			claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
 			dh->drv_flow = NULL;
 		}
-		if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
-		    dh->fate_action == MLX5_FLOW_FATE_QUEUE)
+		if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
 			flow_dv_fate_resource_release(dev, dh);
 		if (dh->vf_vlan.tag && dh->vf_vlan.created)
 			mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
drivers/net/mlx5/mlx5_flow_verbs.c +12 -29

75 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#134 lore raw

PATCH v5 12/34 net/mlx5: support concurrent access for hash list

In reply to Suanming Mou

From: Xueming Li <[email protected]>

In order to support hash list concurrent access, adding next:
1. List level read/write lock.
2. Entry reference counter.
3. Entry create/match/remove callback.
4. Remove insert/lookup/remove function which are not thread safe.
5. Add register/unregister function to support entry reuse.

For better performance, lookup function uses read lock to allow concurrent lookup from different thread, all other hash list modification functions uses write lock which blocks concurrent modification and lookups from other thread.

The exact objects change will be applied in the next patches.

---
 drivers/net/mlx5/linux/mlx5_os.c |  27 ++++---
 drivers/net/mlx5/mlx5.c          |  13 ++--
 drivers/net/mlx5/mlx5_flow.c     |  23 +++---
 drivers/net/mlx5/mlx5_flow_dv.c  |   9 ++-
 drivers/net/mlx5/mlx5_utils.c    | 154 ++++++++++++++++++++++++++++++++-------
 drivers/net/mlx5/mlx5_utils.h    | 149 ++++++++++++++++++++++++++++++-------
 6 files changed, 287 insertions(+), 88 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -11
index b12d1d5..4db5d33 100644
@@ -236,14 +236,16 @@
 		return err;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE);
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
+					  0, NULL, NULL, NULL);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ);
+	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+					    0, 0, NULL, NULL, NULL);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
@@ -251,7 +253,8 @@
 	}
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ);
+					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+					      0, 0, NULL, NULL, NULL);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
@@ -333,16 +336,16 @@
 		sh->pop_vlan_action = NULL;
 	}
 	if (sh->encaps_decaps) {
-		mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
+		mlx5_hlist_destroy(sh->encaps_decaps);
 		sh->encaps_decaps = NULL;
 	}
 	if (sh->modify_cmds) {
-		mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
+		mlx5_hlist_destroy(sh->modify_cmds);
 		sh->modify_cmds = NULL;
 	}
 	if (sh->tag_table) {
 		/* tags should be destroyed with flow before. */
-		mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
+		mlx5_hlist_destroy(sh->tag_table);
 		sh->tag_table = NULL;
 	}
 	if (sh->tunnel_hub) {
@@ -396,16 +399,16 @@
 		mlx5_glue->destroy_flow_action
 				(sh->default_miss_action);
 	if (sh->encaps_decaps) {
-		mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
+		mlx5_hlist_destroy(sh->encaps_decaps);
 		sh->encaps_decaps = NULL;
 	}
 	if (sh->modify_cmds) {
-		mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
+		mlx5_hlist_destroy(sh->modify_cmds);
 		sh->modify_cmds = NULL;
 	}
 	if (sh->tag_table) {
 		/* tags should be destroyed with flow before. */
-		mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
+		mlx5_hlist_destroy(sh->tag_table);
 		sh->tag_table = NULL;
 	}
 	if (sh->tunnel_hub) {
@@ -1472,7 +1475,9 @@
 	    mlx5_flow_ext_mreg_supported(eth_dev) &&
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
-						      MLX5_FLOW_MREG_HTABLE_SZ);
+						      MLX5_FLOW_MREG_HTABLE_SZ,
+						      0, 0,
+						      NULL, NULL, NULL);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
@@ -1483,7 +1488,7 @@
 error:
 	if (priv) {
 		if (priv->mreg_cp_tbl)
-			mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
+			mlx5_hlist_destroy(priv->mreg_cp_tbl);
 		if (priv->sh)
 			mlx5_os_free_shared_dr(priv);
 		if (priv->nl_socket_route >= 0)
drivers/net/mlx5/mlx5.c +7 -6
index 2ebc179..c74b55e 100644
@@ -1034,7 +1034,7 @@ struct mlx5_dev_ctx_shared *
 
 	if (!sh->flow_tbls)
 		return;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	if (pos) {
 		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 					entry);
@@ -1043,7 +1043,7 @@ struct mlx5_dev_ctx_shared *
 		mlx5_free(tbl_data);
 	}
 	table_key.direction = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	if (pos) {
 		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 					entry);
@@ -1053,7 +1053,7 @@ struct mlx5_dev_ctx_shared *
 	}
 	table_key.direction = 0;
 	table_key.domain = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	if (pos) {
 		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 					entry);
@@ -1061,7 +1061,7 @@ struct mlx5_dev_ctx_shared *
 		mlx5_hlist_remove(sh->flow_tbls, pos);
 		mlx5_free(tbl_data);
 	}
-	mlx5_hlist_destroy(sh->flow_tbls, NULL, NULL);
+	mlx5_hlist_destroy(sh->flow_tbls);
 }
 
 /**
@@ -1083,7 +1083,8 @@ struct mlx5_dev_ctx_shared *
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
-	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE);
+	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
+					  0, 0, NULL, NULL, NULL);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
@@ -1311,7 +1312,7 @@ struct mlx5_dev_ctx_shared *
 	if (priv->drop_queue.hrxq)
 		mlx5_drop_action_destroy(dev);
 	if (priv->mreg_cp_tbl)
-		mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
+		mlx5_hlist_destroy(priv->mreg_cp_tbl);
 	mlx5_mprq_free_mp(dev);
 	mlx5_os_free_shared_dr(priv);
 	if (priv->rss_conf.rss_key != NULL)
drivers/net/mlx5/mlx5_flow.c +12 -11
index cc31801..29ff033 100644
@@ -3747,7 +3747,7 @@ struct rte_flow_shared_action *
 	cp_mreg.src = ret;
 	/* Check if already registered. */
 	MLX5_ASSERT(priv->mreg_cp_tbl);
-	mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
+	mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, NULL);
 	if (mcp_res) {
 		/* For non-default rule. */
 		if (mark_id != MLX5_DEFAULT_COPY_ID)
@@ -3824,8 +3824,7 @@ struct rte_flow_shared_action *
 		goto error;
 	mcp_res->refcnt++;
 	mcp_res->hlist_ent.key = mark_id;
-	ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
-				&mcp_res->hlist_ent);
+	ret = !mlx5_hlist_insert(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
 	MLX5_ASSERT(!ret);
 	if (ret)
 		goto error;
@@ -3975,7 +3974,7 @@ struct rte_flow_shared_action *
 	if (!priv->mreg_cp_tbl)
 		return;
 	mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
-					    MLX5_DEFAULT_COPY_ID);
+					    MLX5_DEFAULT_COPY_ID, NULL);
 	if (!mcp_res)
 		return;
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -7562,7 +7561,7 @@ struct mlx5_meter_domains_infos *
 			.direction = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
@@ -7584,7 +7583,7 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_hlist *group_hash;
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_lookup(group_hash, key.val);
+	he = mlx5_hlist_lookup(group_hash, key.val, NULL);
 	if (!he) {
 		tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
 				  sizeof(*tte), 0,
@@ -8091,7 +8090,7 @@ struct mlx5_meter_domains_infos *
 	LIST_REMOVE(tunnel, chain);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID],
 			tunnel->tunnel_id);
-	mlx5_hlist_destroy(tunnel->groups, NULL, NULL);
+	mlx5_hlist_destroy(tunnel->groups);
 	mlx5_free(tunnel);
 }
 
@@ -8139,7 +8138,8 @@ struct mlx5_meter_domains_infos *
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024);
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+					   NULL, NULL, NULL);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(priv->sh->ipool
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
@@ -8204,7 +8204,7 @@ void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
 		return;
 	if (!LIST_EMPTY(&thub->tunnels))
 		DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
-	mlx5_hlist_destroy(thub->groups, NULL, NULL);
+	mlx5_hlist_destroy(thub->groups);
 	mlx5_free(thub);
 }
 
@@ -8218,7 +8218,8 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	if (!thub)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
-	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES);
+	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
+					 0, NULL, NULL, NULL);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
@@ -8229,7 +8230,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 
 err:
 	if (thub->groups)
-		mlx5_hlist_destroy(thub->groups, NULL, NULL);
+		mlx5_hlist_destroy(thub->groups);
 	if (thub)
 		mlx5_free(thub);
 	return err;
drivers/net/mlx5/mlx5_flow_dv.c +5 -4
index 4322c59..13bc46d 100644
@@ -7961,7 +7961,7 @@ struct field_modify_info modify_tcp[] = {
 		}
 	};
 	struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
-							 table_key.v64);
+							 table_key.v64, NULL);
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	uint32_t idx = 0;
 	int ret;
@@ -8022,7 +8022,7 @@ struct field_modify_info modify_tcp[] = {
 		}
 	}
 	pos->key = table_key.v64;
-	ret = mlx5_hlist_insert(sh->flow_tbls, pos);
+	ret = !mlx5_hlist_insert(sh->flow_tbls, pos);
 	if (ret < 0) {
 		rte_flow_error_set(error, -ret,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -8079,7 +8079,8 @@ struct field_modify_info modify_tcp[] = {
 			tunnel_grp_hash = tbl_data->tunnel ?
 						tbl_data->tunnel->groups :
 						thub->groups;
-			he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val);
+			he = mlx5_hlist_lookup(tunnel_grp_hash,
+					       tunnel_key.val, NULL);
 			if (he) {
 				struct tunnel_tbl_entry *tte;
 				tte = container_of(he, typeof(*tte), hash);
@@ -8238,7 +8239,7 @@ struct field_modify_info modify_tcp[] = {
 	int ret;
 
 	/* Lookup a matching resource from cache. */
-	entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
+	entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24, NULL);
 	if (entry) {
 		cache_resource = container_of
 			(entry, struct mlx5_flow_dv_tag_resource, entry);
drivers/net/mlx5/mlx5_utils.c +126 -28

226 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +121 -28

229 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#135 lore raw

PATCH v5 13/34 net/mlx5: add flow table tunnel offload attribute

In reply to Suanming Mou

As flow table is shared between the ports in the same shared IB device, flow table may be created by one port and released by other port.

Currently, the tunnel offloading active check in flow table release is based on the port which release the flow table. Since the flow table create port and release port may have different tunnel offloading configuration, it will cause invalid tunnel offloading release or tunnel offloading resource leaks.

Add the flow table tunnel offloading attribute to indicate the flow table has tunnel offloading resource or not to avoid wrong tunnel offloading operation.

---
 drivers/net/mlx5/mlx5_flow.h    | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
drivers/net/mlx5/mlx5_flow.h +1 -0
index cc64792..62c5e7c 100644
@@ -538,6 +538,7 @@ struct mlx5_flow_tbl_data_entry {
 	const struct mlx5_flow_tunnel *tunnel;
 	uint32_t group_id;
 	bool external;
+	bool tunnel_offload; /* Tunnel offlod table or not. */
 };
 
 /* Sub rdma-core actions list. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index 13bc46d..0b4b7ff 100644
@@ -7986,6 +7986,7 @@ struct field_modify_info modify_tcp[] = {
 	tbl_data->tunnel = tunnel;
 	tbl_data->group_id = group_id;
 	tbl_data->external = external;
+	tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
 	tbl = &tbl_data->tbl;
 	pos = &tbl_data->entry;
 	if (transfer)
@@ -8061,7 +8062,7 @@ struct field_modify_info modify_tcp[] = {
 
 		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
 		tbl->obj = NULL;
-		if (is_tunnel_offload_active(dev) && tbl_data->external) {
+		if (tbl_data->tunnel_offload && tbl_data->external) {
 			struct mlx5_hlist_entry *he;
 			struct mlx5_hlist *tunnel_grp_hash;
 			struct mlx5_flow_tunnel_hub *thub =
Signature · 1 line
1.8.3.1
#136 lore raw

PATCH v5 14/34 net/mlx5: make flow table cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion/removal, this patch uses thread safe hash list API for flow table cache hash list.

---
 drivers/net/mlx5/mlx5.c         |  99 +++------------
 drivers/net/mlx5/mlx5.h         |   2 +-
 drivers/net/mlx5/mlx5_flow.c    |   2 +-
 drivers/net/mlx5/mlx5_flow.h    |  25 ++++
 drivers/net/mlx5/mlx5_flow_dv.c | 265 +++++++++++++++++++++-------------------
 5 files changed, 181 insertions(+), 212 deletions(-)
drivers/net/mlx5/mlx5.c +15 -84
index c74b55e..1d25a8e 100644
@@ -1012,7 +1012,7 @@ struct mlx5_dev_ctx_shared *
 }
 
 /**
- * Destroy table hash list and all the root entries per domain.
+ * Destroy table hash list.
  *
  * @param[in] priv
  *   Pointer to the private device data structure.
@@ -1021,46 +1021,9 @@ struct mlx5_dev_ctx_shared *
 mlx5_free_table_hash_list(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_flow_tbl_data_entry *tbl_data;
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = 0,
-			.reserved = 0,
-			.domain = 0,
-			.direction = 0,
-		}
-	};
-	struct mlx5_hlist_entry *pos;
 
 	if (!sh->flow_tbls)
 		return;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
-	table_key.direction = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
-	table_key.direction = 0;
-	table_key.domain = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
 	mlx5_hlist_destroy(sh->flow_tbls);
 }
 
@@ -1075,77 +1038,45 @@ struct mlx5_dev_ctx_shared *
  *   Zero on success, positive error code otherwise.
  */
 int
-mlx5_alloc_table_hash_list(struct mlx5_priv *priv)
+mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 {
+	int err = 0;
+	/* Tables are only used in DV and DR modes. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	char s[MLX5_HLIST_NAMESIZE];
-	int err = 0;
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, NULL, NULL, NULL);
+					  0, 0, flow_dv_tbl_create_cb, NULL,
+					  flow_dv_tbl_remove_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
+	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
+	struct rte_flow_error error;
+	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
+
 	/*
 	 * In case we have not DR support, the zero tables should be created
 	 * because DV expect to see them even if they cannot be created by
 	 * RDMA-CORE.
 	 */
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = 0,
-			.reserved = 0,
-			.domain = 0,
-			.direction = 0,
-		}
-	};
-	struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO,
-							  sizeof(*tbl_data), 0,
-							  SOCKET_ID_ANY);
-
-	if (!tbl_data) {
-		err = ENOMEM;
-		goto error;
-	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	__atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED);
-	table_key.direction = 1;
-	tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
-			       SOCKET_ID_ANY);
-	if (!tbl_data) {
+	if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL, 0, 1, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, NULL, 0, 1, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, NULL, 0, 1, &error)) {
 		err = ENOMEM;
 		goto error;
 	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	__atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED);
-	table_key.direction = 0;
-	table_key.domain = 1;
-	tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
-			       SOCKET_ID_ANY);
-	if (!tbl_data) {
-		err = ENOMEM;
-		goto error;
-	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	__atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED);
 	return err;
 error:
 	mlx5_free_table_hash_list(priv);
 #endif /* HAVE_MLX5DV_DR */
+#endif
 	return err;
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index 3ce890a..81dba2f 100644
@@ -499,7 +499,7 @@ struct mlx5_dev_shared_port {
 	struct {
 		/* Table ID should be at the lowest address. */
 		uint32_t table_id;	/**< ID of the table. */
-		uint16_t reserved;	/**< must be zero for comparison. */
+		uint16_t dummy;		/**< Dummy table for DV API. */
 		uint8_t domain;		/**< 1 - FDB, 0 - NIC TX/RX. */
 		uint8_t direction;	/**< 1 - egress, 0 - ingress. */
 	};
drivers/net/mlx5/mlx5_flow.c +1 -1
index 29ff033..b5b4997 100644
@@ -7556,7 +7556,7 @@ struct mlx5_meter_domains_infos *
 	union mlx5_flow_tbl_key table_key = {
 		{
 			.table_id = tunnel_id_to_flow_tbl(mbits.table_id),
-			.reserved = 0,
+			.dummy = 0,
 			.domain = !!mbits.transfer,
 			.direction = 0,
 		}
drivers/net/mlx5/mlx5_flow.h +25 -0
index 62c5e7c..dae8211 100644
@@ -384,6 +384,13 @@ enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_MAX,
 };
 
+/* Hash list callback context */
+struct mlx5_flow_cb_ctx {
+	struct rte_eth_dev *dev;
+	struct rte_flow_error *error;
+	void *data;
+};
+
 /* Matcher PRM representation */
 struct mlx5_flow_dv_match_params {
 	size_t size;
@@ -523,6 +530,13 @@ struct mlx5_flow_mreg_copy_resource {
 	uint32_t rix_flow; /* Built flow for copy. */
 };
 
+/* Table tunnel parameter. */
+struct mlx5_flow_tbl_tunnel_prm {
+	const struct mlx5_flow_tunnel *tunnel;
+	uint32_t group_id;
+	bool external;
+};
+
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
 	struct mlx5_hlist_entry entry;
@@ -1401,4 +1415,15 @@ int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
 int mlx5_shared_action_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
+
+/* Hash list callbacks for flow tables: */
+struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
+					       uint64_t key, void *entry_ctx);
+void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
+			   struct mlx5_hlist_entry *entry);
+struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
+		uint32_t table_id, uint8_t egress, uint8_t transfer,
+		bool external, const struct mlx5_flow_tunnel *tunnel,
+		uint32_t group_id, uint8_t dummy, struct rte_flow_error *error);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +139 -126

352 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#137 lore raw

PATCH v5 15/34 net/mlx5: fix redundant Direct Verbs resources allocate

In reply to Suanming Mou

All table, tag, header modify, header reformat are supported only on DV mode. For the OFED version doesn't support these, create the related redundant DV resources waste the memory.

Add the code section in the HAVE_IBV_FLOW_DV_SUPPORT macro to avoid the redundant resources allocation.

Fixes 2eb4d0107acc ("net/mlx5: refactor PCI probing on Linux")
---
 drivers/net/mlx5/linux/mlx5_os.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -2
index 4db5d33..db32b39 100644
@@ -225,7 +225,7 @@
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -233,7 +233,9 @@
 		return 0;
 	err = mlx5_alloc_table_hash_list(priv);
 	if (err)
-		return err;
+		goto error;
+	/* The resources below are only valid with DV support. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -260,6 +262,7 @@
 		err = ENOMEM;
 		goto error;
 	}
+#endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
 
Signature · 1 line
1.8.3.1
#138 lore raw

PATCH v5 16/34 net/mlx5: make flow tag list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch updates flow tag list to use thread safe hash list with write-most mode.

---
 drivers/net/mlx5/linux/mlx5_os.c |  5 +-
 drivers/net/mlx5/mlx5_flow.h     |  5 ++
 drivers/net/mlx5/mlx5_flow_dv.c  | 98 +++++++++++++++++++---------------------
 3 files changed, 56 insertions(+), 52 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +4 -1
index db32b39..540e1a7 100644
@@ -239,12 +239,15 @@
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  0, NULL, NULL, NULL);
+					  MLX5_HLIST_WRITE_MOST,
+					  flow_dv_tag_create_cb, NULL,
+					  flow_dv_tag_remove_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
+	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
 					    0, 0, NULL, NULL, NULL);
drivers/net/mlx5/mlx5_flow.h +5 -0
index dae8211..227504d 100644
@@ -1426,4 +1426,9 @@ struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy, struct rte_flow_error *error);
 
+struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
+					       uint64_t key, void *cb_ctx);
+void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
+			   struct mlx5_hlist_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +47 -51

132 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#139 lore raw

PATCH v5 17/34 net/mlx5: make flow modify action list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch updates flow modify action list to use thread safe hash list with write-most mode.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 +-
 drivers/net/mlx5/mlx5_flow.h     |  14 ++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 196 +++++++++++++++++----------------------
 3 files changed, 102 insertions(+), 115 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +6 -1
index 540e1a7..eb63bcd 100644
@@ -250,12 +250,17 @@
 	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, 0, NULL, NULL, NULL);
+					    0, MLX5_HLIST_WRITE_MOST |
+					    MLX5_HLIST_DIRECT_KEY,
+					    flow_dv_modify_create_cb,
+					    flow_dv_modify_match_cb,
+					    flow_dv_modify_remove_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
+	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
drivers/net/mlx5/mlx5_flow.h +10 -4
index 227504d..4fb50de 100644
@@ -467,10 +467,8 @@ struct mlx5_flow_dv_tag_resource {
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_hlist_entry entry;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *action;
-	/**< Modify header action object. */
+	void *action; /**< Modify header action object. */
+	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint32_t actions_num; /**< Number of modification actions. */
 	uint64_t flags; /**< Flags for RDMA API. */
@@ -1431,4 +1429,12 @@ struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
 void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
 			   struct mlx5_hlist_entry *entry);
 
+int flow_dv_modify_match_cb(struct mlx5_hlist *list,
+			    struct mlx5_hlist_entry *entry,
+			    uint64_t key, void *cb_ctx);
+struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
+						  uint64_t key, void *ctx);
+void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
+			      struct mlx5_hlist_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +86 -110

238 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#140 lore raw

PATCH v5 18/34 net/mlx5: remove unused mreg copy code

In reply to Suanming Mou

After non-cache mode feature was implemented, the flows can only be created when port started. No need to check if the mreg flows are created in port stopped status, and apply the mreg flows after port start will also never happen.

This commit removed the relevant not used mreg copy code.

---
 drivers/net/mlx5/mlx5.h      |   2 -
 drivers/net/mlx5/mlx5_flow.c | 185 -------------------------------------------
 drivers/net/mlx5/mlx5_flow.h |   2 -
 3 files changed, 189 deletions(-)
drivers/net/mlx5/mlx5.h +0 -2
index 81dba2f..debb862 100644
@@ -1091,8 +1091,6 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 			 enum rte_filter_type filter_type,
 			 enum rte_filter_op filter_op,
 			 void *arg);
-int mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list);
-void mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list);
 int mlx5_flow_start_default(struct rte_eth_dev *dev);
 void mlx5_flow_stop_default(struct rte_eth_dev *dev);
 int mlx5_flow_verify(struct rte_eth_dev *dev);
drivers/net/mlx5/mlx5_flow.c +0 -185

221 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +0 -2
index 4fb50de..b5e4c9f 100644
@@ -523,7 +523,6 @@ struct mlx5_flow_mreg_copy_resource {
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t refcnt; /* Reference counter. */
-	uint32_t appcnt; /* Apply/Remove counter. */
 	uint32_t idx;
 	uint32_t rix_flow; /* Built flow for copy. */
 };
@@ -1048,7 +1047,6 @@ struct rte_flow {
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t fdir:1; /**< Identifier of associated FDIR if any. */
 	uint32_t tunnel:1;
-	uint32_t copy_applied:1; /**< The MARK copy Flow os applied. */
 	uint32_t meter:16; /**< Holds flow meter id. */
 	uint32_t rix_mreg_copy;
 	/**< Index to metadata register copy table resource. */
Signature · 1 line
1.8.3.1
#141 lore raw

PATCH v5 19/34 net/mlx5: make metadata copy flow list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch updates metadata copy flow list to use thread safe hash list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   5 +-
 drivers/net/mlx5/mlx5_flow.c     | 162 ++++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_flow.h     |   6 +-
 3 files changed, 99 insertions(+), 74 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +4 -1
index eb63bcd..46dbc18 100644
@@ -1488,11 +1488,14 @@
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
 						      0, 0,
-						      NULL, NULL, NULL);
+						      flow_dv_mreg_create_cb,
+						      NULL,
+						      flow_dv_mreg_remove_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
+		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	mlx5_flow_counter_mode_config(eth_dev);
 	return eth_dev;
drivers/net/mlx5/mlx5_flow.c +90 -72

225 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +5 -1
index b5e4c9f..ffbef2a 100644
@@ -522,7 +522,6 @@ struct mlx5_flow_mreg_copy_resource {
 	struct mlx5_hlist_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
-	uint32_t refcnt; /* Reference counter. */
 	uint32_t idx;
 	uint32_t rix_flow; /* Built flow for copy. */
 };
@@ -1435,4 +1434,9 @@ struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
 void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
 			      struct mlx5_hlist_entry *entry);
 
+struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
+						uint64_t key, void *ctx);
+void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
+			    struct mlx5_hlist_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
Signature · 1 line
1.8.3.1
#142 lore raw

PATCH v5 20/34 net/mlx5: make header reformat action thread safe

In reply to Suanming Mou

To support multi-thread flow insertion, this patch updates flow header reformat action list to use thread safe hash list with write-most mode.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 +-
 drivers/net/mlx5/mlx5_flow.h     |   7 ++
 drivers/net/mlx5/mlx5_flow_dv.c  | 186 ++++++++++++++++++++++-----------------
 3 files changed, 116 insertions(+), 84 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +6 -1
index 46dbc18..d017c23 100644
@@ -264,12 +264,17 @@
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, 0, NULL, NULL, NULL);
+					      0, MLX5_HLIST_DIRECT_KEY |
+					      MLX5_HLIST_WRITE_MOST,
+					      flow_dv_encap_decap_create_cb,
+					      flow_dv_encap_decap_match_cb,
+					      flow_dv_encap_decap_remove_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
+	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
drivers/net/mlx5/mlx5_flow.h +7 -0
index ffbef2a..d0a1000 100644
@@ -1439,4 +1439,11 @@ struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
 void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
 			    struct mlx5_hlist_entry *entry);
 
+int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
+				 struct mlx5_hlist_entry *entry,
+				 uint64_t key, void *cb_ctx);
+struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
+				uint64_t key, void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +103 -83

242 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#143 lore raw

PATCH v5 21/34 net/mlx5: introduce thread safe linked list cache

In reply to Suanming Mou

From: Xueming Li <[email protected]>

New API of linked list for cache:
- Optimized for small amount cache list.
- Optimized for read-most list.
- Thread safe.
- Since number of entries are limited, entries allocated by API.
- For dynamic entry size, pass 0 as entry size, then the creation callback allocate the entry.
- Since number of entries are limited, no need to use indexed pool to allocate memory. API will remove entry and free with mlx5_free.
- Search API is not supposed to be used in multi-thread.

---
 drivers/net/mlx5/mlx5_utils.c | 160 ++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h | 183 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 343 insertions(+)
drivers/net/mlx5/mlx5_utils.c +160 -0
index d041b07..df15f63 100644
@@ -255,6 +255,166 @@ struct mlx5_hlist_entry *
 	mlx5_free(h);
 }
 
+/********************* Cache list ************************/
+
+static struct mlx5_cache_entry *
+mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry __rte_unused,
+			     void *ctx __rte_unused)
+{
+	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
+}
+
+static void
+mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
+			     struct mlx5_cache_entry *entry)
+{
+	mlx5_free(entry);
+}
+
+int
+mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+		     uint32_t entry_size, void *ctx,
+		     mlx5_cache_create_cb cb_create,
+		     mlx5_cache_match_cb cb_match,
+		     mlx5_cache_remove_cb cb_remove)
+{
+	MLX5_ASSERT(list);
+	if (!cb_match || (!cb_create ^ !cb_remove))
+		return -1;
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->entry_sz = entry_size;
+	list->ctx = ctx;
+	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	LIST_INIT(&list->head);
+	return 0;
+}
+
+static struct mlx5_cache_entry *
+__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+{
+	struct mlx5_cache_entry *entry;
+
+	LIST_FOREACH(entry, &list->head, next) {
+		if (list->cb_match(list, entry, ctx))
+			continue;
+		if (reuse) {
+			__atomic_add_fetch(&entry->ref_cnt, 1,
+					   __ATOMIC_RELAXED);
+			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+		break;
+	}
+	return entry;
+}
+
+static struct mlx5_cache_entry *
+cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+{
+	struct mlx5_cache_entry *entry;
+
+	rte_rwlock_read_lock(&list->lock);
+	entry = __cache_lookup(list, ctx, reuse);
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+struct mlx5_cache_entry *
+mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+{
+	return cache_lookup(list, ctx, false);
+}
+
+struct mlx5_cache_entry *
+mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+{
+	struct mlx5_cache_entry *entry;
+	uint32_t prev_gen_cnt = 0;
+
+	MLX5_ASSERT(list);
+	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	/* Lookup with read lock, reuse if found. */
+	entry = cache_lookup(list, ctx, true);
+	if (entry)
+		return entry;
+	/* Not found, append with write lock - block read from other threads. */
+	rte_rwlock_write_lock(&list->lock);
+	/* If list changed by other threads before lock, search again. */
+	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
+		/* Lookup and reuse w/o read lock. */
+		entry = __cache_lookup(list, ctx, true);
+		if (entry)
+			goto done;
+	}
+	entry = list->cb_create(list, entry, ctx);
+	if (!entry) {
+		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+			list->name, (void *)entry);
+		goto done;
+	}
+	entry->ref_cnt = 1;
+	LIST_INSERT_HEAD(&list->head, entry, next);
+	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+done:
+	rte_rwlock_write_unlock(&list->lock);
+	return entry;
+}
+
+int
+mlx5_cache_unregister(struct mlx5_cache_list *list,
+		      struct mlx5_cache_entry *entry)
+{
+	rte_rwlock_write_lock(&list->lock);
+	MLX5_ASSERT(entry && entry->next.le_prev);
+	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	if (--entry->ref_cnt) {
+		rte_rwlock_write_unlock(&list->lock);
+		return 1;
+	}
+	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	LIST_REMOVE(entry, next);
+	list->cb_remove(list, entry);
+	rte_rwlock_write_unlock(&list->lock);
+	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+		list->name, (void *)entry);
+	return 0;
+}
+
+void
+mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+{
+	struct mlx5_cache_entry *entry;
+
+	MLX5_ASSERT(list);
+	/* no LIST_FOREACH_SAFE, using while instead */
+	while (!LIST_EMPTY(&list->head)) {
+		entry = LIST_FIRST(&list->head);
+		LIST_REMOVE(entry, next);
+		list->cb_remove(list, entry);
+		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+			list->name, (void *)entry);
+	}
+	memset(list, 0, sizeof(*list));
+}
+
+uint32_t
+mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+{
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+}
+
 /********************* Indexed pool **********************/
 
 static inline void
drivers/net/mlx5/mlx5_utils.h +183 -0

189 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#144 lore raw

PATCH v5 22/34 net/mlx5: optimize shared RSS list operation

In reply to Suanming Mou

When create shared RSS hrxq, the hrxq will be created directly, no hrxq will be reused.

In this case, add the shared RSS hrxq to the queue list is redundant. And it also hurts the generic queue lookup.

This commit avoids add the shared RSS hrxq to the queue list.

---
 drivers/net/mlx5/mlx5.h      |  2 +-
 drivers/net/mlx5/mlx5_rxq.c  | 57 +++++++++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_rxtx.h |  5 ++--
 3 files changed, 39 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.h +1 -1
index debb862..9f96bd0 100644
@@ -725,7 +725,7 @@ struct mlx5_ind_table_obj {
 struct mlx5_hrxq {
 	ILIST_ENTRY(uint32_t)next; /* Index to the next element. */
 	uint32_t refcnt; /* Reference counter. */
-	uint32_t shared:1; /* This object used in shared action. */
+	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
 	union {
drivers/net/mlx5/mlx5_rxq.c +35 -22

165 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxtx.h +3 -2
index 1b5fba4..8fe0473 100644
@@ -347,12 +347,13 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev,
 						  const uint16_t *queues,
 						  uint32_t queues_n);
 int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
-			       struct mlx5_ind_table_obj *ind_tbl);
+			       struct mlx5_ind_table_obj *ind_tbl,
+			       bool standalone);
 uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev,
 		       const uint8_t *rss_key, uint32_t rss_key_len,
 		       uint64_t hash_fields,
 		       const uint16_t *queues, uint32_t queues_n,
-		       int tunnel, bool shared);
+		       int tunnel, bool standalone);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       const uint8_t *rss_key, uint32_t rss_key_len,
 		       uint64_t hash_fields,
Signature · 1 line
1.8.3.1
#145 lore raw

PATCH v5 23/34 net/mlx5: make Rx queue thread safe

In reply to Suanming Mou

This commit applies the cache linked list to Rx queue to make it thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c   |   5 +
 drivers/net/mlx5/mlx5.c            |   1 +
 drivers/net/mlx5/mlx5.h            |  28 +++-
 drivers/net/mlx5/mlx5_flow.h       |  16 ---
 drivers/net/mlx5/mlx5_flow_dv.c    |  74 ++++------
 drivers/net/mlx5/mlx5_flow_verbs.c |  21 +--
 drivers/net/mlx5/mlx5_rxq.c        | 281 ++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_rxtx.h       |  20 +--
 8 files changed, 228 insertions(+), 218 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -0
index d017c23..10fc7c5 100644
@@ -1468,6 +1468,10 @@
 			err = ENOTSUP;
 			goto error;
 	}
+	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+			     mlx5_hrxq_create_cb,
+			     mlx5_hrxq_match_cb,
+			     mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
@@ -1520,6 +1524,7 @@
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+		mlx5_cache_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -0
index 1d25a8e..862bd40 100644
@@ -1286,6 +1286,7 @@ struct mlx5_dev_ctx_shared *
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
+	mlx5_cache_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +26 -2
index 9f96bd0..7804d3a 100644
@@ -65,6 +65,13 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* Hash and cache list callback context. */
+struct mlx5_flow_cb_ctx {
+	struct rte_eth_dev *dev;
+	struct rte_flow_error *error;
+	void *data;
+};
+
 /* Device attributes used in mlx5 PMD */
 struct mlx5_dev_attr {
 	uint64_t	device_cap_flags_ex;
@@ -684,6 +691,22 @@ struct mlx5_proc_priv {
 /* MTR list. */
 TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter);
 
+/* RSS description. */
+struct mlx5_flow_rss_desc {
+	uint32_t level;
+	uint32_t queue_num; /**< Number of entries in @p queue. */
+	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
+	uint64_t hash_fields; /* Verbs Hash fields. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint32_t key_len; /**< RSS hash key len. */
+	uint32_t tunnel; /**< Queue in tunnel. */
+	union {
+		uint16_t *queue; /**< Destination queues. */
+		const uint16_t *const_q; /**< Const pointer convert. */
+	};
+	bool standalone; /**< Queue is standalone or not. */
+};
+
 #define MLX5_PROC_PRIV(port_id) \
 	((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)
 
@@ -723,7 +746,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	ILIST_ENTRY(uint32_t)next; /* Index to the next element. */
+	struct mlx5_cache_entry entry; /* Cache entry. */
 	uint32_t refcnt; /* Reference counter. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
@@ -737,6 +760,7 @@ struct mlx5_hrxq {
 #endif
 	uint64_t hash_fields; /* Verbs Hash fields. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
+	uint32_t idx; /* Hash Rx queue index. */
 	uint8_t rss_key[]; /* Hash key. */
 };
 
@@ -853,7 +877,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	uint32_t hrxqs; /* Verbs Hash Rx queues. */
+	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +0 -16
index d0a1000..84988c6 100644
@@ -384,13 +384,6 @@ enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_MAX,
 };
 
-/* Hash list callback context */
-struct mlx5_flow_cb_ctx {
-	struct rte_eth_dev *dev;
-	struct rte_flow_error *error;
-	void *data;
-};
-
 /* Matcher PRM representation */
 struct mlx5_flow_dv_match_params {
 	size_t size;
@@ -609,15 +602,6 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
-/* RSS description. */
-struct mlx5_flow_rss_desc {
-	uint32_t level;
-	uint32_t queue_num; /**< Number of entries in @p queue. */
-	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
-	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
-	uint16_t *queue; /**< Destination queues. */
-};
-
 /* PMD flow priority for tunnel */
 #define MLX5_TUNNEL_PRIO_GET(rss_desc) \
 	((rss_desc)->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4)
drivers/net/mlx5/mlx5_flow_dv.c +27 -47

131 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_verbs.c +6 -15
index 3c04727..28fd120 100644
@@ -1968,21 +1968,12 @@
 				&wks->rss_desc[!!wks->flow_nested_idx];
 
 			MLX5_ASSERT(rss_desc->queue_num);
-			hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
-						 MLX5_RSS_HASH_KEY_LEN,
-						 dev_flow->hash_fields,
-						 rss_desc->queue,
-						 rss_desc->queue_num);
-			if (!hrxq_idx)
-				hrxq_idx = mlx5_hrxq_new
-						(dev, rss_desc->key,
-						 MLX5_RSS_HASH_KEY_LEN,
-						 dev_flow->hash_fields,
-						 rss_desc->queue,
-						 rss_desc->queue_num,
-						 !!(handle->layers &
-						 MLX5_FLOW_LAYER_TUNNEL),
-						 false);
+			rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
+			rss_desc->hash_fields = dev_flow->hash_fields;
+			rss_desc->tunnel = !!(handle->layers &
+					      MLX5_FLOW_LAYER_TUNNEL);
+			rss_desc->standalone = false;
+			hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
 			hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
 					      hrxq_idx);
 			if (!hrxq) {
drivers/net/mlx5/mlx5_rxq.c +152 -129

342 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxtx.h +11 -9
index 8fe0473..c361bbd 100644
@@ -349,17 +349,19 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_obj *ind_tbl,
 			       bool standalone);
-uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev,
-		       const uint8_t *rss_key, uint32_t rss_key_len,
-		       uint64_t hash_fields,
-		       const uint16_t *queues, uint32_t queues_n,
-		       int tunnel, bool standalone);
+struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
+		       struct mlx5_cache_entry *entry,
+		       void *cb_ctx);
+void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
+			 struct mlx5_cache_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
-		       const uint8_t *rss_key, uint32_t rss_key_len,
-		       uint64_t hash_fields,
-		       const uint16_t *queues, uint32_t queues_n);
+		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
-int mlx5_hrxq_verify(struct rte_eth_dev *dev);
+uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev);
+
+
 enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx);
 const struct rte_eth_hairpin_conf *mlx5_rxq_get_hairpin_conf
 	(struct rte_eth_dev *dev, uint16_t idx);
Signature · 1 line
1.8.3.1
#146 lore raw

PATCH v5 24/34 net/mlx5: make matcher list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this path converts matcher list to use thread safe cache list API.

---
 drivers/net/mlx5/mlx5.h         |   3 +
 drivers/net/mlx5/mlx5_flow.h    |  15 ++-
 drivers/net/mlx5/mlx5_flow_dv.c | 215 +++++++++++++++++++++-------------------
 3 files changed, 129 insertions(+), 104 deletions(-)
drivers/net/mlx5/mlx5.h +3 -0
index 7804d3a..eadb797 100644
@@ -32,6 +32,9 @@
 #include "mlx5_os.h"
 #include "mlx5_autoconf.h"
 
+
+#define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
+
 enum mlx5_ipool_index {
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
drivers/net/mlx5/mlx5_flow.h +11 -4
index 84988c6..6c0815a 100644
@@ -394,11 +394,9 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	LIST_ENTRY(mlx5_flow_dv_matcher) next;
-	/**< Pointer to the next element. */
+	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
-	uint32_t refcnt; /**< Reference counter. */
 	void *matcher_object; /**< Pointer to DV matcher */
 	uint16_t crc; /**< CRC of key. */
 	uint16_t priority; /**< Priority of matcher. */
@@ -532,7 +530,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
+	struct mlx5_cache_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -542,6 +540,7 @@ struct mlx5_flow_tbl_data_entry {
 	uint32_t group_id;
 	bool external;
 	bool tunnel_offload; /* Tunnel offlod table or not. */
+	bool is_egress; /**< Egress table. */
 };
 
 /* Sub rdma-core actions list. */
@@ -1430,4 +1429,12 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 				uint64_t key, void *cb_ctx);
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
+
+int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry, void *ctx);
+struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry, void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +115 -100

346 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#147 lore raw

PATCH v5 25/34 net/mlx5: make port ID action cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch convert port id action cache list to thread safe cache list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 ++
 drivers/net/mlx5/mlx5.h          |   2 +-
 drivers/net/mlx5/mlx5_flow.h     |  15 ++--
 drivers/net/mlx5/mlx5_flow_dv.c  | 143 +++++++++++++++++++++------------------
 4 files changed, 94 insertions(+), 73 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index 10fc7c5..3d2bf57 100644
@@ -236,6 +236,12 @@
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	/* Init port id action cache list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+			     flow_dv_port_id_create_cb,
+			     flow_dv_port_id_match_cb,
+			     flow_dv_port_id_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -431,6 +437,7 @@
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
+	mlx5_cache_list_destroy(&sh->port_id_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index eadb797..1cd42eb 100644
@@ -659,7 +659,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	uint32_t port_id_action_list; /* List of port ID actions. */
+	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	uint32_t push_vlan_action_list; /* List of push VLAN actions. */
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
drivers/net/mlx5/mlx5_flow.h +10 -5
index 6c0815a..0e4d47f 100644
@@ -485,12 +485,10 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	ILIST_ENTRY(uint32_t)next;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *action;
-	/**< Action object. */
+	struct mlx5_cache_entry entry;
+	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
+	uint32_t idx; /**< Indexed pool memory index. */
 };
 
 /* Push VLAN action resource structure */
@@ -1437,4 +1435,11 @@ struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
 			       struct mlx5_cache_entry *entry);
 
+int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +76 -67

170 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#148 lore raw

PATCH v5 26/34 net/mlx5: make push VLAN action cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch converts push VLAN action cache list to thread safe cache list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 ++
 drivers/net/mlx5/mlx5.h          |   2 +-
 drivers/net/mlx5/mlx5_flow.h     |  13 +++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 159 +++++++++++++++++++++------------------
 4 files changed, 102 insertions(+), 79 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index 3d2bf57..9ab3ba9 100644
@@ -242,6 +242,12 @@
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
+	/* Init push vlan action cache list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+			     flow_dv_push_vlan_create_cb,
+			     flow_dv_push_vlan_match_cb,
+			     flow_dv_push_vlan_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -438,6 +444,7 @@
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_cache_list_destroy(&sh->port_id_action_list);
+	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index 1cd42eb..9efa354 100644
@@ -660,7 +660,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	uint32_t push_vlan_action_list; /* List of push VLAN actions. */
+	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
drivers/net/mlx5/mlx5_flow.h +10 -3
index 0e4d47f..500a580 100644
@@ -493,12 +493,11 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	ILIST_ENTRY(uint32_t)next;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
+	uint32_t idx; /**< Indexed pool memory index. */
 };
 
 /* Metadata register copy table entry. */
@@ -1442,4 +1441,12 @@ struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
 			       struct mlx5_cache_entry *entry);
 
+int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +84 -75

189 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#149 lore raw

PATCH v5 27/34 net/mlx5: simplify sample attributes

In reply to Suanming Mou

Currently, the sample action resource already has ft_type to indicate the action domain attribute, the extra flow attributes parameter can be optimized.

This commit uses action resource ty_type as domain attribute instead of the flow attribute.

---
 drivers/net/mlx5/mlx5_flow_dv.c | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_flow_dv.c +14 -15
index d6414eb..fbe114c 100644
@@ -8605,8 +8605,6 @@ struct mlx5_hlist_entry *
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] attr
- *   Attributes of flow that includes this item.
  * @param[in] resource
  *   Pointer to sample resource.
  * @parm[in, out] dev_flow
@@ -8621,7 +8619,6 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 const struct rte_flow_attr *attr,
 			 struct mlx5_flow_dv_sample_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 void **sample_dv_actions,
@@ -8635,6 +8632,8 @@ struct mlx5_hlist_entry *
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
 	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint8_t is_egress = 0;
+	uint8_t is_transfer = 0;
 
 	/* Lookup a matching resource from cache. */
 	ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
@@ -8667,8 +8666,12 @@ struct mlx5_hlist_entry *
 					  "cannot allocate resource memory");
 	*cache_resource = *resource;
 	/* Create normal path table level */
+	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+		is_transfer = 1;
+	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
-					attr->egress, attr->transfer,
+					is_egress, is_transfer,
 					dev_flow->external, NULL, 0, 0, error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENOMEM,
@@ -8751,8 +8754,6 @@ struct mlx5_hlist_entry *
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] attr
- *   Attributes of flow that includes this item.
  * @param[in] resource
  *   Pointer to destination array resource.
  * @parm[in, out] dev_flow
@@ -8765,7 +8766,6 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 const struct rte_flow_attr *attr,
 			 struct mlx5_flow_dv_dest_array_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
@@ -8809,9 +8809,9 @@ struct mlx5_hlist_entry *
 					  NULL,
 					  "cannot allocate resource memory");
 	*cache_resource = *resource;
-	if (attr->transfer)
+	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (attr->ingress)
+	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
@@ -9100,6 +9100,8 @@ struct mlx5_hlist_entry *
 		res->set_action = action_ctx.set_action;
 	} else if (attr->ingress) {
 		res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
+	} else {
+		res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
 	}
 	return 0;
 }
@@ -9111,8 +9113,6 @@ struct mlx5_hlist_entry *
  *   Pointer to rte_eth_dev structure.
  * @param[in, out] dev_flow
  *   Pointer to the mlx5_flow.
- * @param[in] attr
- *   Pointer to the flow attributes.
  * @param[in] num_of_dest
  *   The num of destination.
  * @param[in, out] res
@@ -9132,7 +9132,6 @@ struct mlx5_hlist_entry *
 static int
 flow_dv_create_action_sample(struct rte_eth_dev *dev,
 			     struct mlx5_flow *dev_flow,
-			     const struct rte_flow_attr *attr,
 			     uint32_t num_of_dest,
 			     struct mlx5_flow_dv_sample_resource *res,
 			     struct mlx5_flow_dv_dest_array_resource *mdest_res,
@@ -9192,14 +9191,14 @@ struct mlx5_hlist_entry *
 		memcpy(&mdest_res->sample_act[0], &res->sample_act,
 				sizeof(struct mlx5_flow_sub_actions_list));
 		mdest_res->num_of_dest = num_of_dest;
-		if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
+		if (flow_dv_dest_array_resource_register(dev, mdest_res,
 							 dev_flow, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL, "can't create sample "
 						  "action");
 	} else {
-		if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
+		if (flow_dv_sample_resource_register(dev, res, dev_flow,
 						     sample_actions, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -9839,7 +9838,7 @@ struct mlx5_hlist_entry *
 			}
 			if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
 				ret = flow_dv_create_action_sample(dev,
-							  dev_flow, attr,
+							  dev_flow,
 							  num_of_dest,
 							  &sample_res,
 							  &mdest_res,
Signature · 1 line
1.8.3.1
#150 lore raw

PATCH v5 28/34 net/mlx5: fix sample register error flow

In reply to Suanming Mou

Currently, sample flow need to prepare and register the sub-actions before sample action is created.

Once the same sample action exists, the sub-actions registered by the second flow should be released, or these sub-actions will be leaked. Since the exist sample action only release these same sub-actions when the sample action itself releases.

When same sample action exists, call the sub-action release function for the later flow to release the redundant prepared sub-actions.

Fixes 0756228b2704 ("net/mlx5: update translate function for sample action")
---
 drivers/net/mlx5/mlx5_flow_dv.c | 117 ++++++++++++++++++++--------------------
 1 file changed, 60 insertions(+), 57 deletions(-)
drivers/net/mlx5/mlx5_flow_dv.c +60 -57
index fbe114c..3b36704 100644
@@ -8601,6 +8601,42 @@ struct mlx5_hlist_entry *
 }
 
 /**
+ * Release sample sub action resource.
+ *
+ * @param[in, out] dev
+ *   Pointer to rte_eth_dev structure.
+ * @param[in] act_res
+ *   Pointer to sample sub action resource.
+ */
+static void
+flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
+				   struct mlx5_flow_sub_actions_idx *act_res)
+{
+	if (act_res->rix_hrxq) {
+		mlx5_hrxq_release(dev, act_res->rix_hrxq);
+		act_res->rix_hrxq = 0;
+	}
+	if (act_res->rix_encap_decap) {
+		flow_dv_encap_decap_resource_release(dev,
+						     act_res->rix_encap_decap);
+		act_res->rix_encap_decap = 0;
+	}
+	if (act_res->rix_port_id_action) {
+		flow_dv_port_id_action_resource_release(dev,
+						act_res->rix_port_id_action);
+		act_res->rix_port_id_action = 0;
+	}
+	if (act_res->rix_tag) {
+		flow_dv_tag_release(dev, act_res->rix_tag);
+		act_res->rix_tag = 0;
+	}
+	if (act_res->cnt) {
+		flow_dv_counter_release(dev, act_res->cnt);
+		act_res->cnt = 0;
+	}
+}
+
+/**
  * Find existing sample resource or create and register a new one.
  *
  * @param[in, out] dev
@@ -8653,6 +8689,12 @@ struct mlx5_hlist_entry *
 					   __ATOMIC_RELAXED);
 			dev_flow->handle->dvh.rix_sample = idx;
 			dev_flow->dv.sample_res = cache_resource;
+			/*
+			 * Existing smaple action should release the prepared
+			 * sub-actions reference counter.
+			 */
+			flow_dv_sample_sub_actions_release(dev,
+							&resource->sample_idx);
 			return 0;
 		}
 	}
@@ -8721,25 +8763,13 @@ struct mlx5_hlist_entry *
 		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
 	return 0;
 error:
-	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
-		if (cache_resource->default_miss)
-			claim_zero(mlx5_glue->destroy_flow_action
+	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB &&
+	    cache_resource->default_miss)
+		claim_zero(mlx5_glue->destroy_flow_action
 				(cache_resource->default_miss));
-	} else {
-		if (cache_resource->sample_idx.rix_hrxq &&
-		    !mlx5_hrxq_release(dev,
-				cache_resource->sample_idx.rix_hrxq))
-			cache_resource->sample_idx.rix_hrxq = 0;
-		if (cache_resource->sample_idx.rix_tag &&
-		    !flow_dv_tag_release(dev,
-				cache_resource->sample_idx.rix_tag))
-			cache_resource->sample_idx.rix_tag = 0;
-		if (cache_resource->sample_idx.cnt) {
-			flow_dv_counter_release(dev,
-				cache_resource->sample_idx.cnt);
-			cache_resource->sample_idx.cnt = 0;
-		}
-	}
+	else
+		flow_dv_sample_sub_actions_release(dev,
+						   &cache_resource->sample_idx);
 	if (cache_resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
@@ -8797,6 +8827,13 @@ struct mlx5_hlist_entry *
 					   __ATOMIC_RELAXED);
 			dev_flow->handle->dvh.rix_dest_array = idx;
 			dev_flow->dv.dest_array_res = cache_resource;
+			/*
+			 * Existing smaple action should release the prepared
+			 * sub-actions reference counter.
+			 */
+			for (idx = 0; idx < resource->num_of_dest; idx++)
+				flow_dv_sample_sub_actions_release(dev,
+						&resource->sample_idx[idx]);
 			return 0;
 		}
 	}
@@ -10693,21 +10730,8 @@ struct mlx5_hlist_entry *
 		if (cache_resource->normal_path_tbl)
 			flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
-	}
-	if (cache_resource->sample_idx.rix_hrxq &&
-		!mlx5_hrxq_release(dev,
-			cache_resource->sample_idx.rix_hrxq))
-		cache_resource->sample_idx.rix_hrxq = 0;
-	if (cache_resource->sample_idx.rix_tag &&
-		!flow_dv_tag_release(dev,
-			cache_resource->sample_idx.rix_tag))
-		cache_resource->sample_idx.rix_tag = 0;
-	if (cache_resource->sample_idx.cnt) {
-		flow_dv_counter_release(dev,
-			cache_resource->sample_idx.cnt);
-		cache_resource->sample_idx.cnt = 0;
-	}
-	if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
+		flow_dv_sample_sub_actions_release(dev,
+					&cache_resource->sample_idx);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
 			     &priv->sh->sample_action_list, idx,
 			     cache_resource, next);
@@ -10736,7 +10760,6 @@ struct mlx5_hlist_entry *
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_sub_actions_idx *mdest_act_res;
 	uint32_t idx = handle->dvh.rix_dest_array;
 	uint32_t i = 0;
 
@@ -10753,29 +10776,9 @@ struct mlx5_hlist_entry *
 		if (cache_resource->action)
 			claim_zero(mlx5_glue->destroy_flow_action
 						(cache_resource->action));
-		for (; i < cache_resource->num_of_dest; i++) {
-			mdest_act_res = &cache_resource->sample_idx[i];
-			if (mdest_act_res->rix_hrxq) {
-				mlx5_hrxq_release(dev,
-					mdest_act_res->rix_hrxq);
-				mdest_act_res->rix_hrxq = 0;
-			}
-			if (mdest_act_res->rix_encap_decap) {
-				flow_dv_encap_decap_resource_release(dev,
-					mdest_act_res->rix_encap_decap);
-				mdest_act_res->rix_encap_decap = 0;
-			}
-			if (mdest_act_res->rix_port_id_action) {
-				flow_dv_port_id_action_resource_release(dev,
-					mdest_act_res->rix_port_id_action);
-				mdest_act_res->rix_port_id_action = 0;
-			}
-			if (mdest_act_res->rix_tag) {
-				flow_dv_tag_release(dev,
-					mdest_act_res->rix_tag);
-				mdest_act_res->rix_tag = 0;
-			}
-		}
+		for (; i < cache_resource->num_of_dest; i++)
+			flow_dv_sample_sub_actions_release(dev,
+					&cache_resource->sample_idx[i]);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 			     &priv->sh->dest_array_list, idx,
 			     cache_resource, next);
Signature · 1 line
1.8.3.1
#151 lore raw

PATCH v5 29/34 net/mlx5: make sample and mirror action thread safe

In reply to Suanming Mou

This commit uses cache list to make sample and mirror action thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c |  14 ++
 drivers/net/mlx5/mlx5.h          |   5 +-
 drivers/net/mlx5/mlx5_flow.h     |  28 ++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 396 ++++++++++++++++++++++-----------------
 4 files changed, 260 insertions(+), 183 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +14 -0
index 9ab3ba9..5856981 100644
@@ -248,6 +248,20 @@
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
+	/* Init sample action cache list. */
+	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->sample_action_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_sample_create_cb,
+			     flow_dv_sample_match_cb,
+			     flow_dv_sample_remove_cb);
+	/* Init dest array action cache list. */
+	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->dest_array_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_dest_array_create_cb,
+			     flow_dv_dest_array_match_cb,
+			     flow_dv_dest_array_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
drivers/net/mlx5/mlx5.h +3 -2
index 9efa354..ac7a026 100644
@@ -661,8 +661,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	uint32_t sample_action_list; /* List of sample actions. */
-	uint32_t dest_array_list; /* List of destination array actions. */
+	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
+	struct mlx5_cache_list dest_array_list;
+	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
drivers/net/mlx5/mlx5_flow.h +23 -5
index 500a580..4a8c2bf 100644
@@ -562,9 +562,12 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *verbs_action; /**< Verbs sample action object. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	union {
+		void *verbs_action; /**< Verbs sample action object. */
+		void **sub_actions; /**< Sample sub-action array. */
+	};
+	uint32_t idx; /** Sample object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint32_t ft_id; /** Flow Table Level */
 	uint32_t ratio;   /** Sample Ratio */
@@ -581,8 +584,8 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
 	void *action; /**< Pointer to the rdma core action. */
@@ -1449,4 +1452,19 @@ struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
 void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
 				 struct mlx5_cache_entry *entry);
 
+int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
+			    struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_sample_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
+			      struct mlx5_cache_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
+				struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_dest_array_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
+				  struct mlx5_cache_entry *entry);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +220 -176

510 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#152 lore raw

PATCH v5 30/34 net/mlx5: make tunnel offloading table thread safe

In reply to Suanming Mou

To support multi-thread flow insertion, this patch updates tunnel offloading hash table to use thread safe hash list.

---
 drivers/net/mlx5/mlx5_flow.c    | 92 ++++++++++++++++++++++++++---------------
 drivers/net/mlx5/mlx5_flow_dv.c | 11 +----
 2 files changed, 61 insertions(+), 42 deletions(-)
drivers/net/mlx5/mlx5_flow.c +59 -33
index 37e7fc3..1e82030 100644
@@ -7399,13 +7399,56 @@ struct mlx5_meter_domains_infos *
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
+static void
+mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+			tunnel_flow_tbl_to_id(tte->flow_table));
+	mlx5_free(tte);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
+				   uint64_t key __rte_unused,
+				   void *ctx __rte_unused)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct tunnel_tbl_entry *tte;
+
+	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
+			  sizeof(*tte), 0,
+			  SOCKET_ID_ANY);
+	if (!tte)
+		goto err;
+	mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+			  &tte->flow_table);
+	if (tte->flow_table >= MLX5_MAX_TABLES) {
+		DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
+			tte->flow_table);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+				tte->flow_table);
+		goto err;
+	} else if (!tte->flow_table) {
+		goto err;
+	}
+	tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
+	return &tte->hash;
+err:
+	if (tte)
+		mlx5_free(tte);
+	return NULL;
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hlist_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
@@ -7416,40 +7459,17 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_hlist *group_hash;
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_lookup(group_hash, key.val, NULL);
-	if (!he) {
-		tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
-				  sizeof(*tte), 0,
-				  SOCKET_ID_ANY);
-		if (!tte)
-			goto err;
-		tte->hash.key = key.val;
-		mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-				  &tte->flow_table);
-		if (tte->flow_table >= MLX5_MAX_TABLES) {
-			DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
-				tte->flow_table);
-			mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-					tte->flow_table);
-			goto err;
-		} else if (!tte->flow_table) {
-			goto err;
-		}
-		tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
-		mlx5_hlist_insert(group_hash, &tte->hash);
-	} else {
-		tte = container_of(he, typeof(*tte), hash);
-	}
+	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	if (!he)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "tunnel group index not supported");
+	tte = container_of(he, typeof(*tte), hash);
 	*table = tte->flow_table;
 	DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
 		dev->data->port_id, key.tunnel_id, group, *table);
 	return 0;
-
-err:
-	if (tte)
-		mlx5_free(tte);
-	return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				  NULL, "tunnel group index not supported");
 }
 
 static int
@@ -7972,13 +7992,16 @@ struct mlx5_meter_domains_infos *
 		return NULL;
 	}
 	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
-					   NULL, NULL, NULL);
+					   mlx5_flow_tunnel_grp2tbl_create_cb,
+					   NULL,
+					   mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(priv->sh->ipool
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
 		mlx5_free(tunnel);
 		return NULL;
 	}
+	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8052,11 +8075,14 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
-					 0, NULL, NULL, NULL);
+					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 NULL,
+					 mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
+	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
drivers/net/mlx5/mlx5_flow_dv.c +2 -9
index 79d1317..08bbc5d 100644
@@ -8102,15 +8102,8 @@ struct mlx5_flow_tbl_resource *
 					tbl_data->tunnel->groups :
 					thub->groups;
 		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
-		if (he) {
-			struct tunnel_tbl_entry *tte;
-			tte = container_of(he, typeof(*tte), hash);
-			MLX5_ASSERT(tte->flow_table == table_id);
-			mlx5_hlist_remove(tunnel_grp_hash, he);
-			mlx5_free(tte);
-		}
-		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-				tunnel_flow_tbl_to_id(table_id));
+		if (he)
+			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
 			"Table_id %#x tunnel %u group %u released.",
 			table_id,
Signature · 1 line
1.8.3.1
#153 lore raw

PATCH v5 31/34 net/mlx5: remove unused hash list operations

In reply to Suanming Mou

In previous commits the hash list objects have been converted to new thread safe hash list. The legacy hash list code can be removed now.

---
 drivers/net/mlx5/mlx5_utils.c | 38 -------------------------
 drivers/net/mlx5/mlx5_utils.h | 66 -------------------------------------------
 2 files changed, 104 deletions(-)
drivers/net/mlx5/mlx5_utils.c +0 -38
index df15f63..13590dd 100644
@@ -170,44 +170,6 @@ struct mlx5_hlist_entry*
 	return entry;
 }
 
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup_ex(struct mlx5_hlist *h, uint64_t key,
-		     mlx5_hlist_match_callback_fn cb, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h && cb && ctx);
-	idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	first = &h->heads[idx];
-	LIST_FOREACH(node, first, next) {
-		if (!cb(node, ctx))
-			return node;
-	}
-	return NULL;
-}
-
-int
-mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry,
-		     mlx5_hlist_match_callback_fn cb, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h && entry && cb && ctx);
-	idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
-	first = &h->heads[idx];
-	/* No need to reuse the lookup function. */
-	LIST_FOREACH(node, first, next) {
-		if (!cb(node, ctx))
-			return -EEXIST;
-	}
-	LIST_INSERT_HEAD(first, entry, next);
-	return 0;
-}
-
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
 {
drivers/net/mlx5/mlx5_utils.h +0 -66

84 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#154 lore raw

PATCH v5 32/34 net/mlx5: make tunnel hub list thread safe

In reply to Suanming Mou

This commit uses spinlock to protect the tunnel hub list in multiple thread.

---
 drivers/net/mlx5/mlx5_flow.c | 20 +++++++++++++++++---
 drivers/net/mlx5/mlx5_flow.h |  1 +
 2 files changed, 18 insertions(+), 3 deletions(-)
drivers/net/mlx5/mlx5_flow.c +17 -3
index 1e82030..a6e60af 100644
@@ -669,10 +669,14 @@ enum mlx5_expansion {
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
-		if (&tun->item == pmd_items)
+		if (&tun->item == pmd_items) {
+			LIST_REMOVE(tun, chain);
 			break;
+		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (!tun || num_items != 1)
 		return rte_flow_error_set(err, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
@@ -690,10 +694,14 @@ enum mlx5_expansion {
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
-		if (&tun->action == pmd_actions)
+		if (&tun->action == pmd_actions) {
+			LIST_REMOVE(tun, chain);
 			break;
+		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (!tun || num_actions != 1)
 		return rte_flow_error_set(err, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
@@ -5880,8 +5888,12 @@ struct rte_flow *
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
+
+		rte_spinlock_lock(&mlx5_tunnel_hub(dev)->sl);
 		tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
 		RTE_VERIFY(tunnel);
+		LIST_REMOVE(tunnel, chain);
+		rte_spinlock_unlock(&mlx5_tunnel_hub(dev)->sl);
 		if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
@@ -7940,7 +7952,6 @@ struct mlx5_meter_domains_infos *
 	DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
 		dev->data->port_id, tunnel->tunnel_id);
 	RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED));
-	LIST_REMOVE(tunnel, chain);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID],
 			tunnel->tunnel_id);
 	mlx5_hlist_destroy(tunnel->groups);
@@ -8029,6 +8040,7 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
 		if (!memcmp(app_tunnel, &tun->app_tunnel,
 			    sizeof(*app_tunnel))) {
@@ -8046,6 +8058,7 @@ struct mlx5_meter_domains_infos *
 			ret = -ENOMEM;
 		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (tun)
 		__atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED);
 
@@ -8074,6 +8087,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	if (!thub)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
+	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
 					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
 					 NULL,
drivers/net/mlx5/mlx5_flow.h +1 -0
index 4a8c2bf..8ef2a85 100644
@@ -945,6 +945,7 @@ struct mlx5_flow_tunnel {
 /** PMD tunnel related context */
 struct mlx5_flow_tunnel_hub {
 	LIST_HEAD(, mlx5_flow_tunnel) tunnels;
+	rte_spinlock_t sl;			/* Tunnel list spinlock. */
 	struct mlx5_hlist *groups;		/** non tunnel groups */
 };
 
Signature · 1 line
1.8.3.1
#155 lore raw

PATCH v5 33/34 net/mlx5: make shared action list thread safe

In reply to Suanming Mou

This commit uses spinlock to protect the shared action list in multiple thread.

---
 drivers/net/mlx5/linux/mlx5_os.c | 1 +
 drivers/net/mlx5/mlx5.h          | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c  | 5 +++++
 3 files changed, 7 insertions(+)
drivers/net/mlx5/linux/mlx5_os.c +1 -0
index 5856981..8612cab 100644
@@ -1534,6 +1534,7 @@
 		}
 		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
+	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
 	return eth_dev;
 error:
drivers/net/mlx5/mlx5.h +1 -0
index ac7a026..be21a9a 100644
@@ -912,6 +912,7 @@ struct mlx5_priv {
 	uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
 	struct mlx5_mp_id mp_id; /* ID of a multi-process process */
 	LIST_HEAD(fdir, mlx5_fdir_flow) fdir_flows; /* fdir flows. */
+	rte_spinlock_t shared_act_sl; /* Shared actions spinlock. */
 	LIST_HEAD(shared_action, rte_flow_shared_action) shared_actions;
 	/* shared actions */
 };
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index 08bbc5d..cf48402 100644
@@ -11191,7 +11191,9 @@ struct mlx5_cache_entry *
 	if (shared_action) {
 		__atomic_add_fetch(&shared_action->refcnt, 1,
 				   __ATOMIC_RELAXED);
+		rte_spinlock_lock(&priv->shared_act_sl);
 		LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next);
+		rte_spinlock_unlock(&priv->shared_act_sl);
 	}
 	return shared_action;
 }
@@ -11218,6 +11220,7 @@ struct mlx5_cache_entry *
 			 struct rte_flow_shared_action *action,
 			 struct rte_flow_error *error)
 {
+	struct mlx5_priv *priv = dev->data->dev_private;
 	int ret;
 
 	switch (action->type) {
@@ -11232,7 +11235,9 @@ struct mlx5_cache_entry *
 	}
 	if (ret)
 		return ret;
+	rte_spinlock_lock(&priv->shared_act_sl);
 	LIST_REMOVE(action, next);
+	rte_spinlock_unlock(&priv->shared_act_sl);
 	rte_free(action);
 	return 0;
 }
Signature · 1 line
1.8.3.1
#156 lore raw

PATCH v5 34/34 net/mlx5: remove shared context lock

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch removes shared data lock since all resources should support concurrent protection.

---
 doc/guides/nics/mlx5.rst               |   1 +
 doc/guides/rel_notes/release_20_11.rst |   1 +
 drivers/net/mlx5/linux/mlx5_os.c       |   4 +-
 drivers/net/mlx5/mlx5.h                |   1 -
 drivers/net/mlx5/mlx5_flow_dv.c        | 209 ++++-----------------------------
 5 files changed, 27 insertions(+), 189 deletions(-)
doc/guides/nics/mlx5.rst +1 -0
index e5e55fc..8c59cc6 100644
@@ -96,6 +96,7 @@ Features
 - Per packet no-inline hint flag to disable packet data copying into Tx descriptors.
 - Hardware LRO.
 - Hairpin.
+- Multiple-thread flow insertion.
 
 Limitations
 -----------
doc/guides/rel_notes/release_20_11.rst +1 -0
index e284771..0742fc8 100644
@@ -358,6 +358,7 @@ New Features
   * Added support for QinQ packets matching.
   * Added support for the new vlan fields ``has_vlan`` in the eth item and
     ``has_more_vlan`` in the vlan item.
+  * Added support for PMD level multiple-thread flow insertion.
 
 * **Updated vhost sample application.**
 
drivers/net/mlx5/linux/mlx5_os.c +2 -2
index 8612cab..d4f2194 100644
@@ -321,7 +321,6 @@
 		err = errno;
 		goto error;
 	}
-	pthread_mutex_init(&sh->dv_mutex, NULL);
 	sh->tx_domain = domain;
 #ifdef HAVE_MLX5DV_DR_ESWITCH
 	if (priv->config.dv_esw_en) {
@@ -435,7 +434,6 @@
 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 		sh->pop_vlan_action = NULL;
 	}
-	pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
 	if (sh->default_miss_action)
 		mlx5_glue->destroy_flow_action
@@ -1536,6 +1534,8 @@
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
+	if (priv->config.dv_flow_en)
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
 	return eth_dev;
 error:
 	if (priv) {
drivers/net/mlx5/mlx5.h +0 -1
index be21a9a..9a458cd 100644
@@ -639,7 +639,6 @@ struct mlx5_dev_ctx_shared {
 	/* Packet pacing related structure. */
 	struct mlx5_dev_txpp txpp;
 	/* Shared DV/DR flow data section. */
-	pthread_mutex_t dv_mutex; /* DV context mutex. */
 	uint32_t dv_meta_mask; /* flow META metadata supported mask. */
 	uint32_t dv_mark_mask; /* flow MARK metadata supported mask. */
 	uint32_t dv_regc0_mask; /* available bits of metatada reg_c[0]. */
drivers/net/mlx5/mlx5_flow_dv.c +23 -186

296 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#157 lore raw

v5 cover net/mlx5: support multiple-thread flow operations

In reply to Suanming Mou

This patch set contains multiple-thread flow operations support for the flow objects.

Suanming Mou (19):
net/mlx5: use thread safe index pool for flow objects net/mlx5: make meter action thread safe net/mlx5: make VLAN network interface thread safe net/mlx5: create global jump action
net/mlx5: create global default miss action net/mlx5: create global drop action
net/mlx5: add flow table tunnel offload attribute net/mlx5: fix redundant Direct Verbs resources allocate net/mlx5: remove unused mreg copy code net/mlx5: make header reformat action thread safe net/mlx5: optimize shared RSS list operation net/mlx5: make Rx queue thread safe
net/mlx5: simplify sample attributes
net/mlx5: fix sample register error flow net/mlx5: make sample and mirror action thread safe net/mlx5: make tunnel offloading table thread safe net/mlx5: remove unused hash list operations net/mlx5: make tunnel hub list thread safe net/mlx5: make shared action list thread safe

Xueming Li (15):
net/mlx5: use thread specific flow workspace net/mlx5: reuse flow Id as hairpin Id net/mlx5: indexed pool supports zero size entry net/mlx5: use indexed pool as ID generator net/mlx5: make rte flow list thread safe net/mlx5: support concurrent access for hash list net/mlx5: make flow table cache thread safe net/mlx5: make flow tag list thread safe net/mlx5: make flow modify action list thread safe net/mlx5: make metadata copy flow list thread safe net/mlx5: introduce thread safe linked list cache net/mlx5: make matcher list thread safe net/mlx5: make port ID action cache thread safe net/mlx5: make push VLAN action cache thread safe net/mlx5: remove shared context lock
---

v5:
- fix table refcnt leak issue.

v4:
- rebase the code on top of the latest code to fix the conflict.

v3:
- rebase the code on top of the new added features.
- New feature code adaption for multiple-thread support.

v2:
- fix review comments.
- remove unused mreg copy code.
- drop the sh lock removing patch as new feature still not fully updated.

---

 doc/guides/nics/mlx5.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst |    1 +
 drivers/common/mlx5/linux/mlx5_nl.h    |    1 +
 drivers/net/mlx5/linux/mlx5_os.c       |  119 +-
 drivers/net/mlx5/linux/mlx5_vlan_os.c  |    5 +
 drivers/net/mlx5/mlx5.c                |  278 +---
 drivers/net/mlx5/mlx5.h                |   69 +-
 drivers/net/mlx5/mlx5_flow.c           |  756 +++++------
 drivers/net/mlx5/mlx5_flow.h           |  164 ++-
 drivers/net/mlx5/mlx5_flow_dv.c        | 2259 ++++++++++++++------------------
 drivers/net/mlx5/mlx5_flow_meter.c     |   72 +-
 drivers/net/mlx5/mlx5_flow_verbs.c     |   86 +-
 drivers/net/mlx5/mlx5_rxq.c            |  302 +++--
 drivers/net/mlx5/mlx5_rxtx.h           |   23 +-
 drivers/net/mlx5/mlx5_utils.c          |  345 ++++-
 drivers/net/mlx5/mlx5_utils.h          |  320 ++++-
 16 files changed, 2464 insertions(+), 2337 deletions(-)
Signature · 1 line
1.8.3.1
#158 lore raw

PATCH v5 1/34 net/mlx5: use thread safe index pool for flow objects

As mlx5 PMD is changed to be thread safe, all the flow-related sub-objects inside the PMD should be thread safe. This commit changes the index memory pools' lock configuration to be enabled. That makes the index pool be thread safe.

---
 drivers/net/mlx5/mlx5.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5.c +11 -11
index 27c9c2a..dc5b7a4 100644
@@ -191,7 +191,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -202,7 +202,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -213,7 +213,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -224,7 +224,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -235,7 +235,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -246,7 +246,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -257,7 +257,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -269,7 +269,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -280,7 +280,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -291,7 +291,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -306,7 +306,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
Signature · 1 line
1.8.3.1
#159 lore raw

Re: [dpdk-dev] [PATCH v5 01/34] net/mlx5: use thread safe index pool for flow objects

Hi,

Quoted 135 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Wednesday, October 28, 2020 11:33 AM
To: Matan Azrad <[email protected]>; Shahaf Shuler <[email protected]>; Slava Ovsiienko <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH v5 01/34] net/mlx5: use thread safe index pool for flow objects

As mlx5 PMD is changed to be thread safe, all the flow-related sub-objects inside the PMD should be thread safe. This commit changes the index memory pools' lock configuration to be enabled. That makes the index pool be thread safe.

---
 drivers/net/mlx5/mlx5.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5.c +0 -0
index 27c9c2a..dc5b7a4 100644
@@ -191,7 +191,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -202,7 +202,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -213,7 +213,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -224,7 +224,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -235,7 +235,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -246,7 +246,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -257,7 +257,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -269,7 +269,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -280,7 +280,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -291,7 +291,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -1
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
@@ -306,7 +306,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared)

mlx5_dev_ctx_list =

drivers/net/mlx5/mlx5.c +1 -2
 		.trunk_size = 64,
 		.grow_trunk = 3,
 		.grow_shift = 2,
-		.need_lock = 0,
+		.need_lock = 1,
 		.release_mem_en = 1,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
--
Quoted 1 line

1.8.3.1

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh

#160 lore raw

PATCH v5 2/34 net/mlx5: use thread specific flow workspace

In reply to Suanming Mou

From: Xueming Li <[email protected]>

As part of multi-thread flow support, this patch moves flow intermediate data to thread specific, makes them a flow workspace. The workspace is allocated per thread, destroyed along with thread life-cycle.

---
 drivers/net/mlx5/linux/mlx5_os.c   |   5 --
 drivers/net/mlx5/mlx5.c            |   2 -
 drivers/net/mlx5/mlx5.h            |   6 --
 drivers/net/mlx5/mlx5_flow.c       | 168 ++++++++++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_flow.h       |  15 +++-
 drivers/net/mlx5/mlx5_flow_dv.c    |  40 +++++----
 drivers/net/mlx5/mlx5_flow_verbs.c |  24 +++---
 7 files changed, 171 insertions(+), 89 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +0 -5
index c890998..4ba6d8e 100644
@@ -1449,11 +1449,6 @@
 			err = ENOTSUP;
 			goto error;
 	}
-	/*
-	 * Allocate the buffer for flow creating, just once.
-	 * The allocation must be done before any flow creating.
-	 */
-	mlx5_flow_alloc_intermediate(eth_dev);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.c +0 -2
index dc5b7a4..bb494c1 100644
@@ -1400,8 +1400,6 @@ struct mlx5_dev_ctx_shared *
 	mlx5_flow_list_flush(dev, &priv->flows, true);
 	mlx5_shared_action_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
-	/* Free the intermediate buffers for flow creation. */
-	mlx5_flow_free_intermediate(dev);
 	/* Prevent crashes when queues are still in use. */
 	dev->rx_pkt_burst = removed_rx_burst;
 	dev->tx_pkt_burst = removed_tx_burst;
drivers/net/mlx5/mlx5.h +0 -6
index 8de5842..5400b95 100644
@@ -854,10 +854,6 @@ struct mlx5_priv {
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	uint32_t flows; /* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
-	void *inter_flows; /* Intermediate resources for flow creation. */
-	void *rss_desc; /* Intermediate rss description resources. */
-	int flow_idx; /* Intermediate device flow index. */
-	int flow_nested_idx; /* Intermediate device flow index, nested. */
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
@@ -1104,8 +1100,6 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 void mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list);
 int mlx5_flow_start_default(struct rte_eth_dev *dev);
 void mlx5_flow_stop_default(struct rte_eth_dev *dev);
-void mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev);
-void mlx5_flow_free_intermediate(struct rte_eth_dev *dev);
 int mlx5_flow_verify(struct rte_eth_dev *dev);
 int mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, uint32_t queue);
 int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow.c +120 -48

235 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +14 -1
index 06db440..668fcc3 100644
@@ -75,6 +75,9 @@ enum mlx5_feature_name {
 	MLX5_MTR_SFX,
 };
 
+/* Default queue number. */
+#define MLX5_RSSQ_DEFAULT_NUM 16
+
 #define MLX5_FLOW_LAYER_OUTER_L2 (1u << 0)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV4 (1u << 1)
 #define MLX5_FLOW_LAYER_OUTER_L3_IPV6 (1u << 2)
@@ -603,7 +606,7 @@ struct mlx5_flow_rss_desc {
 	uint32_t queue_num; /**< Number of entries in @p queue. */
 	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
-	uint16_t queue[]; /**< Destination queues to redirect traffic to. */
+	uint16_t *queue; /**< Destination queues. */
 };
 
 /* PMD flow priority for tunnel */
@@ -1102,6 +1105,15 @@ struct rte_flow_shared_action {
 	};
 };
 
+/* Thread specific flow workspace intermediate data. */
+struct mlx5_flow_workspace {
+	struct mlx5_flow flows[MLX5_NUM_MAX_DEV_FLOWS];
+	struct mlx5_flow_rss_desc rss_desc[2];
+	uint32_t rssq_num[2]; /* Allocated queue num in rss_desc. */
+	int flow_idx; /* Intermediate device flow index. */
+	int flow_nested_idx; /* Intermediate device flow index, nested. */
+};
+
 typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
 				    const struct rte_flow_attr *attr,
 				    const struct rte_flow_item items[],
@@ -1204,6 +1216,7 @@ struct mlx5_flow_driver_ops {
 
 /* mlx5_flow.c */
 
+struct mlx5_flow_workspace *mlx5_flow_get_thread_workspace(void);
 struct mlx5_flow_id_pool *mlx5_flow_id_pool_alloc(uint32_t max_id);
 void mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool);
 uint32_t mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id);
drivers/net/mlx5/mlx5_flow_dv.c +23 -17
index 4a35010..f4f8e15 100644
@@ -6253,9 +6253,11 @@ struct field_modify_info modify_tcp[] = {
 	struct mlx5_flow *dev_flow;
 	struct mlx5_flow_handle *dev_handle;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
+	MLX5_ASSERT(wks);
 	/* In case of corrupting the memory. */
-	if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+	if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
 		rte_flow_error_set(error, ENOSPC,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "not free temporary device flow");
@@ -6269,8 +6271,8 @@ struct field_modify_info modify_tcp[] = {
 				   "not enough memory to create flow handle");
 		return NULL;
 	}
-	/* No multi-thread supporting. */
-	dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+	MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
+	dev_flow = &wks->flows[wks->flow_idx++];
 	dev_flow->handle = dev_handle;
 	dev_flow->handle_idx = handle_idx;
 	/*
@@ -8942,11 +8944,12 @@ struct field_modify_info modify_tcp[] = {
 	const struct rte_flow_action_queue *queue;
 	struct mlx5_flow_sub_actions_list *sample_act;
 	struct mlx5_flow_sub_actions_idx *sample_idx;
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	uint64_t action_flags = 0;
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	sample_act = &res->sample_act;
 	sample_idx = &res->sample_idx;
 	sample_action = (const struct rte_flow_action_sample *)action->conf;
@@ -9148,18 +9151,18 @@ struct field_modify_info modify_tcp[] = {
 			     uint64_t action_flags,
 			     struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	/* update normal path action resource into last index of array */
 	uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
 	struct mlx5_flow_sub_actions_list *sample_act =
 					&mdest_res->sample_act[dest_index];
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	uint32_t normal_idx = 0;
 	struct mlx5_hrxq *hrxq;
 	uint32_t hrxq_idx;
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	if (num_of_dest > 1) {
 		if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
 			/* Handle QP action for mirroring */
@@ -9249,9 +9252,8 @@ struct field_modify_info modify_tcp[] = {
 	struct mlx5_dev_config *dev_conf = &priv->config;
 	struct rte_flow *flow = dev_flow->flow;
 	struct mlx5_flow_handle *handle = dev_flow->handle;
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	uint64_t item_flags = 0;
 	uint64_t last_item = 0;
 	uint64_t action_flags = 0;
@@ -9297,6 +9299,8 @@ struct field_modify_info modify_tcp[] = {
 		.fdb_def_rule = !!priv->fdb_def_rule,
 	};
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
 	memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
 	mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
@@ -10280,6 +10284,7 @@ struct field_modify_info modify_tcp[] = {
 			   struct mlx5_hrxq **hrxq)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 	uint32_t hrxq_idx;
 
 	if (flow->shared_rss) {
@@ -10295,8 +10300,7 @@ struct field_modify_info modify_tcp[] = {
 		}
 	} else {
 		struct mlx5_flow_rss_desc *rss_desc =
-				&((struct mlx5_flow_rss_desc *)priv->rss_desc)
-				[!!priv->flow_nested_idx];
+				&wks->rss_desc[!!wks->flow_nested_idx];
 
 		MLX5_ASSERT(rss_desc->queue_num);
 		hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
@@ -10347,9 +10351,11 @@ struct field_modify_info modify_tcp[] = {
 	int n;
 	int err;
 	int idx;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
-	for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
-		dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
+	MLX5_ASSERT(wks);
+	for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) {
+		dev_flow = &wks->flows[idx];
 		dv = &dev_flow->dv;
 		dh = dev_flow->handle;
 		dv_h = &dh->dvh;
drivers/net/mlx5/mlx5_flow_verbs.c +14 -10
index d04c37f..0ec9acd 100644
@@ -1626,7 +1626,9 @@
 	struct mlx5_flow *dev_flow;
 	struct mlx5_flow_handle *dev_handle;
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
+	MLX5_ASSERT(wks);
 	size += flow_verbs_get_actions_size(actions);
 	size += flow_verbs_get_items_size(items);
 	if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
@@ -1636,7 +1638,7 @@
 		return NULL;
 	}
 	/* In case of corrupting the memory. */
-	if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+	if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
 		rte_flow_error_set(error, ENOSPC,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				   "not free temporary device flow");
@@ -1650,8 +1652,8 @@
 				   "not enough memory to create flow handle");
 		return NULL;
 	}
-	/* No multi-thread supporting. */
-	dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+	MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
+	dev_flow = &wks->flows[wks->flow_idx++];
 	dev_flow->handle = dev_handle;
 	dev_flow->handle_idx = handle_idx;
 	/* Memcpy is used, only size needs to be cleared to 0. */
@@ -1695,11 +1697,12 @@
 	uint64_t priority = attr->priority;
 	uint32_t subpriority = 0;
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-					      priv->rss_desc)
-					      [!!priv->flow_nested_idx];
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+	struct mlx5_flow_rss_desc *rss_desc;
 	char errstr[32];
 
+	MLX5_ASSERT(wks);
+	rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
 	if (priority == MLX5_FLOW_PRIO_RSVD)
 		priority = priv->config.flow_prio - 1;
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
@@ -1956,9 +1959,11 @@
 	uint32_t dev_handles;
 	int err;
 	int idx;
+	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
-	for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
-		dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
+	MLX5_ASSERT(wks);
+	for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) {
+		dev_flow = &wks->flows[idx];
 		handle = dev_flow->handle;
 		if (handle->fate_action == MLX5_FLOW_FATE_DROP) {
 			hrxq = mlx5_drop_action_create(dev);
@@ -1972,8 +1977,7 @@
 		} else {
 			uint32_t hrxq_idx;
 			struct mlx5_flow_rss_desc *rss_desc =
-				&((struct mlx5_flow_rss_desc *)priv->rss_desc)
-				[!!priv->flow_nested_idx];
+				&wks->rss_desc[!!wks->flow_nested_idx];
 
 			MLX5_ASSERT(rss_desc->queue_num);
 			hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
Signature · 1 line
1.8.3.1
#161 lore raw

PATCH v5 3/34 net/mlx5: reuse flow Id as hairpin Id

In reply to Suanming Mou

From: Xueming Li <[email protected]>

Hairpin flow matching required a unique flow ID for matching. This patch reuses flow ID as hairpin flow ID, this will save some code to generate a separate hairpin ID, also saves flow memory by removing hairpin ID.

---
 drivers/net/mlx5/mlx5.c      | 11 -----------
 drivers/net/mlx5/mlx5.h      |  1 -
 drivers/net/mlx5/mlx5_flow.c | 30 +++++++++---------------------
 drivers/net/mlx5/mlx5_flow.h |  6 +-----
 4 files changed, 10 insertions(+), 38 deletions(-)
drivers/net/mlx5/mlx5.c +0 -11
index bb494c1..eca97b0 100644
@@ -979,13 +979,6 @@ struct mlx5_dev_ctx_shared *
 		MLX5_ASSERT(sh->devx_rx_uar);
 		MLX5_ASSERT(mlx5_os_get_devx_uar_base_addr(sh->devx_rx_uar));
 	}
-	sh->flow_id_pool = mlx5_flow_id_pool_alloc
-					((1 << HAIRPIN_FLOW_ID_BITS) - 1);
-	if (!sh->flow_id_pool) {
-		DRV_LOG(ERR, "can't create flow id pool");
-		err = ENOMEM;
-		goto error;
-	}
 #ifndef RTE_ARCH_64
 	/* Initialize UAR access locks for 32bit implementations. */
 	rte_spinlock_init(&sh->uar_lock_cq);
@@ -1047,8 +1040,6 @@ struct mlx5_dev_ctx_shared *
 		claim_zero(mlx5_glue->dealloc_pd(sh->pd));
 	if (sh->ctx)
 		claim_zero(mlx5_glue->close_device(sh->ctx));
-	if (sh->flow_id_pool)
-		mlx5_flow_id_pool_release(sh->flow_id_pool);
 	mlx5_free(sh);
 	MLX5_ASSERT(err > 0);
 	rte_errno = err;
@@ -1119,8 +1110,6 @@ struct mlx5_dev_ctx_shared *
 		mlx5_glue->devx_free_uar(sh->devx_rx_uar);
 	if (sh->ctx)
 		claim_zero(mlx5_glue->close_device(sh->ctx));
-	if (sh->flow_id_pool)
-		mlx5_flow_id_pool_release(sh->flow_id_pool);
 	pthread_mutex_destroy(&sh->txpp.mutex);
 	mlx5_free(sh);
 	return;
drivers/net/mlx5/mlx5.h +0 -1
index 5400b95..8e00065 100644
@@ -668,7 +668,6 @@ struct mlx5_dev_ctx_shared {
 	void *devx_comp; /* DEVX async comp obj. */
 	struct mlx5_devx_obj *tis; /* TIS object. */
 	struct mlx5_devx_obj *td; /* Transport domain. */
-	struct mlx5_flow_id_pool *flow_id_pool; /* Flow ID pool. */
 	void *tx_uar; /* Tx/packet pacing shared UAR. */
 	struct mlx5_flex_parser_profiles fp[MLX5_FLEX_PARSER_MAX];
 	/* Flex parser profiles information. */
drivers/net/mlx5/mlx5_flow.c +9 -21
index e6a4571..4733885 100644
@@ -4154,9 +4154,8 @@ struct rte_flow_shared_action *
 		   struct rte_flow_action actions_rx[],
 		   struct rte_flow_action actions_tx[],
 		   struct rte_flow_item pattern_tx[],
-		   uint32_t *flow_id)
+		   uint32_t flow_id)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	const struct rte_flow_action_raw_encap *raw_encap;
 	const struct rte_flow_action_raw_decap *raw_decap;
 	struct mlx5_rte_flow_action_set_tag *set_tag;
@@ -4166,7 +4165,6 @@ struct rte_flow_shared_action *
 	char *addr;
 	int encap = 0;
 
-	mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
@@ -4235,7 +4233,7 @@ struct rte_flow_shared_action *
 	set_tag = (void *)actions_rx;
 	set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
 	MLX5_ASSERT(set_tag->id > REG_NON);
-	set_tag->data = *flow_id;
+	set_tag->data = flow_id;
 	tag_action->conf = set_tag;
 	/* Create Tx item list. */
 	rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
@@ -4244,7 +4242,7 @@ struct rte_flow_shared_action *
 	item->type = (enum rte_flow_item_type)
 		     MLX5_RTE_FLOW_ITEM_TYPE_TAG;
 	tag_item = (void *)addr;
-	tag_item->data = *flow_id;
+	tag_item->data = flow_id;
 	tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
 	MLX5_ASSERT(set_tag->id > REG_NON);
 	item->spec = tag_item;
@@ -5630,7 +5628,6 @@ struct tunnel_default_miss_ctx {
 	uint32_t i;
 	uint32_t idx = 0;
 	int hairpin_flow;
-	uint32_t hairpin_id = 0;
 	struct rte_flow_attr attr_tx = { .priority = 0 };
 	struct rte_flow_attr attr_factor = {0};
 	const struct rte_flow_action *actions;
@@ -5659,6 +5656,11 @@ struct tunnel_default_miss_ctx {
 				external, hairpin_flow, error);
 	if (ret < 0)
 		goto error_before_hairpin_split;
+	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
+	if (!flow) {
+		rte_errno = ENOMEM;
+		goto error_before_hairpin_split;
+	}
 	if (hairpin_flow > 0) {
 		if (hairpin_flow > MLX5_MAX_SPLIT_ACTIONS) {
 			rte_errno = EINVAL;
@@ -5666,17 +5668,10 @@ struct tunnel_default_miss_ctx {
 		}
 		flow_hairpin_split(dev, actions, actions_rx.actions,
 				   actions_hairpin_tx.actions, items_tx.items,
-				   &hairpin_id);
+				   idx);
 		p_actions_rx = actions_rx.actions;
 	}
-	flow = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], &idx);
-	if (!flow) {
-		rte_errno = ENOMEM;
-		goto error_before_flow;
-	}
 	flow->drv_type = flow_get_drv_type(dev, &attr_factor);
-	if (hairpin_id != 0)
-		flow->hairpin_flow_id = hairpin_id;
 	MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
 		    flow->drv_type < MLX5_FLOW_TYPE_MAX);
 	memset(rss_desc, 0, offsetof(struct mlx5_flow_rss_desc, queue));
@@ -5812,11 +5807,7 @@ struct tunnel_default_miss_ctx {
 	flow_drv_destroy(dev, flow);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], idx);
 	rte_errno = ret; /* Restore rte_errno. */
-error_before_flow:
 	ret = rte_errno;
-	if (hairpin_id)
-		mlx5_flow_id_release(priv->sh->flow_id_pool,
-				     hairpin_id);
 	rte_errno = ret;
 	wks->flow_idx = wks->flow_nested_idx;
 	if (wks->flow_nested_idx)
@@ -5974,9 +5965,6 @@ struct rte_flow *
 	 */
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
-	if (flow->hairpin_flow_id)
-		mlx5_flow_id_release(priv->sh->flow_id_pool,
-				     flow->hairpin_flow_id);
 	flow_drv_destroy(dev, flow);
 	if (list)
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
drivers/net/mlx5/mlx5_flow.h +1 -5
index 668fcc3..e6ec709 100644
@@ -922,8 +922,6 @@ struct mlx5_fdir_flow {
 	uint32_t rix_flow; /* Index to flow. */
 };
 
-#define HAIRPIN_FLOW_ID_BITS 28
-
 #define MLX5_MAX_TUNNELS 256
 #define MLX5_TNL_MISS_RULE_PRIORITY 3
 #define MLX5_TNL_MISS_FDB_JUMP_GRP  0x1234faac
@@ -1039,14 +1037,12 @@ struct rte_flow {
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t fdir:1; /**< Identifier of associated FDIR if any. */
 	uint32_t tunnel:1;
-	uint32_t hairpin_flow_id:HAIRPIN_FLOW_ID_BITS;
-	/**< The flow id used for hairpin. */
 	uint32_t copy_applied:1; /**< The MARK copy Flow os applied. */
+	uint32_t meter:16; /**< Holds flow meter id. */
 	uint32_t rix_mreg_copy;
 	/**< Index to metadata register copy table resource. */
 	uint32_t counter; /**< Holds flow counter. */
 	uint32_t tunnel_id;  /**< Tunnel id */
-	uint16_t meter; /**< Holds flow meter id. */
 } __rte_packed;
 
 /*
Signature · 1 line
1.8.3.1
#162 lore raw

PATCH v5 4/34 net/mlx5: indexed pool supports zero size entry

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To make indexed pool to be used as ID generator, this patch allows entry size to be zero.

---
 drivers/net/mlx5/mlx5_utils.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +2 -2
index 9a54fda..3a1f87a 100644
@@ -230,7 +230,7 @@ struct mlx5_indexed_pool *
 	struct mlx5_indexed_pool *pool;
 	uint32_t i;
 
-	if (!cfg || !cfg->size || (!cfg->malloc ^ !cfg->free) ||
+	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -391,7 +391,7 @@ struct mlx5_indexed_pool *
 {
 	void *entry = mlx5_ipool_malloc(pool, idx);
 
-	if (entry)
+	if (entry && pool->cfg.size)
 		memset(entry, 0, pool->cfg.size);
 	return entry;
 }
Signature · 1 line
1.8.3.1
#163 lore raw

PATCH v5 5/34 net/mlx5: use indexed pool as ID generator

In reply to Suanming Mou

From: Xueming Li <[email protected]>

The ID generation API used an integer pool to save released ID, To support multiple flow, it has to be enhanced to be thread safe.

Indexed pool could be used to generate unique ID by setting size of pool entry to zero. Since bitmap is used, an extra benefits is saving memory to about one bit per entry. Further more indexed pool could be thread safe by enabling lock.

This patch leverages indexed pool to generate ID, removes unused ID generating API.

---
 drivers/net/mlx5/linux/mlx5_os.c |  13 ----
 drivers/net/mlx5/mlx5.c          | 136 +++++----------------------------------
 drivers/net/mlx5/mlx5.h          |   4 +-
 drivers/net/mlx5/mlx5_flow.c     | 116 +++++++++++++++------------------
 drivers/net/mlx5/mlx5_flow.h     |   7 --
 drivers/net/mlx5/mlx5_flow_dv.c  |   4 +-
 drivers/net/mlx5/mlx5_utils.c    |   5 ++
 7 files changed, 78 insertions(+), 207 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +0 -13
index 4ba6d8e..0b59e74 100644
@@ -1395,17 +1395,6 @@
 		err = mlx5_alloc_shared_dr(priv);
 		if (err)
 			goto error;
-		/*
-		 * RSS id is shared with meter flow id. Meter flow id can only
-		 * use the 24 MSB of the register.
-		 */
-		priv->qrss_id_pool = mlx5_flow_id_pool_alloc(UINT32_MAX >>
-				     MLX5_MTR_COLOR_BITS);
-		if (!priv->qrss_id_pool) {
-			DRV_LOG(ERR, "can't create flow id pool");
-			err = ENOMEM;
-			goto error;
-		}
 	}
 	if (config->devx && config->dv_flow_en && config->dest_tir) {
 		priv->obj_ops = devx_obj_ops;
@@ -1492,8 +1481,6 @@
 			close(priv->nl_socket_rdma);
 		if (priv->vmwa_context)
 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
-		if (priv->qrss_id_pool)
-			mlx5_flow_id_pool_release(priv->qrss_id_pool);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		mlx5_free(priv);
drivers/net/mlx5/mlx5.c +15 -121

148 lines not on this page · raw message

drivers/net/mlx5/mlx5.h +3 -1
index 8e00065..5bda233 100644
@@ -47,6 +47,9 @@ enum mlx5_ipool_index {
 	MLX5_IPOOL_HRXQ, /* Pool for hrxq resource. */
 	MLX5_IPOOL_MLX5_FLOW, /* Pool for mlx5 flow handle. */
 	MLX5_IPOOL_RTE_FLOW, /* Pool for rte_flow. */
+	MLX5_IPOOL_RSS_EXPANTION_FLOW_ID, /* Pool for Queue/RSS flow ID. */
+	MLX5_IPOOL_TUNNEL_ID, /* Pool for flow tunnel ID. */
+	MLX5_IPOOL_TNL_TBL_ID, /* Pool for tunnel table ID. */
 	MLX5_IPOOL_MAX,
 };
 
@@ -877,7 +880,6 @@ struct mlx5_priv {
 	int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
 	struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */
 	struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
-	struct mlx5_flow_id_pool *qrss_id_pool;
 	struct mlx5_hlist *mreg_cp_tbl;
 	/* Hash table of Rx metadata register copy table. */
 	uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */
drivers/net/mlx5/mlx5_flow.c +53 -63

249 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +0 -7
index e6ec709..629cdbe 100644
@@ -951,8 +951,6 @@ struct mlx5_flow_tunnel {
 /** PMD tunnel related context */
 struct mlx5_flow_tunnel_hub {
 	LIST_HEAD(, mlx5_flow_tunnel) tunnels;
-	struct mlx5_flow_id_pool *tunnel_ids;
-	struct mlx5_flow_id_pool *table_ids;
 	struct mlx5_hlist *groups;		/** non tunnel groups */
 };
 
@@ -1213,11 +1211,6 @@ struct mlx5_flow_driver_ops {
 /* mlx5_flow.c */
 
 struct mlx5_flow_workspace *mlx5_flow_get_thread_workspace(void);
-struct mlx5_flow_id_pool *mlx5_flow_id_pool_alloc(uint32_t max_id);
-void mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool);
-uint32_t mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id);
-uint32_t mlx5_flow_id_release(struct mlx5_flow_id_pool *pool,
-			      uint32_t id);
 __extension__
 struct flow_grp_info {
 	uint64_t external:1;
drivers/net/mlx5/mlx5_flow_dv.c +2 -2
index f4f8e15..bfcba45 100644
@@ -8132,8 +8132,8 @@ struct field_modify_info modify_tcp[] = {
 				mlx5_hlist_remove(tunnel_grp_hash, he);
 				mlx5_free(tte);
 			}
-			mlx5_flow_id_release(mlx5_tunnel_hub(dev)->table_ids,
-					     tunnel_flow_tbl_to_id(table_id));
+			mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+					tunnel_flow_tbl_to_id(table_id));
 			DRV_LOG(DEBUG,
 				"port %u release table_id %#x tunnel %u group %u",
 				dev->data->port_id, table_id,
drivers/net/mlx5/mlx5_utils.c +5 -0
index 3a1f87a..7a6b0c6 100644
@@ -362,6 +362,11 @@ struct mlx5_indexed_pool *
 	MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx));
 	rte_bitmap_clear(trunk->bmp, iidx);
 	p = &trunk->data[iidx * pool->cfg.size];
+	/*
+	 * The ipool index should grow continually from small to big,
+	 * some features as metering only accept limited bits of index.
+	 * Random index with MSB set may be rejected.
+	 */
 	iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx);
 	iidx += 1; /* non-zero index. */
 	trunk->free--;
Signature · 1 line
1.8.3.1
#164 lore raw

PATCH v5 6/34 net/mlx5: make rte flow list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow operations, this patch introduces list lock for the rte_flow list manages all the rte_flow handlers.

---
 drivers/net/mlx5/linux/mlx5_os.c |  1 +
 drivers/net/mlx5/mlx5.h          |  1 +
 drivers/net/mlx5/mlx5_flow.c     | 10 ++++++++--
 3 files changed, 10 insertions(+), 2 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -0
index 0b59e74..a579dde 100644
@@ -1358,6 +1358,7 @@
 				      MLX5_MAX_MAC_ADDRESSES);
 	priv->flows = 0;
 	priv->ctrl_flows = 0;
+	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
 	TAILQ_INIT(&priv->flow_meter_profiles);
 	/* Hint libmlx5 to use PMD allocator for data plane resources */
drivers/net/mlx5/mlx5.h +1 -0
index 5bda233..9ab2976 100644
@@ -856,6 +856,7 @@ struct mlx5_priv {
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
 	uint32_t flows; /* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
+	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
drivers/net/mlx5/mlx5_flow.c +8 -2
index ed2acd1..cc31801 100644
@@ -5774,9 +5774,12 @@ struct tunnel_default_miss_ctx {
 		if (ret < 0)
 			goto error;
 	}
-	if (list)
+	if (list) {
+		rte_spinlock_lock(&priv->flow_list_lock);
 		ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list, idx,
 			     flow, next);
+		rte_spinlock_unlock(&priv->flow_list_lock);
+	}
 	flow_rxq_flags_set(dev, flow);
 	rte_free(translated_actions);
 	/* Nested flow creation index recovery. */
@@ -5957,9 +5960,12 @@ struct rte_flow *
 	if (dev->data->dev_started)
 		flow_rxq_flags_trim(dev, flow);
 	flow_drv_destroy(dev, flow);
-	if (list)
+	if (list) {
+		rte_spinlock_lock(&priv->flow_list_lock);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], list,
 			     flow_idx, flow, next);
+		rte_spinlock_unlock(&priv->flow_list_lock);
+	}
 	flow_mreg_del_copy_action(dev, flow);
 	if (flow->fdir) {
 		LIST_FOREACH(priv_fdir_flow, &priv->fdir_flows, next) {
Signature · 1 line
1.8.3.1
#165 lore raw

PATCH v5 7/34 net/mlx5: make meter action thread safe

In reply to Suanming Mou

This commit adds the spinlock for the meter action to make it be thread safe. Atomic reference counter in all is not enough as the meter action should be created synchronized with reference counter increment. With only atomic reference counter, even the counter is increased, the action may still not be created.

---
 drivers/net/mlx5/mlx5_flow.h       |  2 ++
 drivers/net/mlx5/mlx5_flow_meter.c | 72 ++++++++++++++++++++------------------
 2 files changed, 39 insertions(+), 35 deletions(-)
drivers/net/mlx5/mlx5_flow.h +2 -0
index 629cdbe..fde5007 100644
@@ -845,6 +845,8 @@ struct mlx5_flow_meter {
 	struct mlx5_flow_meter_profile *profile;
 	/**< Meter profile parameters. */
 
+	rte_spinlock_t sl; /**< Meter action spinlock. */
+
 	/** Policer actions (per meter output color). */
 	enum rte_mtr_policer_action action[RTE_COLORS];
 
drivers/net/mlx5/mlx5_flow_meter.c +37 -35

107 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#166 lore raw

PATCH v5 8/34 net/mlx5: make VLAN network interface thread safe

In reply to Suanming Mou

This commit protects the VLAN VM workaround area using a spinlock in multiple-thread flow insertion to make it thread safe.

---
 drivers/common/mlx5/linux/mlx5_nl.h   | 1 +
 drivers/net/mlx5/linux/mlx5_vlan_os.c | 5 +++++
 2 files changed, 6 insertions(+)
drivers/common/mlx5/linux/mlx5_nl.h +1 -0
index 53021e1..15129ff 100644
@@ -25,6 +25,7 @@ struct mlx5_nl_vlan_dev {
 struct mlx5_nl_vlan_vmwa_context {
 	int nl_socket;
 	uint32_t vf_ifindex;
+	rte_spinlock_t sl;
 	struct mlx5_nl_vlan_dev vlan_dev[4096];
 };
 
drivers/net/mlx5/linux/mlx5_vlan_os.c +5 -0
index 92fc17d..40e895e 100644
@@ -44,12 +44,14 @@
 	if (!vlan->created || !vmwa)
 		return;
 	vlan->created = 0;
+	rte_spinlock_lock(&vmwa->sl);
 	MLX5_ASSERT(vlan_dev[vlan->tag].refcnt);
 	if (--vlan_dev[vlan->tag].refcnt == 0 &&
 	    vlan_dev[vlan->tag].ifindex) {
 		mlx5_nl_vlan_vmwa_delete(vmwa, vlan_dev[vlan->tag].ifindex);
 		vlan_dev[vlan->tag].ifindex = 0;
 	}
+	rte_spinlock_unlock(&vmwa->sl);
 }
 
 /**
@@ -72,6 +74,7 @@
 	MLX5_ASSERT(priv->vmwa_context);
 	if (vlan->created || !vmwa)
 		return;
+	rte_spinlock_lock(&vmwa->sl);
 	if (vlan_dev[vlan->tag].refcnt == 0) {
 		MLX5_ASSERT(!vlan_dev[vlan->tag].ifindex);
 		vlan_dev[vlan->tag].ifindex =
@@ -82,6 +85,7 @@
 		vlan_dev[vlan->tag].refcnt++;
 		vlan->created = 1;
 	}
+	rte_spinlock_unlock(&vmwa->sl);
 }
 
 /*
@@ -131,6 +135,7 @@
 			" for VLAN workaround context");
 		return NULL;
 	}
+	rte_spinlock_init(&vmwa->sl);
 	vmwa->nl_socket = mlx5_nl_init(NETLINK_ROUTE);
 	if (vmwa->nl_socket < 0) {
 		DRV_LOG(WARNING,
Signature · 1 line
1.8.3.1
#167 lore raw

PATCH v5 9/34 net/mlx5: create global jump action

In reply to Suanming Mou

This commit changes the jump action in table to be created with table creation in advanced. In this case, the jump action is safe to be used in multiple thread. The jump action will be destroyed when table is not used anymore and released.

---
 drivers/net/mlx5/mlx5_flow.h    |  2 --
 drivers/net/mlx5/mlx5_flow_dv.c | 56 +++++++++++++----------------------------
 2 files changed, 17 insertions(+), 41 deletions(-)
drivers/net/mlx5/mlx5_flow.h +0 -2
index fde5007..cc64792 100644
@@ -484,8 +484,6 @@ struct mlx5_flow_dv_modify_hdr_resource {
 
 /* Jump action resource structure. */
 struct mlx5_flow_dv_jump_tbl_resource {
-	uint32_t refcnt; /**< Reference counter. */
-	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	void *action; /**< Pointer to the rdma core action. */
 };
 
drivers/net/mlx5/mlx5_flow_dv.c +17 -39

85 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#168 lore raw

PATCH v5 10/34 net/mlx5: create global default miss action

In reply to Suanming Mou

This commit creates the global default miss action instead of maintain it in flow insertion time. This makes the action to be thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c |  7 ++++
 drivers/net/mlx5/mlx5.h          |  9 +---
 drivers/net/mlx5/mlx5_flow_dv.c  | 90 +++-------------------------------------
 3 files changed, 13 insertions(+), 93 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index a579dde..ae735a3 100644
@@ -305,6 +305,10 @@
 	}
 	sh->pop_vlan_action = mlx5_glue->dr_create_flow_action_pop_vlan();
 #endif /* HAVE_MLX5DV_DR */
+	sh->default_miss_action =
+			mlx5_glue->dr_create_flow_action_default_miss();
+	if (!sh->default_miss_action)
+		DRV_LOG(WARNING, "Default miss action is not supported.");
 	return 0;
 error:
 	/* Rollback the created objects. */
@@ -388,6 +392,9 @@
 	}
 	pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
+	if (sh->default_miss_action)
+		mlx5_glue->destroy_flow_action
+				(sh->default_miss_action);
 	if (sh->encaps_decaps) {
 		mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
 		sh->encaps_decaps = NULL;
drivers/net/mlx5/mlx5.h +1 -8
index 9ab2976..3ce890a 100644
@@ -461,12 +461,6 @@ struct mlx5_flow_counter_mng {
 	LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;
 };
 
-/* Default miss action resource structure. */
-struct mlx5_flow_default_miss_resource {
-	void *action; /* Pointer to the rdma-core action. */
-	uint32_t refcnt; /* Default miss action reference counter. */
-};
-
 #define MLX5_AGE_EVENT_NEW		1
 #define MLX5_AGE_TRIGGER		2
 #define MLX5_AGE_SET(age_info, BIT) \
@@ -660,8 +654,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
-	struct mlx5_flow_default_miss_resource default_miss;
-	/* Default miss action resource structure. */
+	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -85

136 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#169 lore raw

PATCH v5 11/34 net/mlx5: create global drop action

In reply to Suanming Mou

This commit creates the global drop action for flows instead of maintain it in flow insertion time. The uniqueu global drop action makes it thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  5 +++++
 drivers/net/mlx5/mlx5.c            |  2 ++
 drivers/net/mlx5/mlx5_flow_dv.c    | 38 +++++++----------------------------
 drivers/net/mlx5/mlx5_flow_verbs.c | 41 +++++++++++---------------------------
 4 files changed, 26 insertions(+), 60 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -0
index ae735a3..b12d1d5 100644
@@ -1424,6 +1424,9 @@
 	} else {
 		priv->obj_ops = ibv_obj_ops;
 	}
+	priv->drop_queue.hrxq = mlx5_drop_action_create(eth_dev);
+	if (!priv->drop_queue.hrxq)
+		goto error;
 	/* Supported Verbs flow priority number detection. */
 	err = mlx5_flow_discover_priorities(eth_dev);
 	if (err < 0) {
@@ -1489,6 +1492,8 @@
 			close(priv->nl_socket_rdma);
 		if (priv->vmwa_context)
 			mlx5_vlan_vmwa_exit(priv->vmwa_context);
+		if (eth_dev && priv->drop_queue.hrxq)
+			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
 		mlx5_free(priv);
drivers/net/mlx5/mlx5.c +2 -0
index fa00e45..2ebc179 100644
@@ -1308,6 +1308,8 @@ struct mlx5_dev_ctx_shared *
 		priv->txqs = NULL;
 	}
 	mlx5_proc_priv_uninit(dev);
+	if (priv->drop_queue.hrxq)
+		mlx5_drop_action_destroy(dev);
 	if (priv->mreg_cp_tbl)
 		mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
 	mlx5_mprq_free_mp(dev);
drivers/net/mlx5/mlx5_flow_dv.c +7 -31
index 34f5608..4322c59 100644
@@ -10319,24 +10319,9 @@ struct field_modify_info modify_tcp[] = {
 			if (dv->transfer) {
 				dv->actions[n++] = priv->sh->esw_drop_action;
 			} else {
-				struct mlx5_hrxq *drop_hrxq;
-				drop_hrxq = mlx5_drop_action_create(dev);
-				if (!drop_hrxq) {
-					rte_flow_error_set
-						(error, errno,
-						 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-						 NULL,
-						 "cannot get drop hash queue");
-					goto error;
-				}
-				/*
-				 * Drop queues will be released by the specify
-				 * mlx5_drop_action_destroy() function. Assign
-				 * the special index to hrxq to mark the queue
-				 * has been allocated.
-				 */
-				dh->rix_hrxq = UINT32_MAX;
-				dv->actions[n++] = drop_hrxq->action;
+				MLX5_ASSERT(priv->drop_queue.hrxq);
+				dv->actions[n++] =
+						priv->drop_queue.hrxq->action;
 			}
 		} else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE &&
 			   !dv_h->rix_sample && !dv_h->rix_dest_array) {
@@ -10390,14 +10375,9 @@ struct field_modify_info modify_tcp[] = {
 	SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
 		       handle_idx, dh, next) {
 		/* hrxq is union, don't clear it if the flag is not set. */
-		if (dh->rix_hrxq) {
-			if (dh->fate_action == MLX5_FLOW_FATE_DROP) {
-				mlx5_drop_action_destroy(dev);
-				dh->rix_hrxq = 0;
-			} else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
-				mlx5_hrxq_release(dev, dh->rix_hrxq);
-				dh->rix_hrxq = 0;
-			}
+		if (dh->fate_action == MLX5_FLOW_FATE_QUEUE && dh->rix_hrxq) {
+			mlx5_hrxq_release(dev, dh->rix_hrxq);
+			dh->rix_hrxq = 0;
 		}
 		if (dh->vf_vlan.tag && dh->vf_vlan.created)
 			mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
@@ -10643,9 +10623,6 @@ struct field_modify_info modify_tcp[] = {
 	if (!handle->rix_fate)
 		return;
 	switch (handle->fate_action) {
-	case MLX5_FLOW_FATE_DROP:
-		mlx5_drop_action_destroy(dev);
-		break;
 	case MLX5_FLOW_FATE_QUEUE:
 		mlx5_hrxq_release(dev, handle->rix_hrxq);
 		break;
@@ -10825,8 +10802,7 @@ struct field_modify_info modify_tcp[] = {
 			claim_zero(mlx5_flow_os_destroy_flow(dh->drv_flow));
 			dh->drv_flow = NULL;
 		}
-		if (dh->fate_action == MLX5_FLOW_FATE_DROP ||
-		    dh->fate_action == MLX5_FLOW_FATE_QUEUE)
+		if (dh->fate_action == MLX5_FLOW_FATE_QUEUE)
 			flow_dv_fate_resource_release(dev, dh);
 		if (dh->vf_vlan.tag && dh->vf_vlan.created)
 			mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
drivers/net/mlx5/mlx5_flow_verbs.c +12 -29

75 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#170 lore raw

PATCH v5 12/34 net/mlx5: support concurrent access for hash list

In reply to Suanming Mou

From: Xueming Li <[email protected]>

In order to support hash list concurrent access, adding next:
1. List level read/write lock.
2. Entry reference counter.
3. Entry create/match/remove callback.
4. Remove insert/lookup/remove function which are not thread safe.
5. Add register/unregister function to support entry reuse.

For better performance, lookup function uses read lock to allow concurrent lookup from different thread, all other hash list modification functions uses write lock which blocks concurrent modification and lookups from other thread.

The exact objects change will be applied in the next patches.

---
 drivers/net/mlx5/linux/mlx5_os.c |  27 ++++---
 drivers/net/mlx5/mlx5.c          |  13 ++--
 drivers/net/mlx5/mlx5_flow.c     |  23 +++---
 drivers/net/mlx5/mlx5_flow_dv.c  |   9 ++-
 drivers/net/mlx5/mlx5_utils.c    | 154 ++++++++++++++++++++++++++++++++-------
 drivers/net/mlx5/mlx5_utils.h    | 149 ++++++++++++++++++++++++++++++-------
 6 files changed, 287 insertions(+), 88 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -11
index b12d1d5..4db5d33 100644
@@ -236,14 +236,16 @@
 		return err;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE);
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
+					  0, NULL, NULL, NULL);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ);
+	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
+					    0, 0, NULL, NULL, NULL);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
@@ -251,7 +253,8 @@
 	}
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ);
+					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
+					      0, 0, NULL, NULL, NULL);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
@@ -333,16 +336,16 @@
 		sh->pop_vlan_action = NULL;
 	}
 	if (sh->encaps_decaps) {
-		mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
+		mlx5_hlist_destroy(sh->encaps_decaps);
 		sh->encaps_decaps = NULL;
 	}
 	if (sh->modify_cmds) {
-		mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
+		mlx5_hlist_destroy(sh->modify_cmds);
 		sh->modify_cmds = NULL;
 	}
 	if (sh->tag_table) {
 		/* tags should be destroyed with flow before. */
-		mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
+		mlx5_hlist_destroy(sh->tag_table);
 		sh->tag_table = NULL;
 	}
 	if (sh->tunnel_hub) {
@@ -396,16 +399,16 @@
 		mlx5_glue->destroy_flow_action
 				(sh->default_miss_action);
 	if (sh->encaps_decaps) {
-		mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
+		mlx5_hlist_destroy(sh->encaps_decaps);
 		sh->encaps_decaps = NULL;
 	}
 	if (sh->modify_cmds) {
-		mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
+		mlx5_hlist_destroy(sh->modify_cmds);
 		sh->modify_cmds = NULL;
 	}
 	if (sh->tag_table) {
 		/* tags should be destroyed with flow before. */
-		mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
+		mlx5_hlist_destroy(sh->tag_table);
 		sh->tag_table = NULL;
 	}
 	if (sh->tunnel_hub) {
@@ -1472,7 +1475,9 @@
 	    mlx5_flow_ext_mreg_supported(eth_dev) &&
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
-						      MLX5_FLOW_MREG_HTABLE_SZ);
+						      MLX5_FLOW_MREG_HTABLE_SZ,
+						      0, 0,
+						      NULL, NULL, NULL);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
@@ -1483,7 +1488,7 @@
 error:
 	if (priv) {
 		if (priv->mreg_cp_tbl)
-			mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
+			mlx5_hlist_destroy(priv->mreg_cp_tbl);
 		if (priv->sh)
 			mlx5_os_free_shared_dr(priv);
 		if (priv->nl_socket_route >= 0)
drivers/net/mlx5/mlx5.c +7 -6
index 2ebc179..c74b55e 100644
@@ -1034,7 +1034,7 @@ struct mlx5_dev_ctx_shared *
 
 	if (!sh->flow_tbls)
 		return;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	if (pos) {
 		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 					entry);
@@ -1043,7 +1043,7 @@ struct mlx5_dev_ctx_shared *
 		mlx5_free(tbl_data);
 	}
 	table_key.direction = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	if (pos) {
 		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 					entry);
@@ -1053,7 +1053,7 @@ struct mlx5_dev_ctx_shared *
 	}
 	table_key.direction = 0;
 	table_key.domain = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	if (pos) {
 		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
 					entry);
@@ -1061,7 +1061,7 @@ struct mlx5_dev_ctx_shared *
 		mlx5_hlist_remove(sh->flow_tbls, pos);
 		mlx5_free(tbl_data);
 	}
-	mlx5_hlist_destroy(sh->flow_tbls, NULL, NULL);
+	mlx5_hlist_destroy(sh->flow_tbls);
 }
 
 /**
@@ -1083,7 +1083,8 @@ struct mlx5_dev_ctx_shared *
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
-	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE);
+	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
+					  0, 0, NULL, NULL, NULL);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
@@ -1311,7 +1312,7 @@ struct mlx5_dev_ctx_shared *
 	if (priv->drop_queue.hrxq)
 		mlx5_drop_action_destroy(dev);
 	if (priv->mreg_cp_tbl)
-		mlx5_hlist_destroy(priv->mreg_cp_tbl, NULL, NULL);
+		mlx5_hlist_destroy(priv->mreg_cp_tbl);
 	mlx5_mprq_free_mp(dev);
 	mlx5_os_free_shared_dr(priv);
 	if (priv->rss_conf.rss_key != NULL)
drivers/net/mlx5/mlx5_flow.c +12 -11
index cc31801..29ff033 100644
@@ -3747,7 +3747,7 @@ struct rte_flow_shared_action *
 	cp_mreg.src = ret;
 	/* Check if already registered. */
 	MLX5_ASSERT(priv->mreg_cp_tbl);
-	mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
+	mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id, NULL);
 	if (mcp_res) {
 		/* For non-default rule. */
 		if (mark_id != MLX5_DEFAULT_COPY_ID)
@@ -3824,8 +3824,7 @@ struct rte_flow_shared_action *
 		goto error;
 	mcp_res->refcnt++;
 	mcp_res->hlist_ent.key = mark_id;
-	ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
-				&mcp_res->hlist_ent);
+	ret = !mlx5_hlist_insert(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
 	MLX5_ASSERT(!ret);
 	if (ret)
 		goto error;
@@ -3975,7 +3974,7 @@ struct rte_flow_shared_action *
 	if (!priv->mreg_cp_tbl)
 		return;
 	mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
-					    MLX5_DEFAULT_COPY_ID);
+					    MLX5_DEFAULT_COPY_ID, NULL);
 	if (!mcp_res)
 		return;
 	MLX5_ASSERT(mcp_res->rix_flow);
@@ -7562,7 +7561,7 @@ struct mlx5_meter_domains_infos *
 			.direction = 0,
 		}
 	};
-	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
+	he = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
 	return he ?
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
@@ -7584,7 +7583,7 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_hlist *group_hash;
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_lookup(group_hash, key.val);
+	he = mlx5_hlist_lookup(group_hash, key.val, NULL);
 	if (!he) {
 		tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
 				  sizeof(*tte), 0,
@@ -8091,7 +8090,7 @@ struct mlx5_meter_domains_infos *
 	LIST_REMOVE(tunnel, chain);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID],
 			tunnel->tunnel_id);
-	mlx5_hlist_destroy(tunnel->groups, NULL, NULL);
+	mlx5_hlist_destroy(tunnel->groups);
 	mlx5_free(tunnel);
 }
 
@@ -8139,7 +8138,8 @@ struct mlx5_meter_domains_infos *
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024);
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
+					   NULL, NULL, NULL);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(priv->sh->ipool
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
@@ -8204,7 +8204,7 @@ void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id)
 		return;
 	if (!LIST_EMPTY(&thub->tunnels))
 		DRV_LOG(WARNING, "port %u tunnels present\n", port_id);
-	mlx5_hlist_destroy(thub->groups, NULL, NULL);
+	mlx5_hlist_destroy(thub->groups);
 	mlx5_free(thub);
 }
 
@@ -8218,7 +8218,8 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	if (!thub)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
-	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES);
+	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
+					 0, NULL, NULL, NULL);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
@@ -8229,7 +8230,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 
 err:
 	if (thub->groups)
-		mlx5_hlist_destroy(thub->groups, NULL, NULL);
+		mlx5_hlist_destroy(thub->groups);
 	if (thub)
 		mlx5_free(thub);
 	return err;
drivers/net/mlx5/mlx5_flow_dv.c +5 -4
index 4322c59..13bc46d 100644
@@ -7961,7 +7961,7 @@ struct field_modify_info modify_tcp[] = {
 		}
 	};
 	struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
-							 table_key.v64);
+							 table_key.v64, NULL);
 	struct mlx5_flow_tbl_data_entry *tbl_data;
 	uint32_t idx = 0;
 	int ret;
@@ -8022,7 +8022,7 @@ struct field_modify_info modify_tcp[] = {
 		}
 	}
 	pos->key = table_key.v64;
-	ret = mlx5_hlist_insert(sh->flow_tbls, pos);
+	ret = !mlx5_hlist_insert(sh->flow_tbls, pos);
 	if (ret < 0) {
 		rte_flow_error_set(error, -ret,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -8079,7 +8079,8 @@ struct field_modify_info modify_tcp[] = {
 			tunnel_grp_hash = tbl_data->tunnel ?
 						tbl_data->tunnel->groups :
 						thub->groups;
-			he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val);
+			he = mlx5_hlist_lookup(tunnel_grp_hash,
+					       tunnel_key.val, NULL);
 			if (he) {
 				struct tunnel_tbl_entry *tte;
 				tte = container_of(he, typeof(*tte), hash);
@@ -8238,7 +8239,7 @@ struct field_modify_info modify_tcp[] = {
 	int ret;
 
 	/* Lookup a matching resource from cache. */
-	entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24);
+	entry = mlx5_hlist_lookup(sh->tag_table, (uint64_t)tag_be24, NULL);
 	if (entry) {
 		cache_resource = container_of
 			(entry, struct mlx5_flow_dv_tag_resource, entry);
drivers/net/mlx5/mlx5_utils.c +126 -28

226 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +121 -28

229 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#171 lore raw

PATCH v5 13/34 net/mlx5: add flow table tunnel offload attribute

In reply to Suanming Mou

As flow table is shared between the ports in the same shared IB device, flow table may be created by one port and released by other port.

Currently, the tunnel offloading active check in flow table release is based on the port which release the flow table. Since the flow table create port and release port may have different tunnel offloading configuration, it will cause invalid tunnel offloading release or tunnel offloading resource leaks.

Add the flow table tunnel offloading attribute to indicate the flow table has tunnel offloading resource or not to avoid wrong tunnel offloading operation.

---
 drivers/net/mlx5/mlx5_flow.h    | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)
drivers/net/mlx5/mlx5_flow.h +1 -0
index cc64792..62c5e7c 100644
@@ -538,6 +538,7 @@ struct mlx5_flow_tbl_data_entry {
 	const struct mlx5_flow_tunnel *tunnel;
 	uint32_t group_id;
 	bool external;
+	bool tunnel_offload; /* Tunnel offlod table or not. */
 };
 
 /* Sub rdma-core actions list. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index 13bc46d..0b4b7ff 100644
@@ -7986,6 +7986,7 @@ struct field_modify_info modify_tcp[] = {
 	tbl_data->tunnel = tunnel;
 	tbl_data->group_id = group_id;
 	tbl_data->external = external;
+	tbl_data->tunnel_offload = is_tunnel_offload_active(dev);
 	tbl = &tbl_data->tbl;
 	pos = &tbl_data->entry;
 	if (transfer)
@@ -8061,7 +8062,7 @@ struct field_modify_info modify_tcp[] = {
 
 		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
 		tbl->obj = NULL;
-		if (is_tunnel_offload_active(dev) && tbl_data->external) {
+		if (tbl_data->tunnel_offload && tbl_data->external) {
 			struct mlx5_hlist_entry *he;
 			struct mlx5_hlist *tunnel_grp_hash;
 			struct mlx5_flow_tunnel_hub *thub =
Signature · 1 line
1.8.3.1
#172 lore raw

PATCH v5 14/34 net/mlx5: make flow table cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion/removal, this patch uses thread safe hash list API for flow table cache hash list.

---
 drivers/net/mlx5/mlx5.c         |  99 +++------------
 drivers/net/mlx5/mlx5.h         |   2 +-
 drivers/net/mlx5/mlx5_flow.c    |   2 +-
 drivers/net/mlx5/mlx5_flow.h    |  25 ++++
 drivers/net/mlx5/mlx5_flow_dv.c | 265 +++++++++++++++++++++-------------------
 5 files changed, 181 insertions(+), 212 deletions(-)
drivers/net/mlx5/mlx5.c +15 -84
index c74b55e..1d25a8e 100644
@@ -1012,7 +1012,7 @@ struct mlx5_dev_ctx_shared *
 }
 
 /**
- * Destroy table hash list and all the root entries per domain.
+ * Destroy table hash list.
  *
  * @param[in] priv
  *   Pointer to the private device data structure.
@@ -1021,46 +1021,9 @@ struct mlx5_dev_ctx_shared *
 mlx5_free_table_hash_list(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_flow_tbl_data_entry *tbl_data;
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = 0,
-			.reserved = 0,
-			.domain = 0,
-			.direction = 0,
-		}
-	};
-	struct mlx5_hlist_entry *pos;
 
 	if (!sh->flow_tbls)
 		return;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
-	table_key.direction = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
-	table_key.direction = 0;
-	table_key.domain = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
 	mlx5_hlist_destroy(sh->flow_tbls);
 }
 
@@ -1075,77 +1038,45 @@ struct mlx5_dev_ctx_shared *
  *   Zero on success, positive error code otherwise.
  */
 int
-mlx5_alloc_table_hash_list(struct mlx5_priv *priv)
+mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 {
+	int err = 0;
+	/* Tables are only used in DV and DR modes. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	char s[MLX5_HLIST_NAMESIZE];
-	int err = 0;
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, NULL, NULL, NULL);
+					  0, 0, flow_dv_tbl_create_cb, NULL,
+					  flow_dv_tbl_remove_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
+	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
+	struct rte_flow_error error;
+	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
+
 	/*
 	 * In case we have not DR support, the zero tables should be created
 	 * because DV expect to see them even if they cannot be created by
 	 * RDMA-CORE.
 	 */
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = 0,
-			.reserved = 0,
-			.domain = 0,
-			.direction = 0,
-		}
-	};
-	struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO,
-							  sizeof(*tbl_data), 0,
-							  SOCKET_ID_ANY);
-
-	if (!tbl_data) {
-		err = ENOMEM;
-		goto error;
-	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	__atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED);
-	table_key.direction = 1;
-	tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
-			       SOCKET_ID_ANY);
-	if (!tbl_data) {
+	if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL, 0, 1, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 1, 0, 0, NULL, 0, 1, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 0, 1, 0, NULL, 0, 1, &error)) {
 		err = ENOMEM;
 		goto error;
 	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	__atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED);
-	table_key.direction = 0;
-	table_key.domain = 1;
-	tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
-			       SOCKET_ID_ANY);
-	if (!tbl_data) {
-		err = ENOMEM;
-		goto error;
-	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	__atomic_store_n(&tbl_data->tbl.refcnt, 1, __ATOMIC_RELAXED);
 	return err;
 error:
 	mlx5_free_table_hash_list(priv);
 #endif /* HAVE_MLX5DV_DR */
+#endif
 	return err;
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index 3ce890a..81dba2f 100644
@@ -499,7 +499,7 @@ struct mlx5_dev_shared_port {
 	struct {
 		/* Table ID should be at the lowest address. */
 		uint32_t table_id;	/**< ID of the table. */
-		uint16_t reserved;	/**< must be zero for comparison. */
+		uint16_t dummy;		/**< Dummy table for DV API. */
 		uint8_t domain;		/**< 1 - FDB, 0 - NIC TX/RX. */
 		uint8_t direction;	/**< 1 - egress, 0 - ingress. */
 	};
drivers/net/mlx5/mlx5_flow.c +1 -1
index 29ff033..b5b4997 100644
@@ -7556,7 +7556,7 @@ struct mlx5_meter_domains_infos *
 	union mlx5_flow_tbl_key table_key = {
 		{
 			.table_id = tunnel_id_to_flow_tbl(mbits.table_id),
-			.reserved = 0,
+			.dummy = 0,
 			.domain = !!mbits.transfer,
 			.direction = 0,
 		}
drivers/net/mlx5/mlx5_flow.h +25 -0
index 62c5e7c..dae8211 100644
@@ -384,6 +384,13 @@ enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_MAX,
 };
 
+/* Hash list callback context */
+struct mlx5_flow_cb_ctx {
+	struct rte_eth_dev *dev;
+	struct rte_flow_error *error;
+	void *data;
+};
+
 /* Matcher PRM representation */
 struct mlx5_flow_dv_match_params {
 	size_t size;
@@ -523,6 +530,13 @@ struct mlx5_flow_mreg_copy_resource {
 	uint32_t rix_flow; /* Built flow for copy. */
 };
 
+/* Table tunnel parameter. */
+struct mlx5_flow_tbl_tunnel_prm {
+	const struct mlx5_flow_tunnel *tunnel;
+	uint32_t group_id;
+	bool external;
+};
+
 /* Table data structure of the hash organization. */
 struct mlx5_flow_tbl_data_entry {
 	struct mlx5_hlist_entry entry;
@@ -1401,4 +1415,15 @@ int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
 int mlx5_shared_action_flush(struct rte_eth_dev *dev);
 void mlx5_release_tunnel_hub(struct mlx5_dev_ctx_shared *sh, uint16_t port_id);
 int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh);
+
+/* Hash list callbacks for flow tables: */
+struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
+					       uint64_t key, void *entry_ctx);
+void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
+			   struct mlx5_hlist_entry *entry);
+struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
+		uint32_t table_id, uint8_t egress, uint8_t transfer,
+		bool external, const struct mlx5_flow_tunnel *tunnel,
+		uint32_t group_id, uint8_t dummy, struct rte_flow_error *error);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +139 -126

352 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#173 lore raw

PATCH v5 15/34 net/mlx5: fix redundant Direct Verbs resources allocate

In reply to Suanming Mou

All table, tag, header modify, header reformat are supported only on DV mode. For the OFED version doesn't support these, create the related redundant DV resources waste the memory.

Add the code section in the HAVE_IBV_FLOW_DV_SUPPORT macro to avoid the redundant resources allocation.

Fixes 2eb4d0107acc ("net/mlx5: refactor PCI probing on Linux")
---
 drivers/net/mlx5/linux/mlx5_os.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -2
index 4db5d33..db32b39 100644
@@ -225,7 +225,7 @@
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -233,7 +233,9 @@
 		return 0;
 	err = mlx5_alloc_table_hash_list(priv);
 	if (err)
-		return err;
+		goto error;
+	/* The resources below are only valid with DV support. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -260,6 +262,7 @@
 		err = ENOMEM;
 		goto error;
 	}
+#endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
 
Signature · 1 line
1.8.3.1
#174 lore raw

PATCH v5 16/34 net/mlx5: make flow tag list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch updates flow tag list to use thread safe hash list with write-most mode.

---
 drivers/net/mlx5/linux/mlx5_os.c |  5 +-
 drivers/net/mlx5/mlx5_flow.h     |  5 ++
 drivers/net/mlx5/mlx5_flow_dv.c  | 98 +++++++++++++++++++---------------------
 3 files changed, 56 insertions(+), 52 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +4 -1
index db32b39..540e1a7 100644
@@ -239,12 +239,15 @@
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  0, NULL, NULL, NULL);
+					  MLX5_HLIST_WRITE_MOST,
+					  flow_dv_tag_create_cb, NULL,
+					  flow_dv_tag_remove_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
+	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
 					    0, 0, NULL, NULL, NULL);
drivers/net/mlx5/mlx5_flow.h +5 -0
index dae8211..227504d 100644
@@ -1426,4 +1426,9 @@ struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
 		bool external, const struct mlx5_flow_tunnel *tunnel,
 		uint32_t group_id, uint8_t dummy, struct rte_flow_error *error);
 
+struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
+					       uint64_t key, void *cb_ctx);
+void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
+			   struct mlx5_hlist_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +47 -51

132 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#175 lore raw

PATCH v5 17/34 net/mlx5: make flow modify action list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch updates flow modify action list to use thread safe hash list with write-most mode.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 +-
 drivers/net/mlx5/mlx5_flow.h     |  14 ++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 196 +++++++++++++++++----------------------
 3 files changed, 102 insertions(+), 115 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +6 -1
index 540e1a7..eb63bcd 100644
@@ -250,12 +250,17 @@
 	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, 0, NULL, NULL, NULL);
+					    0, MLX5_HLIST_WRITE_MOST |
+					    MLX5_HLIST_DIRECT_KEY,
+					    flow_dv_modify_create_cb,
+					    flow_dv_modify_match_cb,
+					    flow_dv_modify_remove_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
+	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
drivers/net/mlx5/mlx5_flow.h +10 -4
index 227504d..4fb50de 100644
@@ -467,10 +467,8 @@ struct mlx5_flow_dv_tag_resource {
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_hlist_entry entry;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *action;
-	/**< Modify header action object. */
+	void *action; /**< Modify header action object. */
+	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint32_t actions_num; /**< Number of modification actions. */
 	uint64_t flags; /**< Flags for RDMA API. */
@@ -1431,4 +1429,12 @@ struct mlx5_hlist_entry *flow_dv_tag_create_cb(struct mlx5_hlist *list,
 void flow_dv_tag_remove_cb(struct mlx5_hlist *list,
 			   struct mlx5_hlist_entry *entry);
 
+int flow_dv_modify_match_cb(struct mlx5_hlist *list,
+			    struct mlx5_hlist_entry *entry,
+			    uint64_t key, void *cb_ctx);
+struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
+						  uint64_t key, void *ctx);
+void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
+			      struct mlx5_hlist_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +86 -110

238 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#176 lore raw

PATCH v5 18/34 net/mlx5: remove unused mreg copy code

In reply to Suanming Mou

After non-cache mode feature was implemented, the flows can only be created when port started. No need to check if the mreg flows are created in port stopped status, and apply the mreg flows after port start will also never happen.

This commit removed the relevant not used mreg copy code.

---
 drivers/net/mlx5/mlx5.h      |   2 -
 drivers/net/mlx5/mlx5_flow.c | 185 -------------------------------------------
 drivers/net/mlx5/mlx5_flow.h |   2 -
 3 files changed, 189 deletions(-)
drivers/net/mlx5/mlx5.h +0 -2
index 81dba2f..debb862 100644
@@ -1091,8 +1091,6 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
 			 enum rte_filter_type filter_type,
 			 enum rte_filter_op filter_op,
 			 void *arg);
-int mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list);
-void mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list);
 int mlx5_flow_start_default(struct rte_eth_dev *dev);
 void mlx5_flow_stop_default(struct rte_eth_dev *dev);
 int mlx5_flow_verify(struct rte_eth_dev *dev);
drivers/net/mlx5/mlx5_flow.c +0 -185

221 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +0 -2
index 4fb50de..b5e4c9f 100644
@@ -523,7 +523,6 @@ struct mlx5_flow_mreg_copy_resource {
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
 	uint32_t refcnt; /* Reference counter. */
-	uint32_t appcnt; /* Apply/Remove counter. */
 	uint32_t idx;
 	uint32_t rix_flow; /* Built flow for copy. */
 };
@@ -1048,7 +1047,6 @@ struct rte_flow {
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t fdir:1; /**< Identifier of associated FDIR if any. */
 	uint32_t tunnel:1;
-	uint32_t copy_applied:1; /**< The MARK copy Flow os applied. */
 	uint32_t meter:16; /**< Holds flow meter id. */
 	uint32_t rix_mreg_copy;
 	/**< Index to metadata register copy table resource. */
Signature · 1 line
1.8.3.1
#177 lore raw

PATCH v5 19/34 net/mlx5: make metadata copy flow list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch updates metadata copy flow list to use thread safe hash list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   5 +-
 drivers/net/mlx5/mlx5_flow.c     | 162 ++++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_flow.h     |   6 +-
 3 files changed, 99 insertions(+), 74 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +4 -1
index eb63bcd..46dbc18 100644
@@ -1488,11 +1488,14 @@
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
 						      0, 0,
-						      NULL, NULL, NULL);
+						      flow_dv_mreg_create_cb,
+						      NULL,
+						      flow_dv_mreg_remove_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
+		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	mlx5_flow_counter_mode_config(eth_dev);
 	return eth_dev;
drivers/net/mlx5/mlx5_flow.c +90 -72

225 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +5 -1
index b5e4c9f..ffbef2a 100644
@@ -522,7 +522,6 @@ struct mlx5_flow_mreg_copy_resource {
 	struct mlx5_hlist_entry hlist_ent;
 	LIST_ENTRY(mlx5_flow_mreg_copy_resource) next;
 	/* List entry for device flows. */
-	uint32_t refcnt; /* Reference counter. */
 	uint32_t idx;
 	uint32_t rix_flow; /* Built flow for copy. */
 };
@@ -1435,4 +1434,9 @@ struct mlx5_hlist_entry *flow_dv_modify_create_cb(struct mlx5_hlist *list,
 void flow_dv_modify_remove_cb(struct mlx5_hlist *list,
 			      struct mlx5_hlist_entry *entry);
 
+struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
+						uint64_t key, void *ctx);
+void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
+			    struct mlx5_hlist_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
Signature · 1 line
1.8.3.1
#178 lore raw

PATCH v5 20/34 net/mlx5: make header reformat action thread safe

In reply to Suanming Mou

To support multi-thread flow insertion, this patch updates flow header reformat action list to use thread safe hash list with write-most mode.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 +-
 drivers/net/mlx5/mlx5_flow.h     |   7 ++
 drivers/net/mlx5/mlx5_flow_dv.c  | 186 ++++++++++++++++++++++-----------------
 3 files changed, 116 insertions(+), 84 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +6 -1
index 46dbc18..d017c23 100644
@@ -264,12 +264,17 @@
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, 0, NULL, NULL, NULL);
+					      0, MLX5_HLIST_DIRECT_KEY |
+					      MLX5_HLIST_WRITE_MOST,
+					      flow_dv_encap_decap_create_cb,
+					      flow_dv_encap_decap_match_cb,
+					      flow_dv_encap_decap_remove_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
+	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
drivers/net/mlx5/mlx5_flow.h +7 -0
index ffbef2a..d0a1000 100644
@@ -1439,4 +1439,11 @@ struct mlx5_hlist_entry *flow_dv_mreg_create_cb(struct mlx5_hlist *list,
 void flow_dv_mreg_remove_cb(struct mlx5_hlist *list,
 			    struct mlx5_hlist_entry *entry);
 
+int flow_dv_encap_decap_match_cb(struct mlx5_hlist *list,
+				 struct mlx5_hlist_entry *entry,
+				 uint64_t key, void *cb_ctx);
+struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
+				uint64_t key, void *cb_ctx);
+void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +103 -83

242 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#179 lore raw

PATCH v5 21/34 net/mlx5: introduce thread safe linked list cache

In reply to Suanming Mou

From: Xueming Li <[email protected]>

New API of linked list for cache:
- Optimized for small amount cache list.
- Optimized for read-most list.
- Thread safe.
- Since number of entries are limited, entries allocated by API.
- For dynamic entry size, pass 0 as entry size, then the creation callback allocate the entry.
- Since number of entries are limited, no need to use indexed pool to allocate memory. API will remove entry and free with mlx5_free.
- Search API is not supposed to be used in multi-thread.

---
 drivers/net/mlx5/mlx5_utils.c | 160 ++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h | 183 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 343 insertions(+)
drivers/net/mlx5/mlx5_utils.c +160 -0
index d041b07..df15f63 100644
@@ -255,6 +255,166 @@ struct mlx5_hlist_entry *
 	mlx5_free(h);
 }
 
+/********************* Cache list ************************/
+
+static struct mlx5_cache_entry *
+mlx5_clist_default_create_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry __rte_unused,
+			     void *ctx __rte_unused)
+{
+	return mlx5_malloc(MLX5_MEM_ZERO, list->entry_sz, 0, SOCKET_ID_ANY);
+}
+
+static void
+mlx5_clist_default_remove_cb(struct mlx5_cache_list *list __rte_unused,
+			     struct mlx5_cache_entry *entry)
+{
+	mlx5_free(entry);
+}
+
+int
+mlx5_cache_list_init(struct mlx5_cache_list *list, const char *name,
+		     uint32_t entry_size, void *ctx,
+		     mlx5_cache_create_cb cb_create,
+		     mlx5_cache_match_cb cb_match,
+		     mlx5_cache_remove_cb cb_remove)
+{
+	MLX5_ASSERT(list);
+	if (!cb_match || (!cb_create ^ !cb_remove))
+		return -1;
+	if (name)
+		snprintf(list->name, sizeof(list->name), "%s", name);
+	list->entry_sz = entry_size;
+	list->ctx = ctx;
+	list->cb_create = cb_create ? cb_create : mlx5_clist_default_create_cb;
+	list->cb_match = cb_match;
+	list->cb_remove = cb_remove ? cb_remove : mlx5_clist_default_remove_cb;
+	rte_rwlock_init(&list->lock);
+	DRV_LOG(DEBUG, "Cache list %s initialized.", list->name);
+	LIST_INIT(&list->head);
+	return 0;
+}
+
+static struct mlx5_cache_entry *
+__cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+{
+	struct mlx5_cache_entry *entry;
+
+	LIST_FOREACH(entry, &list->head, next) {
+		if (list->cb_match(list, entry, ctx))
+			continue;
+		if (reuse) {
+			__atomic_add_fetch(&entry->ref_cnt, 1,
+					   __ATOMIC_RELAXED);
+			DRV_LOG(DEBUG, "Cache list %s entry %p ref++: %u.",
+				list->name, (void *)entry, entry->ref_cnt);
+		}
+		break;
+	}
+	return entry;
+}
+
+static struct mlx5_cache_entry *
+cache_lookup(struct mlx5_cache_list *list, void *ctx, bool reuse)
+{
+	struct mlx5_cache_entry *entry;
+
+	rte_rwlock_read_lock(&list->lock);
+	entry = __cache_lookup(list, ctx, reuse);
+	rte_rwlock_read_unlock(&list->lock);
+	return entry;
+}
+
+struct mlx5_cache_entry *
+mlx5_cache_lookup(struct mlx5_cache_list *list, void *ctx)
+{
+	return cache_lookup(list, ctx, false);
+}
+
+struct mlx5_cache_entry *
+mlx5_cache_register(struct mlx5_cache_list *list, void *ctx)
+{
+	struct mlx5_cache_entry *entry;
+	uint32_t prev_gen_cnt = 0;
+
+	MLX5_ASSERT(list);
+	prev_gen_cnt = __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE);
+	/* Lookup with read lock, reuse if found. */
+	entry = cache_lookup(list, ctx, true);
+	if (entry)
+		return entry;
+	/* Not found, append with write lock - block read from other threads. */
+	rte_rwlock_write_lock(&list->lock);
+	/* If list changed by other threads before lock, search again. */
+	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
+		/* Lookup and reuse w/o read lock. */
+		entry = __cache_lookup(list, ctx, true);
+		if (entry)
+			goto done;
+	}
+	entry = list->cb_create(list, entry, ctx);
+	if (!entry) {
+		DRV_LOG(ERR, "Failed to init cache list %s entry %p.",
+			list->name, (void *)entry);
+		goto done;
+	}
+	entry->ref_cnt = 1;
+	LIST_INSERT_HEAD(&list->head, entry, next);
+	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "Cache list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+done:
+	rte_rwlock_write_unlock(&list->lock);
+	return entry;
+}
+
+int
+mlx5_cache_unregister(struct mlx5_cache_list *list,
+		      struct mlx5_cache_entry *entry)
+{
+	rte_rwlock_write_lock(&list->lock);
+	MLX5_ASSERT(entry && entry->next.le_prev);
+	DRV_LOG(DEBUG, "Cache list %s entry %p ref--: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	if (--entry->ref_cnt) {
+		rte_rwlock_write_unlock(&list->lock);
+		return 1;
+	}
+	__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
+	__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	LIST_REMOVE(entry, next);
+	list->cb_remove(list, entry);
+	rte_rwlock_write_unlock(&list->lock);
+	DRV_LOG(DEBUG, "Cache list %s entry %p removed.",
+		list->name, (void *)entry);
+	return 0;
+}
+
+void
+mlx5_cache_list_destroy(struct mlx5_cache_list *list)
+{
+	struct mlx5_cache_entry *entry;
+
+	MLX5_ASSERT(list);
+	/* no LIST_FOREACH_SAFE, using while instead */
+	while (!LIST_EMPTY(&list->head)) {
+		entry = LIST_FIRST(&list->head);
+		LIST_REMOVE(entry, next);
+		list->cb_remove(list, entry);
+		DRV_LOG(DEBUG, "Cache list %s entry %p destroyed.",
+			list->name, (void *)entry);
+	}
+	memset(list, 0, sizeof(*list));
+}
+
+uint32_t
+mlx5_cache_list_get_entry_num(struct mlx5_cache_list *list)
+{
+	MLX5_ASSERT(list);
+	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+}
+
 /********************* Indexed pool **********************/
 
 static inline void
drivers/net/mlx5/mlx5_utils.h +183 -0

189 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#180 lore raw

PATCH v5 22/34 net/mlx5: optimize shared RSS list operation

In reply to Suanming Mou

When create shared RSS hrxq, the hrxq will be created directly, no hrxq will be reused.

In this case, add the shared RSS hrxq to the queue list is redundant. And it also hurts the generic queue lookup.

This commit avoids add the shared RSS hrxq to the queue list.

---
 drivers/net/mlx5/mlx5.h      |  2 +-
 drivers/net/mlx5/mlx5_rxq.c  | 57 +++++++++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_rxtx.h |  5 ++--
 3 files changed, 39 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.h +1 -1
index debb862..9f96bd0 100644
@@ -725,7 +725,7 @@ struct mlx5_ind_table_obj {
 struct mlx5_hrxq {
 	ILIST_ENTRY(uint32_t)next; /* Index to the next element. */
 	uint32_t refcnt; /* Reference counter. */
-	uint32_t shared:1; /* This object used in shared action. */
+	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
 	union {
drivers/net/mlx5/mlx5_rxq.c +35 -22

165 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxtx.h +3 -2
index 1b5fba4..8fe0473 100644
@@ -347,12 +347,13 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev,
 						  const uint16_t *queues,
 						  uint32_t queues_n);
 int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
-			       struct mlx5_ind_table_obj *ind_tbl);
+			       struct mlx5_ind_table_obj *ind_tbl,
+			       bool standalone);
 uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev,
 		       const uint8_t *rss_key, uint32_t rss_key_len,
 		       uint64_t hash_fields,
 		       const uint16_t *queues, uint32_t queues_n,
-		       int tunnel, bool shared);
+		       int tunnel, bool standalone);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       const uint8_t *rss_key, uint32_t rss_key_len,
 		       uint64_t hash_fields,
Signature · 1 line
1.8.3.1
#181 lore raw

PATCH v5 23/34 net/mlx5: make Rx queue thread safe

In reply to Suanming Mou

This commit applies the cache linked list to Rx queue to make it thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c   |   5 +
 drivers/net/mlx5/mlx5.c            |   1 +
 drivers/net/mlx5/mlx5.h            |  28 +++-
 drivers/net/mlx5/mlx5_flow.h       |  16 ---
 drivers/net/mlx5/mlx5_flow_dv.c    |  74 ++++------
 drivers/net/mlx5/mlx5_flow_verbs.c |  21 +--
 drivers/net/mlx5/mlx5_rxq.c        | 281 ++++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_rxtx.h       |  20 +--
 8 files changed, 228 insertions(+), 218 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +5 -0
index d017c23..10fc7c5 100644
@@ -1468,6 +1468,10 @@
 			err = ENOTSUP;
 			goto error;
 	}
+	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+			     mlx5_hrxq_create_cb,
+			     mlx5_hrxq_match_cb,
+			     mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
@@ -1520,6 +1524,7 @@
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
+		mlx5_cache_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -0
index 1d25a8e..862bd40 100644
@@ -1286,6 +1286,7 @@ struct mlx5_dev_ctx_shared *
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
+	mlx5_cache_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +26 -2
index 9f96bd0..7804d3a 100644
@@ -65,6 +65,13 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* Hash and cache list callback context. */
+struct mlx5_flow_cb_ctx {
+	struct rte_eth_dev *dev;
+	struct rte_flow_error *error;
+	void *data;
+};
+
 /* Device attributes used in mlx5 PMD */
 struct mlx5_dev_attr {
 	uint64_t	device_cap_flags_ex;
@@ -684,6 +691,22 @@ struct mlx5_proc_priv {
 /* MTR list. */
 TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter);
 
+/* RSS description. */
+struct mlx5_flow_rss_desc {
+	uint32_t level;
+	uint32_t queue_num; /**< Number of entries in @p queue. */
+	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
+	uint64_t hash_fields; /* Verbs Hash fields. */
+	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
+	uint32_t key_len; /**< RSS hash key len. */
+	uint32_t tunnel; /**< Queue in tunnel. */
+	union {
+		uint16_t *queue; /**< Destination queues. */
+		const uint16_t *const_q; /**< Const pointer convert. */
+	};
+	bool standalone; /**< Queue is standalone or not. */
+};
+
 #define MLX5_PROC_PRIV(port_id) \
 	((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)
 
@@ -723,7 +746,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	ILIST_ENTRY(uint32_t)next; /* Index to the next element. */
+	struct mlx5_cache_entry entry; /* Cache entry. */
 	uint32_t refcnt; /* Reference counter. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
@@ -737,6 +760,7 @@ struct mlx5_hrxq {
 #endif
 	uint64_t hash_fields; /* Verbs Hash fields. */
 	uint32_t rss_key_len; /* Hash key length in bytes. */
+	uint32_t idx; /* Hash Rx queue index. */
 	uint8_t rss_key[]; /* Hash key. */
 };
 
@@ -853,7 +877,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	uint32_t hrxqs; /* Verbs Hash Rx queues. */
+	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +0 -16
index d0a1000..84988c6 100644
@@ -384,13 +384,6 @@ enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_MAX,
 };
 
-/* Hash list callback context */
-struct mlx5_flow_cb_ctx {
-	struct rte_eth_dev *dev;
-	struct rte_flow_error *error;
-	void *data;
-};
-
 /* Matcher PRM representation */
 struct mlx5_flow_dv_match_params {
 	size_t size;
@@ -609,15 +602,6 @@ struct ibv_spec_header {
 	uint16_t size;
 };
 
-/* RSS description. */
-struct mlx5_flow_rss_desc {
-	uint32_t level;
-	uint32_t queue_num; /**< Number of entries in @p queue. */
-	uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
-	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
-	uint16_t *queue; /**< Destination queues. */
-};
-
 /* PMD flow priority for tunnel */
 #define MLX5_TUNNEL_PRIO_GET(rss_desc) \
 	((rss_desc)->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4)
drivers/net/mlx5/mlx5_flow_dv.c +27 -47

131 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_verbs.c +6 -15
index 3c04727..28fd120 100644
@@ -1968,21 +1968,12 @@
 				&wks->rss_desc[!!wks->flow_nested_idx];
 
 			MLX5_ASSERT(rss_desc->queue_num);
-			hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
-						 MLX5_RSS_HASH_KEY_LEN,
-						 dev_flow->hash_fields,
-						 rss_desc->queue,
-						 rss_desc->queue_num);
-			if (!hrxq_idx)
-				hrxq_idx = mlx5_hrxq_new
-						(dev, rss_desc->key,
-						 MLX5_RSS_HASH_KEY_LEN,
-						 dev_flow->hash_fields,
-						 rss_desc->queue,
-						 rss_desc->queue_num,
-						 !!(handle->layers &
-						 MLX5_FLOW_LAYER_TUNNEL),
-						 false);
+			rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN;
+			rss_desc->hash_fields = dev_flow->hash_fields;
+			rss_desc->tunnel = !!(handle->layers &
+					      MLX5_FLOW_LAYER_TUNNEL);
+			rss_desc->standalone = false;
+			hrxq_idx = mlx5_hrxq_get(dev, rss_desc);
 			hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
 					      hrxq_idx);
 			if (!hrxq) {
drivers/net/mlx5/mlx5_rxq.c +152 -129

342 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxtx.h +11 -9
index 8fe0473..c361bbd 100644
@@ -349,17 +349,19 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev,
 int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
 			       struct mlx5_ind_table_obj *ind_tbl,
 			       bool standalone);
-uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev,
-		       const uint8_t *rss_key, uint32_t rss_key_len,
-		       uint64_t hash_fields,
-		       const uint16_t *queues, uint32_t queues_n,
-		       int tunnel, bool standalone);
+struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
+		       struct mlx5_cache_entry *entry,
+		       void *cb_ctx);
+void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
+			 struct mlx5_cache_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
-		       const uint8_t *rss_key, uint32_t rss_key_len,
-		       uint64_t hash_fields,
-		       const uint16_t *queues, uint32_t queues_n);
+		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
-int mlx5_hrxq_verify(struct rte_eth_dev *dev);
+uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev);
+
+
 enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx);
 const struct rte_eth_hairpin_conf *mlx5_rxq_get_hairpin_conf
 	(struct rte_eth_dev *dev, uint16_t idx);
Signature · 1 line
1.8.3.1
#182 lore raw

PATCH v5 24/34 net/mlx5: make matcher list thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this path converts matcher list to use thread safe cache list API.

---
 drivers/net/mlx5/mlx5.h         |   3 +
 drivers/net/mlx5/mlx5_flow.h    |  15 ++-
 drivers/net/mlx5/mlx5_flow_dv.c | 215 +++++++++++++++++++++-------------------
 3 files changed, 129 insertions(+), 104 deletions(-)
drivers/net/mlx5/mlx5.h +3 -0
index 7804d3a..eadb797 100644
@@ -32,6 +32,9 @@
 #include "mlx5_os.h"
 #include "mlx5_autoconf.h"
 
+
+#define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
+
 enum mlx5_ipool_index {
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
drivers/net/mlx5/mlx5_flow.h +11 -4
index 84988c6..6c0815a 100644
@@ -394,11 +394,9 @@ struct mlx5_flow_dv_match_params {
 
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
-	LIST_ENTRY(mlx5_flow_dv_matcher) next;
-	/**< Pointer to the next element. */
+	struct mlx5_cache_entry entry; /**< Pointer to the next element. */
 	struct mlx5_flow_tbl_resource *tbl;
 	/**< Pointer to the table(group) the matcher associated with. */
-	uint32_t refcnt; /**< Reference counter. */
 	void *matcher_object; /**< Pointer to DV matcher */
 	uint16_t crc; /**< CRC of key. */
 	uint16_t priority; /**< Priority of matcher. */
@@ -532,7 +530,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
+	struct mlx5_cache_list matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
@@ -542,6 +540,7 @@ struct mlx5_flow_tbl_data_entry {
 	uint32_t group_id;
 	bool external;
 	bool tunnel_offload; /* Tunnel offlod table or not. */
+	bool is_egress; /**< Egress table. */
 };
 
 /* Sub rdma-core actions list. */
@@ -1430,4 +1429,12 @@ struct mlx5_hlist_entry *flow_dv_encap_decap_create_cb(struct mlx5_hlist *list,
 				uint64_t key, void *cb_ctx);
 void flow_dv_encap_decap_remove_cb(struct mlx5_hlist *list,
 				   struct mlx5_hlist_entry *entry);
+
+int flow_dv_matcher_match_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry, void *ctx);
+struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry, void *ctx);
+void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +115 -100

346 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#183 lore raw

PATCH v5 25/34 net/mlx5: make port ID action cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch convert port id action cache list to thread safe cache list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 ++
 drivers/net/mlx5/mlx5.h          |   2 +-
 drivers/net/mlx5/mlx5_flow.h     |  15 ++--
 drivers/net/mlx5/mlx5_flow_dv.c  | 143 +++++++++++++++++++++------------------
 4 files changed, 94 insertions(+), 73 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index 10fc7c5..3d2bf57 100644
@@ -236,6 +236,12 @@
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	/* Init port id action cache list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+			     flow_dv_port_id_create_cb,
+			     flow_dv_port_id_match_cb,
+			     flow_dv_port_id_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -431,6 +437,7 @@
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
+	mlx5_cache_list_destroy(&sh->port_id_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index eadb797..1cd42eb 100644
@@ -659,7 +659,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	uint32_t port_id_action_list; /* List of port ID actions. */
+	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	uint32_t push_vlan_action_list; /* List of push VLAN actions. */
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
drivers/net/mlx5/mlx5_flow.h +10 -5
index 6c0815a..0e4d47f 100644
@@ -485,12 +485,10 @@ struct mlx5_flow_dv_jump_tbl_resource {
 
 /* Port ID resource structure. */
 struct mlx5_flow_dv_port_id_action_resource {
-	ILIST_ENTRY(uint32_t)next;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *action;
-	/**< Action object. */
+	struct mlx5_cache_entry entry;
+	void *action; /**< Action object. */
 	uint32_t port_id; /**< Port ID value. */
+	uint32_t idx; /**< Indexed pool memory index. */
 };
 
 /* Push VLAN action resource structure */
@@ -1437,4 +1435,11 @@ struct mlx5_cache_entry *flow_dv_matcher_create_cb(struct mlx5_cache_list *list,
 void flow_dv_matcher_remove_cb(struct mlx5_cache_list *list,
 			       struct mlx5_cache_entry *entry);
 
+int flow_dv_port_id_match_cb(struct mlx5_cache_list *list,
+			     struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
+		struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +76 -67

170 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#184 lore raw

PATCH v5 26/34 net/mlx5: make push VLAN action cache thread safe

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch converts push VLAN action cache list to thread safe cache list.

---
 drivers/net/mlx5/linux/mlx5_os.c |   7 ++
 drivers/net/mlx5/mlx5.h          |   2 +-
 drivers/net/mlx5/mlx5_flow.h     |  13 +++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 159 +++++++++++++++++++++------------------
 4 files changed, 102 insertions(+), 79 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +7 -0
index 3d2bf57..9ab3ba9 100644
@@ -242,6 +242,12 @@
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
+	/* Init push vlan action cache list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+			     flow_dv_push_vlan_create_cb,
+			     flow_dv_push_vlan_match_cb,
+			     flow_dv_push_vlan_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -438,6 +444,7 @@
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_cache_list_destroy(&sh->port_id_action_list);
+	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
drivers/net/mlx5/mlx5.h +1 -1
index 1cd42eb..9efa354 100644
@@ -660,7 +660,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	uint32_t push_vlan_action_list; /* List of push VLAN actions. */
+	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
 	uint32_t sample_action_list; /* List of sample actions. */
 	uint32_t dest_array_list; /* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
drivers/net/mlx5/mlx5_flow.h +10 -3
index 0e4d47f..500a580 100644
@@ -493,12 +493,11 @@ struct mlx5_flow_dv_port_id_action_resource {
 
 /* Push VLAN action resource structure */
 struct mlx5_flow_dv_push_vlan_action_resource {
-	ILIST_ENTRY(uint32_t)next;
-	/* Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /* Cache entry. */
 	void *action; /**< Action object. */
 	uint8_t ft_type; /**< Flow table type, Rx, Tx or FDB. */
 	rte_be32_t vlan_tag; /**< VLAN tag value. */
+	uint32_t idx; /**< Indexed pool memory index. */
 };
 
 /* Metadata register copy table entry. */
@@ -1442,4 +1441,12 @@ struct mlx5_cache_entry *flow_dv_port_id_create_cb(struct mlx5_cache_list *list,
 void flow_dv_port_id_remove_cb(struct mlx5_cache_list *list,
 			       struct mlx5_cache_entry *entry);
 
+int flow_dv_push_vlan_match_cb(struct mlx5_cache_list *list,
+			       struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +84 -75

189 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#185 lore raw

PATCH v5 27/34 net/mlx5: simplify sample attributes

In reply to Suanming Mou

Currently, the sample action resource already has ft_type to indicate the action domain attribute, the extra flow attributes parameter can be optimized.

This commit uses action resource ty_type as domain attribute instead of the flow attribute.

---
 drivers/net/mlx5/mlx5_flow_dv.c | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_flow_dv.c +14 -15
index d6414eb..fbe114c 100644
@@ -8605,8 +8605,6 @@ struct mlx5_hlist_entry *
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] attr
- *   Attributes of flow that includes this item.
  * @param[in] resource
  *   Pointer to sample resource.
  * @parm[in, out] dev_flow
@@ -8621,7 +8619,6 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_sample_resource_register(struct rte_eth_dev *dev,
-			 const struct rte_flow_attr *attr,
 			 struct mlx5_flow_dv_sample_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 void **sample_dv_actions,
@@ -8635,6 +8632,8 @@ struct mlx5_hlist_entry *
 	uint32_t idx = 0;
 	const uint32_t next_ft_step = 1;
 	uint32_t next_ft_id = resource->ft_id +	next_ft_step;
+	uint8_t is_egress = 0;
+	uint8_t is_transfer = 0;
 
 	/* Lookup a matching resource from cache. */
 	ILIST_FOREACH(sh->ipool[MLX5_IPOOL_SAMPLE], sh->sample_action_list,
@@ -8667,8 +8666,12 @@ struct mlx5_hlist_entry *
 					  "cannot allocate resource memory");
 	*cache_resource = *resource;
 	/* Create normal path table level */
+	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+		is_transfer = 1;
+	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+		is_egress = 1;
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
-					attr->egress, attr->transfer,
+					is_egress, is_transfer,
 					dev_flow->external, NULL, 0, 0, error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENOMEM,
@@ -8751,8 +8754,6 @@ struct mlx5_hlist_entry *
  *
  * @param[in, out] dev
  *   Pointer to rte_eth_dev structure.
- * @param[in] attr
- *   Attributes of flow that includes this item.
  * @param[in] resource
  *   Pointer to destination array resource.
  * @parm[in, out] dev_flow
@@ -8765,7 +8766,6 @@ struct mlx5_hlist_entry *
  */
 static int
 flow_dv_dest_array_resource_register(struct rte_eth_dev *dev,
-			 const struct rte_flow_attr *attr,
 			 struct mlx5_flow_dv_dest_array_resource *resource,
 			 struct mlx5_flow *dev_flow,
 			 struct rte_flow_error *error)
@@ -8809,9 +8809,9 @@ struct mlx5_hlist_entry *
 					  NULL,
 					  "cannot allocate resource memory");
 	*cache_resource = *resource;
-	if (attr->transfer)
+	if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
 		domain = sh->fdb_domain;
-	else if (attr->ingress)
+	else if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
 		domain = sh->rx_domain;
 	else
 		domain = sh->tx_domain;
@@ -9100,6 +9100,8 @@ struct mlx5_hlist_entry *
 		res->set_action = action_ctx.set_action;
 	} else if (attr->ingress) {
 		res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_RX;
+	} else {
+		res->ft_type = MLX5DV_FLOW_TABLE_TYPE_NIC_TX;
 	}
 	return 0;
 }
@@ -9111,8 +9113,6 @@ struct mlx5_hlist_entry *
  *   Pointer to rte_eth_dev structure.
  * @param[in, out] dev_flow
  *   Pointer to the mlx5_flow.
- * @param[in] attr
- *   Pointer to the flow attributes.
  * @param[in] num_of_dest
  *   The num of destination.
  * @param[in, out] res
@@ -9132,7 +9132,6 @@ struct mlx5_hlist_entry *
 static int
 flow_dv_create_action_sample(struct rte_eth_dev *dev,
 			     struct mlx5_flow *dev_flow,
-			     const struct rte_flow_attr *attr,
 			     uint32_t num_of_dest,
 			     struct mlx5_flow_dv_sample_resource *res,
 			     struct mlx5_flow_dv_dest_array_resource *mdest_res,
@@ -9192,14 +9191,14 @@ struct mlx5_hlist_entry *
 		memcpy(&mdest_res->sample_act[0], &res->sample_act,
 				sizeof(struct mlx5_flow_sub_actions_list));
 		mdest_res->num_of_dest = num_of_dest;
-		if (flow_dv_dest_array_resource_register(dev, attr, mdest_res,
+		if (flow_dv_dest_array_resource_register(dev, mdest_res,
 							 dev_flow, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL, "can't create sample "
 						  "action");
 	} else {
-		if (flow_dv_sample_resource_register(dev, attr, res, dev_flow,
+		if (flow_dv_sample_resource_register(dev, res, dev_flow,
 						     sample_actions, error))
 			return rte_flow_error_set(error, EINVAL,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -9839,7 +9838,7 @@ struct mlx5_hlist_entry *
 			}
 			if (action_flags & MLX5_FLOW_ACTION_SAMPLE) {
 				ret = flow_dv_create_action_sample(dev,
-							  dev_flow, attr,
+							  dev_flow,
 							  num_of_dest,
 							  &sample_res,
 							  &mdest_res,
Signature · 1 line
1.8.3.1
#186 lore raw

PATCH v5 28/34 net/mlx5: fix sample register error flow

In reply to Suanming Mou

Currently, sample flow need to prepare and register the sub-actions before sample action is created.

Once the same sample action exists, the sub-actions registered by the second flow should be released, or these sub-actions will be leaked. Since the exist sample action only release these same sub-actions when the sample action itself releases.

When same sample action exists, call the sub-action release function for the later flow to release the redundant prepared sub-actions.

Fixes 0756228b2704 ("net/mlx5: update translate function for sample action")
---
 drivers/net/mlx5/mlx5_flow_dv.c | 117 ++++++++++++++++++++--------------------
 1 file changed, 60 insertions(+), 57 deletions(-)
drivers/net/mlx5/mlx5_flow_dv.c +60 -57
index fbe114c..3b36704 100644
@@ -8601,6 +8601,42 @@ struct mlx5_hlist_entry *
 }
 
 /**
+ * Release sample sub action resource.
+ *
+ * @param[in, out] dev
+ *   Pointer to rte_eth_dev structure.
+ * @param[in] act_res
+ *   Pointer to sample sub action resource.
+ */
+static void
+flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
+				   struct mlx5_flow_sub_actions_idx *act_res)
+{
+	if (act_res->rix_hrxq) {
+		mlx5_hrxq_release(dev, act_res->rix_hrxq);
+		act_res->rix_hrxq = 0;
+	}
+	if (act_res->rix_encap_decap) {
+		flow_dv_encap_decap_resource_release(dev,
+						     act_res->rix_encap_decap);
+		act_res->rix_encap_decap = 0;
+	}
+	if (act_res->rix_port_id_action) {
+		flow_dv_port_id_action_resource_release(dev,
+						act_res->rix_port_id_action);
+		act_res->rix_port_id_action = 0;
+	}
+	if (act_res->rix_tag) {
+		flow_dv_tag_release(dev, act_res->rix_tag);
+		act_res->rix_tag = 0;
+	}
+	if (act_res->cnt) {
+		flow_dv_counter_release(dev, act_res->cnt);
+		act_res->cnt = 0;
+	}
+}
+
+/**
  * Find existing sample resource or create and register a new one.
  *
  * @param[in, out] dev
@@ -8653,6 +8689,12 @@ struct mlx5_hlist_entry *
 					   __ATOMIC_RELAXED);
 			dev_flow->handle->dvh.rix_sample = idx;
 			dev_flow->dv.sample_res = cache_resource;
+			/*
+			 * Existing smaple action should release the prepared
+			 * sub-actions reference counter.
+			 */
+			flow_dv_sample_sub_actions_release(dev,
+							&resource->sample_idx);
 			return 0;
 		}
 	}
@@ -8721,25 +8763,13 @@ struct mlx5_hlist_entry *
 		__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
 	return 0;
 error:
-	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB) {
-		if (cache_resource->default_miss)
-			claim_zero(mlx5_glue->destroy_flow_action
+	if (cache_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB &&
+	    cache_resource->default_miss)
+		claim_zero(mlx5_glue->destroy_flow_action
 				(cache_resource->default_miss));
-	} else {
-		if (cache_resource->sample_idx.rix_hrxq &&
-		    !mlx5_hrxq_release(dev,
-				cache_resource->sample_idx.rix_hrxq))
-			cache_resource->sample_idx.rix_hrxq = 0;
-		if (cache_resource->sample_idx.rix_tag &&
-		    !flow_dv_tag_release(dev,
-				cache_resource->sample_idx.rix_tag))
-			cache_resource->sample_idx.rix_tag = 0;
-		if (cache_resource->sample_idx.cnt) {
-			flow_dv_counter_release(dev,
-				cache_resource->sample_idx.cnt);
-			cache_resource->sample_idx.cnt = 0;
-		}
-	}
+	else
+		flow_dv_sample_sub_actions_release(dev,
+						   &cache_resource->sample_idx);
 	if (cache_resource->normal_path_tbl)
 		flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
@@ -8797,6 +8827,13 @@ struct mlx5_hlist_entry *
 					   __ATOMIC_RELAXED);
 			dev_flow->handle->dvh.rix_dest_array = idx;
 			dev_flow->dv.dest_array_res = cache_resource;
+			/*
+			 * Existing smaple action should release the prepared
+			 * sub-actions reference counter.
+			 */
+			for (idx = 0; idx < resource->num_of_dest; idx++)
+				flow_dv_sample_sub_actions_release(dev,
+						&resource->sample_idx[idx]);
 			return 0;
 		}
 	}
@@ -10693,21 +10730,8 @@ struct mlx5_hlist_entry *
 		if (cache_resource->normal_path_tbl)
 			flow_dv_tbl_resource_release(MLX5_SH(dev),
 				cache_resource->normal_path_tbl);
-	}
-	if (cache_resource->sample_idx.rix_hrxq &&
-		!mlx5_hrxq_release(dev,
-			cache_resource->sample_idx.rix_hrxq))
-		cache_resource->sample_idx.rix_hrxq = 0;
-	if (cache_resource->sample_idx.rix_tag &&
-		!flow_dv_tag_release(dev,
-			cache_resource->sample_idx.rix_tag))
-		cache_resource->sample_idx.rix_tag = 0;
-	if (cache_resource->sample_idx.cnt) {
-		flow_dv_counter_release(dev,
-			cache_resource->sample_idx.cnt);
-		cache_resource->sample_idx.cnt = 0;
-	}
-	if (!__atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED)) {
+		flow_dv_sample_sub_actions_release(dev,
+					&cache_resource->sample_idx);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_SAMPLE],
 			     &priv->sh->sample_action_list, idx,
 			     cache_resource, next);
@@ -10736,7 +10760,6 @@ struct mlx5_hlist_entry *
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_flow_dv_dest_array_resource *cache_resource;
-	struct mlx5_flow_sub_actions_idx *mdest_act_res;
 	uint32_t idx = handle->dvh.rix_dest_array;
 	uint32_t i = 0;
 
@@ -10753,29 +10776,9 @@ struct mlx5_hlist_entry *
 		if (cache_resource->action)
 			claim_zero(mlx5_glue->destroy_flow_action
 						(cache_resource->action));
-		for (; i < cache_resource->num_of_dest; i++) {
-			mdest_act_res = &cache_resource->sample_idx[i];
-			if (mdest_act_res->rix_hrxq) {
-				mlx5_hrxq_release(dev,
-					mdest_act_res->rix_hrxq);
-				mdest_act_res->rix_hrxq = 0;
-			}
-			if (mdest_act_res->rix_encap_decap) {
-				flow_dv_encap_decap_resource_release(dev,
-					mdest_act_res->rix_encap_decap);
-				mdest_act_res->rix_encap_decap = 0;
-			}
-			if (mdest_act_res->rix_port_id_action) {
-				flow_dv_port_id_action_resource_release(dev,
-					mdest_act_res->rix_port_id_action);
-				mdest_act_res->rix_port_id_action = 0;
-			}
-			if (mdest_act_res->rix_tag) {
-				flow_dv_tag_release(dev,
-					mdest_act_res->rix_tag);
-				mdest_act_res->rix_tag = 0;
-			}
-		}
+		for (; i < cache_resource->num_of_dest; i++)
+			flow_dv_sample_sub_actions_release(dev,
+					&cache_resource->sample_idx[i]);
 		ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DEST_ARRAY],
 			     &priv->sh->dest_array_list, idx,
 			     cache_resource, next);
Signature · 1 line
1.8.3.1
#187 lore raw

PATCH v5 29/34 net/mlx5: make sample and mirror action thread safe

In reply to Suanming Mou

This commit uses cache list to make sample and mirror action thread safe.

---
 drivers/net/mlx5/linux/mlx5_os.c |  14 ++
 drivers/net/mlx5/mlx5.h          |   5 +-
 drivers/net/mlx5/mlx5_flow.h     |  28 ++-
 drivers/net/mlx5/mlx5_flow_dv.c  | 396 ++++++++++++++++++++++-----------------
 4 files changed, 260 insertions(+), 183 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +14 -0
index 9ab3ba9..5856981 100644
@@ -248,6 +248,20 @@
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
+	/* Init sample action cache list. */
+	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->sample_action_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_sample_create_cb,
+			     flow_dv_sample_match_cb,
+			     flow_dv_sample_remove_cb);
+	/* Init dest array action cache list. */
+	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
+	mlx5_cache_list_init(&sh->dest_array_list, s, 0,
+			     &rte_eth_devices[priv->dev_data->port_id],
+			     flow_dv_dest_array_create_cb,
+			     flow_dv_dest_array_match_cb,
+			     flow_dv_dest_array_remove_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
drivers/net/mlx5/mlx5.h +3 -2
index 9efa354..ac7a026 100644
@@ -661,8 +661,9 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *tag_table;
 	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
 	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	uint32_t sample_action_list; /* List of sample actions. */
-	uint32_t dest_array_list; /* List of destination array actions. */
+	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
+	struct mlx5_cache_list dest_array_list;
+	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
drivers/net/mlx5/mlx5_flow.h +23 -5
index 500a580..4a8c2bf 100644
@@ -562,9 +562,12 @@ struct mlx5_flow_sub_actions_idx {
 
 /* Sample action resource structure. */
 struct mlx5_flow_dv_sample_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
-	void *verbs_action; /**< Verbs sample action object. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	union {
+		void *verbs_action; /**< Verbs sample action object. */
+		void **sub_actions; /**< Sample sub-action array. */
+	};
+	uint32_t idx; /** Sample object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint32_t ft_id; /** Flow Table Level */
 	uint32_t ratio;   /** Sample Ratio */
@@ -581,8 +584,8 @@ struct mlx5_flow_dv_sample_resource {
 
 /* Destination array action resource structure. */
 struct mlx5_flow_dv_dest_array_resource {
-	ILIST_ENTRY(uint32_t)next; /**< Pointer to next element. */
-	uint32_t refcnt; /**< Reference counter. */
+	struct mlx5_cache_entry entry; /**< Cache entry. */
+	uint32_t idx; /** Destination array action object index. */
 	uint8_t ft_type; /** Flow Table Type */
 	uint8_t num_of_dest; /**< Number of destination actions. */
 	void *action; /**< Pointer to the rdma core action. */
@@ -1449,4 +1452,19 @@ struct mlx5_cache_entry *flow_dv_push_vlan_create_cb
 void flow_dv_push_vlan_remove_cb(struct mlx5_cache_list *list,
 				 struct mlx5_cache_entry *entry);
 
+int flow_dv_sample_match_cb(struct mlx5_cache_list *list,
+			    struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_sample_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_sample_remove_cb(struct mlx5_cache_list *list,
+			      struct mlx5_cache_entry *entry);
+
+int flow_dv_dest_array_match_cb(struct mlx5_cache_list *list,
+				struct mlx5_cache_entry *entry, void *cb_ctx);
+struct mlx5_cache_entry *flow_dv_dest_array_create_cb
+				(struct mlx5_cache_list *list,
+				 struct mlx5_cache_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
+				  struct mlx5_cache_entry *entry);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
drivers/net/mlx5/mlx5_flow_dv.c +220 -176

510 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#188 lore raw

PATCH v5 30/34 net/mlx5: make tunnel offloading table thread safe

In reply to Suanming Mou

To support multi-thread flow insertion, this patch updates tunnel offloading hash table to use thread safe hash list.

---
 drivers/net/mlx5/mlx5_flow.c    | 92 ++++++++++++++++++++++++++---------------
 drivers/net/mlx5/mlx5_flow_dv.c | 11 +----
 2 files changed, 61 insertions(+), 42 deletions(-)
drivers/net/mlx5/mlx5_flow.c +59 -33
index 37e7fc3..1e82030 100644
@@ -7399,13 +7399,56 @@ struct mlx5_meter_domains_infos *
 	       container_of(he, struct mlx5_flow_tbl_data_entry, entry) : NULL;
 }
 
+static void
+mlx5_flow_tunnel_grp2tbl_remove_cb(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct tunnel_tbl_entry *tte = container_of(entry, typeof(*tte), hash);
+
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+			tunnel_flow_tbl_to_id(tte->flow_table));
+	mlx5_free(tte);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_flow_tunnel_grp2tbl_create_cb(struct mlx5_hlist *list,
+				   uint64_t key __rte_unused,
+				   void *ctx __rte_unused)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct tunnel_tbl_entry *tte;
+
+	tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
+			  sizeof(*tte), 0,
+			  SOCKET_ID_ANY);
+	if (!tte)
+		goto err;
+	mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+			  &tte->flow_table);
+	if (tte->flow_table >= MLX5_MAX_TABLES) {
+		DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
+			tte->flow_table);
+		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+				tte->flow_table);
+		goto err;
+	} else if (!tte->flow_table) {
+		goto err;
+	}
+	tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
+	return &tte->hash;
+err:
+	if (tte)
+		mlx5_free(tte);
+	return NULL;
+}
+
 static uint32_t
 tunnel_flow_group_to_flow_table(struct rte_eth_dev *dev,
 				const struct mlx5_flow_tunnel *tunnel,
 				uint32_t group, uint32_t *table,
 				struct rte_flow_error *error)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hlist_entry *he;
 	struct tunnel_tbl_entry *tte;
 	union tunnel_tbl_key key = {
@@ -7416,40 +7459,17 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_hlist *group_hash;
 
 	group_hash = tunnel ? tunnel->groups : thub->groups;
-	he = mlx5_hlist_lookup(group_hash, key.val, NULL);
-	if (!he) {
-		tte = mlx5_malloc(MLX5_MEM_SYS | MLX5_MEM_ZERO,
-				  sizeof(*tte), 0,
-				  SOCKET_ID_ANY);
-		if (!tte)
-			goto err;
-		tte->hash.key = key.val;
-		mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-				  &tte->flow_table);
-		if (tte->flow_table >= MLX5_MAX_TABLES) {
-			DRV_LOG(ERR, "Tunnel TBL ID %d exceed max limit.",
-				tte->flow_table);
-			mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-					tte->flow_table);
-			goto err;
-		} else if (!tte->flow_table) {
-			goto err;
-		}
-		tte->flow_table = tunnel_id_to_flow_tbl(tte->flow_table);
-		mlx5_hlist_insert(group_hash, &tte->hash);
-	} else {
-		tte = container_of(he, typeof(*tte), hash);
-	}
+	he = mlx5_hlist_register(group_hash, key.val, NULL);
+	if (!he)
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+					  NULL,
+					  "tunnel group index not supported");
+	tte = container_of(he, typeof(*tte), hash);
 	*table = tte->flow_table;
 	DRV_LOG(DEBUG, "port %u tunnel %u group=%#x table=%#x",
 		dev->data->port_id, key.tunnel_id, group, *table);
 	return 0;
-
-err:
-	if (tte)
-		mlx5_free(tte);
-	return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
-				  NULL, "tunnel group index not supported");
 }
 
 static int
@@ -7972,13 +7992,16 @@ struct mlx5_meter_domains_infos *
 		return NULL;
 	}
 	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, 0, 0,
-					   NULL, NULL, NULL);
+					   mlx5_flow_tunnel_grp2tbl_create_cb,
+					   NULL,
+					   mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!tunnel->groups) {
 		mlx5_ipool_free(priv->sh->ipool
 				[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], id);
 		mlx5_free(tunnel);
 		return NULL;
 	}
+	tunnel->groups->ctx = priv->sh;
 	/* initiate new PMD tunnel */
 	memcpy(&tunnel->app_tunnel, app_tunnel, sizeof(*app_tunnel));
 	tunnel->tunnel_id = id;
@@ -8052,11 +8075,14 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
-					 0, NULL, NULL, NULL);
+					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
+					 NULL,
+					 mlx5_flow_tunnel_grp2tbl_remove_cb);
 	if (!thub->groups) {
 		err = -rte_errno;
 		goto err;
 	}
+	thub->groups->ctx = sh;
 	sh->tunnel_hub = thub;
 
 	return 0;
drivers/net/mlx5/mlx5_flow_dv.c +2 -9
index 79d1317..08bbc5d 100644
@@ -8102,15 +8102,8 @@ struct mlx5_flow_tbl_resource *
 					tbl_data->tunnel->groups :
 					thub->groups;
 		he = mlx5_hlist_lookup(tunnel_grp_hash, tunnel_key.val, NULL);
-		if (he) {
-			struct tunnel_tbl_entry *tte;
-			tte = container_of(he, typeof(*tte), hash);
-			MLX5_ASSERT(tte->flow_table == table_id);
-			mlx5_hlist_remove(tunnel_grp_hash, he);
-			mlx5_free(tte);
-		}
-		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
-				tunnel_flow_tbl_to_id(table_id));
+		if (he)
+			mlx5_hlist_unregister(tunnel_grp_hash, he);
 		DRV_LOG(DEBUG,
 			"Table_id %#x tunnel %u group %u released.",
 			table_id,
Signature · 1 line
1.8.3.1
#189 lore raw

PATCH v5 31/34 net/mlx5: remove unused hash list operations

In reply to Suanming Mou

In previous commits the hash list objects have been converted to new thread safe hash list. The legacy hash list code can be removed now.

---
 drivers/net/mlx5/mlx5_utils.c | 38 -------------------------
 drivers/net/mlx5/mlx5_utils.h | 66 -------------------------------------------
 2 files changed, 104 deletions(-)
drivers/net/mlx5/mlx5_utils.c +0 -38
index df15f63..13590dd 100644
@@ -170,44 +170,6 @@ struct mlx5_hlist_entry*
 	return entry;
 }
 
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup_ex(struct mlx5_hlist *h, uint64_t key,
-		     mlx5_hlist_match_callback_fn cb, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h && cb && ctx);
-	idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	first = &h->heads[idx];
-	LIST_FOREACH(node, first, next) {
-		if (!cb(node, ctx))
-			return node;
-	}
-	return NULL;
-}
-
-int
-mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry,
-		     mlx5_hlist_match_callback_fn cb, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h && entry && cb && ctx);
-	idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
-	first = &h->heads[idx];
-	/* No need to reuse the lookup function. */
-	LIST_FOREACH(node, first, next) {
-		if (!cb(node, ctx))
-			return -EEXIST;
-	}
-	LIST_INSERT_HEAD(first, entry, next);
-	return 0;
-}
-
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
 {
drivers/net/mlx5/mlx5_utils.h +0 -66

84 lines not on this page · raw message

Signature · 1 line
1.8.3.1
#190 lore raw

PATCH v5 32/34 net/mlx5: make tunnel hub list thread safe

In reply to Suanming Mou

This commit uses spinlock to protect the tunnel hub list in multiple thread.

---
 drivers/net/mlx5/mlx5_flow.c | 20 +++++++++++++++++---
 drivers/net/mlx5/mlx5_flow.h |  1 +
 2 files changed, 18 insertions(+), 3 deletions(-)
drivers/net/mlx5/mlx5_flow.c +17 -3
index 1e82030..a6e60af 100644
@@ -669,10 +669,14 @@ enum mlx5_expansion {
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
-		if (&tun->item == pmd_items)
+		if (&tun->item == pmd_items) {
+			LIST_REMOVE(tun, chain);
 			break;
+		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (!tun || num_items != 1)
 		return rte_flow_error_set(err, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
@@ -690,10 +694,14 @@ enum mlx5_expansion {
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
-		if (&tun->action == pmd_actions)
+		if (&tun->action == pmd_actions) {
+			LIST_REMOVE(tun, chain);
 			break;
+		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (!tun || num_actions != 1)
 		return rte_flow_error_set(err, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
@@ -5880,8 +5888,12 @@ struct rte_flow *
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_RTE_FLOW], flow_idx);
 	if (flow->tunnel) {
 		struct mlx5_flow_tunnel *tunnel;
+
+		rte_spinlock_lock(&mlx5_tunnel_hub(dev)->sl);
 		tunnel = mlx5_find_tunnel_id(dev, flow->tunnel_id);
 		RTE_VERIFY(tunnel);
+		LIST_REMOVE(tunnel, chain);
+		rte_spinlock_unlock(&mlx5_tunnel_hub(dev)->sl);
 		if (!__atomic_sub_fetch(&tunnel->refctn, 1, __ATOMIC_RELAXED))
 			mlx5_flow_tunnel_free(dev, tunnel);
 	}
@@ -7940,7 +7952,6 @@ struct mlx5_meter_domains_infos *
 	DRV_LOG(DEBUG, "port %u release pmd tunnel id=0x%x",
 		dev->data->port_id, tunnel->tunnel_id);
 	RTE_VERIFY(!__atomic_load_n(&tunnel->refctn, __ATOMIC_RELAXED));
-	LIST_REMOVE(tunnel, chain);
 	mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TUNNEL_ID],
 			tunnel->tunnel_id);
 	mlx5_hlist_destroy(tunnel->groups);
@@ -8029,6 +8040,7 @@ struct mlx5_meter_domains_infos *
 	struct mlx5_flow_tunnel_hub *thub = mlx5_tunnel_hub(dev);
 	struct mlx5_flow_tunnel *tun;
 
+	rte_spinlock_lock(&thub->sl);
 	LIST_FOREACH(tun, &thub->tunnels, chain) {
 		if (!memcmp(app_tunnel, &tun->app_tunnel,
 			    sizeof(*app_tunnel))) {
@@ -8046,6 +8058,7 @@ struct mlx5_meter_domains_infos *
 			ret = -ENOMEM;
 		}
 	}
+	rte_spinlock_unlock(&thub->sl);
 	if (tun)
 		__atomic_add_fetch(&tun->refctn, 1, __ATOMIC_RELAXED);
 
@@ -8074,6 +8087,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 	if (!thub)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
+	rte_spinlock_init(&thub->sl);
 	thub->groups = mlx5_hlist_create("flow groups", MLX5_MAX_TABLES, 0,
 					 0, mlx5_flow_tunnel_grp2tbl_create_cb,
 					 NULL,
drivers/net/mlx5/mlx5_flow.h +1 -0
index 4a8c2bf..8ef2a85 100644
@@ -945,6 +945,7 @@ struct mlx5_flow_tunnel {
 /** PMD tunnel related context */
 struct mlx5_flow_tunnel_hub {
 	LIST_HEAD(, mlx5_flow_tunnel) tunnels;
+	rte_spinlock_t sl;			/* Tunnel list spinlock. */
 	struct mlx5_hlist *groups;		/** non tunnel groups */
 };
 
Signature · 1 line
1.8.3.1
#191 lore raw

PATCH v5 33/34 net/mlx5: make shared action list thread safe

In reply to Suanming Mou

This commit uses spinlock to protect the shared action list in multiple thread.

---
 drivers/net/mlx5/linux/mlx5_os.c | 1 +
 drivers/net/mlx5/mlx5.h          | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c  | 5 +++++
 3 files changed, 7 insertions(+)
drivers/net/mlx5/linux/mlx5_os.c +1 -0
index 5856981..8612cab 100644
@@ -1534,6 +1534,7 @@
 		}
 		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
+	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
 	return eth_dev;
 error:
drivers/net/mlx5/mlx5.h +1 -0
index ac7a026..be21a9a 100644
@@ -912,6 +912,7 @@ struct mlx5_priv {
 	uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
 	struct mlx5_mp_id mp_id; /* ID of a multi-process process */
 	LIST_HEAD(fdir, mlx5_fdir_flow) fdir_flows; /* fdir flows. */
+	rte_spinlock_t shared_act_sl; /* Shared actions spinlock. */
 	LIST_HEAD(shared_action, rte_flow_shared_action) shared_actions;
 	/* shared actions */
 };
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index 08bbc5d..cf48402 100644
@@ -11191,7 +11191,9 @@ struct mlx5_cache_entry *
 	if (shared_action) {
 		__atomic_add_fetch(&shared_action->refcnt, 1,
 				   __ATOMIC_RELAXED);
+		rte_spinlock_lock(&priv->shared_act_sl);
 		LIST_INSERT_HEAD(&priv->shared_actions, shared_action, next);
+		rte_spinlock_unlock(&priv->shared_act_sl);
 	}
 	return shared_action;
 }
@@ -11218,6 +11220,7 @@ struct mlx5_cache_entry *
 			 struct rte_flow_shared_action *action,
 			 struct rte_flow_error *error)
 {
+	struct mlx5_priv *priv = dev->data->dev_private;
 	int ret;
 
 	switch (action->type) {
@@ -11232,7 +11235,9 @@ struct mlx5_cache_entry *
 	}
 	if (ret)
 		return ret;
+	rte_spinlock_lock(&priv->shared_act_sl);
 	LIST_REMOVE(action, next);
+	rte_spinlock_unlock(&priv->shared_act_sl);
 	rte_free(action);
 	return 0;
 }
Signature · 1 line
1.8.3.1
#192 lore raw

PATCH v5 34/34 net/mlx5: remove shared context lock

In reply to Suanming Mou

From: Xueming Li <[email protected]>

To support multi-thread flow insertion, this patch removes shared data lock since all resources should support concurrent protection.

---
 doc/guides/nics/mlx5.rst               |   1 +
 doc/guides/rel_notes/release_20_11.rst |   1 +
 drivers/net/mlx5/linux/mlx5_os.c       |   4 +-
 drivers/net/mlx5/mlx5.h                |   1 -
 drivers/net/mlx5/mlx5_flow_dv.c        | 209 ++++-----------------------------
 5 files changed, 27 insertions(+), 189 deletions(-)
doc/guides/nics/mlx5.rst +1 -0
index e5e55fc..8c59cc6 100644
@@ -96,6 +96,7 @@ Features
 - Per packet no-inline hint flag to disable packet data copying into Tx descriptors.
 - Hardware LRO.
 - Hairpin.
+- Multiple-thread flow insertion.
 
 Limitations
 -----------
doc/guides/rel_notes/release_20_11.rst +1 -0
index e284771..0742fc8 100644
@@ -358,6 +358,7 @@ New Features
   * Added support for QinQ packets matching.
   * Added support for the new vlan fields ``has_vlan`` in the eth item and
     ``has_more_vlan`` in the vlan item.
+  * Added support for PMD level multiple-thread flow insertion.
 
 * **Updated vhost sample application.**
 
drivers/net/mlx5/linux/mlx5_os.c +2 -2
index 8612cab..d4f2194 100644
@@ -321,7 +321,6 @@
 		err = errno;
 		goto error;
 	}
-	pthread_mutex_init(&sh->dv_mutex, NULL);
 	sh->tx_domain = domain;
 #ifdef HAVE_MLX5DV_DR_ESWITCH
 	if (priv->config.dv_esw_en) {
@@ -435,7 +434,6 @@
 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 		sh->pop_vlan_action = NULL;
 	}
-	pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
 	if (sh->default_miss_action)
 		mlx5_glue->destroy_flow_action
@@ -1536,6 +1534,8 @@
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
+	if (priv->config.dv_flow_en)
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
 	return eth_dev;
 error:
 	if (priv) {
drivers/net/mlx5/mlx5.h +0 -1
index be21a9a..9a458cd 100644
@@ -639,7 +639,6 @@ struct mlx5_dev_ctx_shared {
 	/* Packet pacing related structure. */
 	struct mlx5_dev_txpp txpp;
 	/* Shared DV/DR flow data section. */
-	pthread_mutex_t dv_mutex; /* DV context mutex. */
 	uint32_t dv_meta_mask; /* flow META metadata supported mask. */
 	uint32_t dv_mark_mask; /* flow MARK metadata supported mask. */
 	uint32_t dv_regc0_mask; /* available bits of metatada reg_c[0]. */
drivers/net/mlx5/mlx5_flow_dv.c +23 -186

296 lines not on this page · raw message

Signature · 1 line
1.8.3.1