Skip to contentSkip to conversation

net/mlx5: add match with Tx queue item

PATCH Superseded Suanming Mou 3 participants 12 messages May 31, 2024 – Jun 6, 2024 Network drivers

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 net/mlx5: add match with Tx queue item

With the item RTE_FLOW_ITEM_TYPE_TX_QUEUE, user will be able to set the Tx queue index and create flow match with that queue index.

This commit adds match with RTE_FLOW_ITEM_TX_QUEUE item.

---
 doc/guides/nics/features/mlx5.ini      |  1 +
 doc/guides/rel_notes/release_24_07.rst |  4 ++
 drivers/net/mlx5/hws/mlx5dr_definer.c  | 50 +++++++++++++++++++
 drivers/net/mlx5/mlx5_flow.h           | 58 ++++++++++++++++++++++
 drivers/net/mlx5/mlx5_flow_dv.c        | 69 ++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_flow_hw.c        |  1 +
 6 files changed, 183 insertions(+)
doc/guides/nics/features/mlx5.ini +1 -0
index 81a7067cc3..056e04275b 100644
@@ -92,6 +92,7 @@ quota                = Y
 random               = Y
 tag                  = Y
 tcp                  = Y
+tx_queue             = Y
 udp                  = Y
 vlan                 = Y
 vxlan                = Y
doc/guides/rel_notes/release_24_07.rst +4 -0
index ffbe9ce051..46efc04eac 100644
@@ -81,6 +81,10 @@ New Features
 
   * Added SSE/NEON vector datapath.
 
+* **Updated NVIDIA mlx5 driver.**
+
+  * Added match with Tx queue.
+
 
 Removed Items
 -------------
drivers/net/mlx5/hws/mlx5dr_definer.c +50 -0

74 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +58 -0

70 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +69 -0

93 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_hw.c +1 -0
index 68c5a36bbb..39c9a0ba50 100644
@@ -7668,6 +7668,7 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_MPLS:
 		case RTE_FLOW_ITEM_TYPE_GENEVE:
 		case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
+		case RTE_FLOW_ITEM_TYPE_TX_QUEUE:
 		case RTE_FLOW_ITEM_TYPE_GRE:
 		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
Signature · 1 line
2.34.1
#1 lore raw

PATCH 2/3 net/mlx5: rename external Rx queue to external queue

Due to external Tx queue will be supported, in order to reuse the external queue struct, rename the current external Rx queue to external queue.

---
 drivers/net/mlx5/linux/mlx5_os.c |  2 +-
 drivers/net/mlx5/mlx5.h          |  2 +-
 drivers/net/mlx5/mlx5_devx.c     |  2 +-
 drivers/net/mlx5/mlx5_rx.h       |  8 ++++----
 drivers/net/mlx5/mlx5_rxq.c      | 16 ++++++++--------
 5 files changed, 15 insertions(+), 15 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index 6dd12f0f68..8cfbc25430 100644
@@ -1216,7 +1216,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	 */
 	if (mlx5_imported_pd_and_ctx(sh->cdev) && mlx5_devx_obj_ops_en(sh)) {
 		priv->ext_rxqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
-					     sizeof(struct mlx5_external_rxq) *
+					     sizeof(struct mlx5_external_q) *
 					     MLX5_MAX_EXT_RX_QUEUES, 0,
 					     SOCKET_ID_ANY);
 		if (priv->ext_rxqs == NULL) {
drivers/net/mlx5/mlx5.h +1 -1
index 9e4a5feb49..07d050b225 100644
@@ -1882,7 +1882,7 @@ struct mlx5_priv {
 	/* RX/TX queues. */
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
-	struct mlx5_external_rxq *ext_rxqs; /* External RX queues array. */
+	struct mlx5_external_q *ext_rxqs; /* External RX queues array. */
 	struct mlx5_rxq_priv *(*rxq_privs)[]; /* RX queue non-shared data. */
 	struct mlx5_txq_data *(*txqs)[]; /* TX queues. */
 	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
drivers/net/mlx5/mlx5_devx.c +1 -1
index 9fa400fc48..cae9d578ab 100644
@@ -673,7 +673,7 @@ mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev,
 	}
 	for (i = 0; i != queues_n; ++i) {
 		if (mlx5_is_external_rxq(dev, queues[i])) {
-			struct mlx5_external_rxq *ext_rxq =
+			struct mlx5_external_q *ext_rxq =
 					mlx5_ext_rxq_get(dev, queues[i]);
 
 			rqt_attr->rq_list[i] = ext_rxq->hw_id;
drivers/net/mlx5/mlx5_rx.h +4 -4
index d008e4dd3a..decb14e708 100644
@@ -186,7 +186,7 @@ struct mlx5_rxq_priv {
 };
 
 /* External RX queue descriptor. */
-struct mlx5_external_rxq {
+struct mlx5_external_q {
 	uint32_t hw_id; /* Queue index in the Hardware. */
 	RTE_ATOMIC(uint32_t) refcnt; /* Reference counter. */
 };
@@ -227,10 +227,10 @@ uint32_t mlx5_rxq_deref(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_priv *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ctrl *mlx5_rxq_ctrl_get(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_data *mlx5_rxq_data_get(struct rte_eth_dev *dev, uint16_t idx);
-struct mlx5_external_rxq *mlx5_ext_rxq_ref(struct rte_eth_dev *dev,
+struct mlx5_external_q *mlx5_ext_rxq_ref(struct rte_eth_dev *dev,
 					   uint16_t idx);
 uint32_t mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx);
-struct mlx5_external_rxq *mlx5_ext_rxq_get(struct rte_eth_dev *dev,
+struct mlx5_external_q *mlx5_ext_rxq_get(struct rte_eth_dev *dev,
 					   uint16_t idx);
 int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_verify(struct rte_eth_dev *dev);
@@ -661,7 +661,7 @@ static __rte_always_inline bool
 mlx5_is_external_rxq(struct rte_eth_dev *dev, uint16_t queue_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_external_rxq *rxq;
+	struct mlx5_external_q *rxq;
 
 	if (!priv->ext_rxqs || queue_idx < RTE_PMD_MLX5_EXTERNAL_RX_QUEUE_ID_MIN)
 		return false;
drivers/net/mlx5/mlx5_rxq.c +8 -8
index f67aaa6178..d6c84b84e4 100644
@@ -2133,10 +2133,10 @@ mlx5_rxq_data_get(struct rte_eth_dev *dev, uint16_t idx)
  * @return
  *   A pointer to the queue if it exists, NULL otherwise.
  */
-struct mlx5_external_rxq *
+struct mlx5_external_q *
 mlx5_ext_rxq_ref(struct rte_eth_dev *dev, uint16_t idx)
 {
-	struct mlx5_external_rxq *rxq = mlx5_ext_rxq_get(dev, idx);
+	struct mlx5_external_q *rxq = mlx5_ext_rxq_get(dev, idx);
 
 	rte_atomic_fetch_add_explicit(&rxq->refcnt, 1, rte_memory_order_relaxed);
 	return rxq;
@@ -2156,7 +2156,7 @@ mlx5_ext_rxq_ref(struct rte_eth_dev *dev, uint16_t idx)
 uint32_t
 mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx)
 {
-	struct mlx5_external_rxq *rxq = mlx5_ext_rxq_get(dev, idx);
+	struct mlx5_external_q *rxq = mlx5_ext_rxq_get(dev, idx);
 
 	return rte_atomic_fetch_sub_explicit(&rxq->refcnt, 1, rte_memory_order_relaxed) - 1;
 }
@@ -2172,7 +2172,7 @@ mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx)
  * @return
  *   A pointer to the queue if it exists, NULL otherwise.
  */
-struct mlx5_external_rxq *
+struct mlx5_external_q *
 mlx5_ext_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -2336,7 +2336,7 @@ int
 mlx5_ext_rxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_external_rxq *rxq;
+	struct mlx5_external_q *rxq;
 	uint32_t i;
 	int ret = 0;
 
@@ -3206,7 +3206,7 @@ mlx5_rxq_timestamp_set(struct rte_eth_dev *dev)
  *   Pointer to concurrent external RxQ on success,
  *   NULL otherwise and rte_errno is set.
  */
-static struct mlx5_external_rxq *
+static struct mlx5_external_q *
 mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx)
 {
 	struct rte_eth_dev *dev;
@@ -3252,7 +3252,7 @@ int
 rte_pmd_mlx5_external_rx_queue_id_map(uint16_t port_id, uint16_t dpdk_idx,
 				      uint32_t hw_idx)
 {
-	struct mlx5_external_rxq *ext_rxq;
+	struct mlx5_external_q *ext_rxq;
 	uint32_t unmapped = 0;
 
 	ext_rxq = mlx5_external_rx_queue_get_validate(port_id, dpdk_idx);
@@ -3284,7 +3284,7 @@ rte_pmd_mlx5_external_rx_queue_id_map(uint16_t port_id, uint16_t dpdk_idx,
 int
 rte_pmd_mlx5_external_rx_queue_id_unmap(uint16_t port_id, uint16_t dpdk_idx)
 {
-	struct mlx5_external_rxq *ext_rxq;
+	struct mlx5_external_q *ext_rxq;
 	uint32_t mapped = 1;
 
 	ext_rxq = mlx5_external_rx_queue_get_validate(port_id, dpdk_idx);
Signature · 1 line
2.34.1
Acked-by on v1 2/3
#2 lore raw

RE: [PATCH 2/3] net/mlx5: rename external Rx queue to external queue

Quoted 9 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Friday, May 31, 2024 05:51
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH 2/3] net/mlx5: rename external Rx queue to external queue

Quoted 4 lines

Due to external Tx queue will be supported, in order to reuse the external queue struct, rename the current external Rx queue to external queue.

Best regards,
Dariusz Sosnowski

#3 lore raw

PATCH 3/3 net/mlx5: add external Tx queue map and unmap

In reply to Suanming Mou

For using external created Tx queues in RTE_FLOW_ITEM_TX_QUEUE, this commit provides the map and unmap functions to convert the external created SQ's devx ID to DPDK flow item Tx queue ID.

---
 drivers/net/mlx5/linux/mlx5_os.c |  12 ++-
 drivers/net/mlx5/mlx5.c          |   5 +
 drivers/net/mlx5/mlx5.h          |   7 ++
 drivers/net/mlx5/mlx5_defs.h     |   3 +
 drivers/net/mlx5/mlx5_devx.c     |  40 ++++++++
 drivers/net/mlx5/mlx5_devx.h     |   1 +
 drivers/net/mlx5/mlx5_ethdev.c   |   8 ++
 drivers/net/mlx5/mlx5_flow.h     |   6 ++
 drivers/net/mlx5/mlx5_rx.h       |   6 --
 drivers/net/mlx5/mlx5_rxq.c      |  22 +----
 drivers/net/mlx5/mlx5_tx.h       |  25 +++++
 drivers/net/mlx5/mlx5_txq.c      | 152 +++++++++++++++++++++++++++++++
 drivers/net/mlx5/rte_pmd_mlx5.h  |  48 ++++++++++
 drivers/net/mlx5/version.map     |   3 +
 14 files changed, 312 insertions(+), 26 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +11 -1
index 8cfbc25430..bb566ea236 100644
@@ -1224,7 +1224,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOMEM;
 			goto error;
 		}
-		DRV_LOG(DEBUG, "External RxQ is supported.");
+		priv->ext_txqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
+					     sizeof(struct mlx5_external_q) *
+					     MLX5_MAX_EXT_TX_QUEUES, 0,
+					     SOCKET_ID_ANY);
+		if (priv->ext_txqs == NULL) {
+			DRV_LOG(ERR, "Fail to allocate external TxQ array.");
+			err = ENOMEM;
+			goto error;
+		}
+		DRV_LOG(DEBUG, "External queue is supported.");
 	}
 	priv->sh = sh;
 	priv->dev_port = spawn->phys_port;
@@ -1763,6 +1772,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		if (eth_dev && priv->flex_item_map)
 			mlx5_flex_item_port_cleanup(eth_dev);
 		mlx5_free(priv->ext_rxqs);
+		mlx5_free(priv->ext_txqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +5 -0
index d15302d00d..e41b1e82d7 100644
@@ -2436,6 +2436,10 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain",
 			dev->data->port_id);
+	ret = mlx5_ext_txq_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "Port %u some external TxQ still remain.",
+			dev->data->port_id);
 	ret = mlx5_txq_verify(dev);
 	if (ret)
 		DRV_LOG(WARNING, "port %u some Tx queues still remain",
@@ -2447,6 +2451,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (priv->hrxqs)
 		mlx5_list_destroy(priv->hrxqs);
 	mlx5_free(priv->ext_rxqs);
+	mlx5_free(priv->ext_txqs);
 	sh->port[priv->dev_port - 1].nl_ih_port_id = RTE_MAX_ETHPORTS;
 	/*
 	 * The interrupt handler port id must be reset before priv is reset
drivers/net/mlx5/mlx5.h +7 -0
index 07d050b225..5b23043b8b 100644
@@ -381,6 +381,12 @@ struct mlx5_lb_ctx {
 	RTE_ATOMIC(uint16_t) refcnt; /* Reference count for representors. */
 };
 
+/* External queue descriptor. */
+struct mlx5_external_q {
+	uint32_t hw_id; /* Queue index in the Hardware. */
+	RTE_ATOMIC(uint32_t) refcnt; /* Reference counter. */
+};
+
 /* HW steering queue job descriptor type. */
 enum mlx5_hw_job_type {
 	MLX5_HW_Q_JOB_TYPE_CREATE, /* Flow create job type. */
@@ -1883,6 +1889,7 @@ struct mlx5_priv {
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
 	struct mlx5_external_q *ext_rxqs; /* External RX queues array. */
+	struct mlx5_external_q *ext_txqs; /* External TX queues array. */
 	struct mlx5_rxq_priv *(*rxq_privs)[]; /* RX queue non-shared data. */
 	struct mlx5_txq_data *(*txqs)[]; /* TX queues. */
 	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
drivers/net/mlx5/mlx5_defs.h +3 -0
index dc5216cb24..9c454983be 100644
@@ -183,6 +183,9 @@
 /* Maximum number of external Rx queues supported by rte_flow */
 #define MLX5_MAX_EXT_RX_QUEUES (UINT16_MAX - RTE_PMD_MLX5_EXTERNAL_RX_QUEUE_ID_MIN + 1)
 
+/* Maximum number of external Tx queues supported by rte_flow */
+#define MLX5_MAX_EXT_TX_QUEUES (UINT16_MAX - MLX5_EXTERNAL_TX_QUEUE_ID_MIN + 1)
+
 /*
  * Linux definition of static_assert is found in /usr/include/assert.h.
  * Windows does not require a redefinition.
drivers/net/mlx5/mlx5_devx.c +40 -0
index cae9d578ab..f23eb1def6 100644
@@ -27,6 +27,46 @@
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
 
+/**
+ * Validate given external queue's port is valid or not.
+ *
+ * @param[in] port_id
+ *   The port identifier of the Ethernet device.
+ *
+ * @return
+ *   0 on success, non-0 otherwise
+ */
+int
+mlx5_devx_extq_port_validate(uint16_t port_id)
+{
+	struct rte_eth_dev *dev;
+	struct mlx5_priv *priv;
+
+	if (rte_eth_dev_is_valid_port(port_id) < 0) {
+		DRV_LOG(ERR, "There is no Ethernet device for port %u.",
+			port_id);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	dev = &rte_eth_devices[port_id];
+	priv = dev->data->dev_private;
+	if (!mlx5_imported_pd_and_ctx(priv->sh->cdev)) {
+		DRV_LOG(ERR, "Port %u "
+			"external queue isn't supported on local PD and CTX.",
+			port_id);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	if (!mlx5_devx_obj_ops_en(priv->sh)) {
+		DRV_LOG(ERR,
+			"Port %u external queue isn't supported by Verbs API.",
+			port_id);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	return 0;
+}
+
 /**
  * Modify RQ vlan stripping offload
  *
drivers/net/mlx5/mlx5_devx.h +1 -0
index ebd1da455a..4ab8cfbd22 100644
@@ -12,6 +12,7 @@ int mlx5_txq_devx_modify(struct mlx5_txq_obj *obj,
 			 enum mlx5_txq_modify_type type, uint8_t dev_port);
 void mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj);
 int mlx5_devx_modify_rq(struct mlx5_rxq_priv *rxq, uint8_t type);
+int mlx5_devx_extq_port_validate(uint16_t port_id);
 
 extern struct mlx5_obj_ops devx_obj_ops;
 
drivers/net/mlx5/mlx5_ethdev.c +8 -0
index aea799341c..1b721cda5e 100644
@@ -123,6 +123,14 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 			dev->data->port_id, priv->txqs_n, txqs_n);
 		priv->txqs_n = txqs_n;
 	}
+	if (priv->ext_txqs && txqs_n >= MLX5_EXTERNAL_TX_QUEUE_ID_MIN) {
+		DRV_LOG(ERR, "port %u cannot handle this many Tx queues (%u), "
+			"the maximal number of internal Tx queues is %u",
+			dev->data->port_id, txqs_n,
+			MLX5_EXTERNAL_TX_QUEUE_ID_MIN - 1);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
 	if (rxqs_n > priv->sh->dev_cap.ind_table_max_size) {
 		DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)",
 			dev->data->port_id, rxqs_n);
drivers/net/mlx5/mlx5_flow.h +6 -0
index 8f53e82985..9a359da042 100644
@@ -1954,12 +1954,18 @@ static __rte_always_inline int
 flow_hw_get_sqn(struct rte_eth_dev *dev, uint16_t tx_queue, uint32_t *sqn)
 {
 	struct mlx5_txq_ctrl *txq;
+	struct mlx5_external_q *ext_txq;
 
 	/* Means Tx queue is PF0. */
 	if (tx_queue == UINT16_MAX) {
 		*sqn = 0;
 		return 0;
 	}
+	if (mlx5_is_external_txq(dev, tx_queue)) {
+		ext_txq = mlx5_ext_txq_get(dev, tx_queue);
+		*sqn = ext_txq->hw_id;
+		return 0;
+	}
 	txq = mlx5_txq_get(dev, tx_queue);
 	if (unlikely(!txq))
 		return -ENOENT;
drivers/net/mlx5/mlx5_rx.h +0 -6
index decb14e708..1485556d89 100644
@@ -185,12 +185,6 @@ struct mlx5_rxq_priv {
 	uint32_t lwm_devx_subscribed:1;
 };
 
-/* External RX queue descriptor. */
-struct mlx5_external_q {
-	uint32_t hw_id; /* Queue index in the Hardware. */
-	RTE_ATOMIC(uint32_t) refcnt; /* Reference counter. */
-};
-
 /* mlx5_rxq.c */
 
 extern uint8_t rss_hash_default_key[];
drivers/net/mlx5/mlx5_rxq.c +3 -19
index d6c84b84e4..f13fc3b353 100644
@@ -3211,6 +3211,7 @@ mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx)
 {
 	struct rte_eth_dev *dev;
 	struct mlx5_priv *priv;
+	int ret;
 
 	if (dpdk_idx < RTE_PMD_MLX5_EXTERNAL_RX_QUEUE_ID_MIN) {
 		DRV_LOG(ERR, "Queue index %u should be in range: [%u, %u].",
@@ -3218,28 +3219,11 @@ mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx)
 		rte_errno = EINVAL;
 		return NULL;
 	}
-	if (rte_eth_dev_is_valid_port(port_id) < 0) {
-		DRV_LOG(ERR, "There is no Ethernet device for port %u.",
-			port_id);
-		rte_errno = ENODEV;
+	ret = mlx5_devx_extq_port_validate(port_id);
+	if (unlikely(ret))
 		return NULL;
-	}
 	dev = &rte_eth_devices[port_id];
 	priv = dev->data->dev_private;
-	if (!mlx5_imported_pd_and_ctx(priv->sh->cdev)) {
-		DRV_LOG(ERR, "Port %u "
-			"external RxQ isn't supported on local PD and CTX.",
-			port_id);
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	if (!mlx5_devx_obj_ops_en(priv->sh)) {
-		DRV_LOG(ERR,
-			"Port %u external RxQ isn't supported by Verbs API.",
-			port_id);
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	/*
 	 * When user configures remote PD and CTX and device creates RxQ by
 	 * DevX, external RxQs array is allocated.
drivers/net/mlx5/mlx5_tx.h +25 -0
index 0d77ff89de..983913faa2 100644
@@ -227,6 +227,8 @@ void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
 int mlx5_count_aggr_ports(struct rte_eth_dev *dev);
 int mlx5_map_aggr_tx_affinity(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 			      uint8_t affinity);
+int mlx5_ext_txq_verify(struct rte_eth_dev *dev);
+struct mlx5_external_q *mlx5_ext_txq_get(struct rte_eth_dev *dev, uint16_t idx);
 
 /* mlx5_tx.c */
 
@@ -3788,4 +3790,27 @@ mlx5_tx_burst_tmpl(struct mlx5_txq_data *__rte_restrict txq,
 	return loc.pkts_sent;
 }
 
+/**
+ * Check whether given TxQ is external.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param queue_idx
+ *   Tx queue index.
+ *
+ * @return
+ *   True if is external TxQ, otherwise false.
+ */
+static __rte_always_inline bool
+mlx5_is_external_txq(struct rte_eth_dev *dev, uint16_t queue_idx)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_external_q *txq;
+
+	if (!priv->ext_txqs || queue_idx < MLX5_EXTERNAL_TX_QUEUE_ID_MIN)
+		return false;
+	txq = &priv->ext_txqs[queue_idx - MLX5_EXTERNAL_TX_QUEUE_ID_MIN];
+	return !!rte_atomic_load_explicit(&txq->refcnt, rte_memory_order_relaxed);
+}
+
 #endif /* RTE_PMD_MLX5_TX_H_ */
drivers/net/mlx5/mlx5_txq.c +152 -0

167 lines not on this page · raw message

drivers/net/mlx5/rte_pmd_mlx5.h +48 -0

60 lines not on this page · raw message

drivers/net/mlx5/version.map +3 -0
index 8fb0e07303..8a78d14786 100644
@@ -20,4 +20,7 @@ EXPERIMENTAL {
 	# added in 24.03
 	rte_pmd_mlx5_create_geneve_tlv_parser;
 	rte_pmd_mlx5_destroy_geneve_tlv_parser;
+	# added in 24.07
+	rte_pmd_mlx5_external_tx_queue_id_map;
+	rte_pmd_mlx5_external_tx_queue_id_unmap;
 };
Signature · 1 line
2.34.1
#4 lore raw

RE: [PATCH 3/3] net/mlx5: add external Tx queue map and unmap

Quoted 9 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Friday, May 31, 2024 05:51
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH 3/3] net/mlx5: add external Tx queue map and unmap

Quoted 5 lines

For using external created Tx queues in RTE_FLOW_ITEM_TX_QUEUE, this commit provides the map and unmap functions to convert the external created SQ's devx ID to DPDK flow item Tx queue ID.

Could you please add external Tx queue feature to release notes and mlx5 PMD docs?

Best regards,
Dariusz Sosnowski

#5 lore raw

RE: [PATCH 3/3] net/mlx5: add external Tx queue map and unmap

Hi,

Quoted 18 lines

-----Original Message-----
From: Dariusz Sosnowski <[email protected]>
Sent: Wednesday, June 5, 2024 4:17 PM
To: Suanming Mou <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: RE: [PATCH 3/3] net/mlx5: add external Tx queue map and unmap

Quoted 9 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Friday, May 31, 2024 05:51
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH 3/3] net/mlx5: add external Tx queue map and unmap

Quoted 8 lines
Quoted 5 lines

For using external created Tx queues in RTE_FLOW_ITEM_TX_QUEUE, this commit provides the map and unmap functions to convert the external created SQ's devx ID to DPDK flow item Tx queue ID.

Could you please add external Tx queue feature to release notes and mlx5 PMD docs?

Sure.

Quoted 3 lines

Best regards,
Dariusz Sosnowski

Acked-by on v1 1/3
#6 lore raw
In reply to Suanming Mou
Quoted 9 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Friday, May 31, 2024 05:51
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH 1/3] net/mlx5: add match with Tx queue item

Quoted 6 lines

With the item RTE_FLOW_ITEM_TYPE_TX_QUEUE, user will be able to set the Tx queue index and create flow match with that queue index.

This commit adds match with RTE_FLOW_ITEM_TX_QUEUE item.

Best regards,
Dariusz Sosnowski

#7 lore raw

PATCH v2 1/3 net/mlx5: add match with Tx queue item

In reply to Suanming Mou

With the item RTE_FLOW_ITEM_TYPE_TX_QUEUE, user will be able to set the Tx queue index and create flow match with that queue index.

This commit adds match with RTE_FLOW_ITEM_TX_QUEUE item.

---
 doc/guides/nics/features/mlx5.ini      |  1 +
 doc/guides/rel_notes/release_24_07.rst |  4 ++
 drivers/net/mlx5/hws/mlx5dr_definer.c  | 50 +++++++++++++++++++
 drivers/net/mlx5/mlx5_flow.h           | 58 ++++++++++++++++++++++
 drivers/net/mlx5/mlx5_flow_dv.c        | 69 ++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_flow_hw.c        |  1 +
 6 files changed, 183 insertions(+)
doc/guides/nics/features/mlx5.ini +1 -0
index 81a7067cc3..056e04275b 100644
@@ -92,6 +92,7 @@ quota                = Y
 random               = Y
 tag                  = Y
 tcp                  = Y
+tx_queue             = Y
 udp                  = Y
 vlan                 = Y
 vxlan                = Y
doc/guides/rel_notes/release_24_07.rst +4 -0
index ffbe9ce051..46efc04eac 100644
@@ -81,6 +81,10 @@ New Features
 
   * Added SSE/NEON vector datapath.
 
+* **Updated NVIDIA mlx5 driver.**
+
+  * Added match with Tx queue.
+
 
 Removed Items
 -------------
drivers/net/mlx5/hws/mlx5dr_definer.c +50 -0

74 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +58 -0

70 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +69 -0

93 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_hw.c +1 -0
index 63194935a3..cf698b3ec8 100644
@@ -8018,6 +8018,7 @@ flow_hw_pattern_validate(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_MPLS:
 		case RTE_FLOW_ITEM_TYPE_GENEVE:
 		case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
+		case RTE_FLOW_ITEM_TYPE_TX_QUEUE:
 		case RTE_FLOW_ITEM_TYPE_GRE:
 		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
Signature · 1 line
2.34.1
#8 lore raw

PATCH v2 2/3 net/mlx5: rename external Rx queue to external queue

Due to external Tx queue will be supported, in order to reuse the external queue struct, rename the current external Rx queue to external queue.

---
 drivers/net/mlx5/linux/mlx5_os.c |  2 +-
 drivers/net/mlx5/mlx5.h          |  2 +-
 drivers/net/mlx5/mlx5_devx.c     |  2 +-
 drivers/net/mlx5/mlx5_rx.h       |  8 ++++----
 drivers/net/mlx5/mlx5_rxq.c      | 16 ++++++++--------
 5 files changed, 15 insertions(+), 15 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index de3df17108..99de52936a 100644
@@ -1216,7 +1216,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	 */
 	if (mlx5_imported_pd_and_ctx(sh->cdev) && mlx5_devx_obj_ops_en(sh)) {
 		priv->ext_rxqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
-					     sizeof(struct mlx5_external_rxq) *
+					     sizeof(struct mlx5_external_q) *
 					     MLX5_MAX_EXT_RX_QUEUES, 0,
 					     SOCKET_ID_ANY);
 		if (priv->ext_rxqs == NULL) {
drivers/net/mlx5/mlx5.h +1 -1
index e2c22ffe97..e85308f6e0 100644
@@ -1882,7 +1882,7 @@ struct mlx5_priv {
 	/* RX/TX queues. */
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
-	struct mlx5_external_rxq *ext_rxqs; /* External RX queues array. */
+	struct mlx5_external_q *ext_rxqs; /* External RX queues array. */
 	struct mlx5_rxq_priv *(*rxq_privs)[]; /* RX queue non-shared data. */
 	struct mlx5_txq_data *(*txqs)[]; /* TX queues. */
 	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
drivers/net/mlx5/mlx5_devx.c +1 -1
index 9fa400fc48..cae9d578ab 100644
@@ -673,7 +673,7 @@ mlx5_devx_ind_table_create_rqt_attr(struct rte_eth_dev *dev,
 	}
 	for (i = 0; i != queues_n; ++i) {
 		if (mlx5_is_external_rxq(dev, queues[i])) {
-			struct mlx5_external_rxq *ext_rxq =
+			struct mlx5_external_q *ext_rxq =
 					mlx5_ext_rxq_get(dev, queues[i]);
 
 			rqt_attr->rq_list[i] = ext_rxq->hw_id;
drivers/net/mlx5/mlx5_rx.h +4 -4
index d008e4dd3a..decb14e708 100644
@@ -186,7 +186,7 @@ struct mlx5_rxq_priv {
 };
 
 /* External RX queue descriptor. */
-struct mlx5_external_rxq {
+struct mlx5_external_q {
 	uint32_t hw_id; /* Queue index in the Hardware. */
 	RTE_ATOMIC(uint32_t) refcnt; /* Reference counter. */
 };
@@ -227,10 +227,10 @@ uint32_t mlx5_rxq_deref(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_priv *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_ctrl *mlx5_rxq_ctrl_get(struct rte_eth_dev *dev, uint16_t idx);
 struct mlx5_rxq_data *mlx5_rxq_data_get(struct rte_eth_dev *dev, uint16_t idx);
-struct mlx5_external_rxq *mlx5_ext_rxq_ref(struct rte_eth_dev *dev,
+struct mlx5_external_q *mlx5_ext_rxq_ref(struct rte_eth_dev *dev,
 					   uint16_t idx);
 uint32_t mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx);
-struct mlx5_external_rxq *mlx5_ext_rxq_get(struct rte_eth_dev *dev,
+struct mlx5_external_q *mlx5_ext_rxq_get(struct rte_eth_dev *dev,
 					   uint16_t idx);
 int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
 int mlx5_rxq_verify(struct rte_eth_dev *dev);
@@ -661,7 +661,7 @@ static __rte_always_inline bool
 mlx5_is_external_rxq(struct rte_eth_dev *dev, uint16_t queue_idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_external_rxq *rxq;
+	struct mlx5_external_q *rxq;
 
 	if (!priv->ext_rxqs || queue_idx < RTE_PMD_MLX5_EXTERNAL_RX_QUEUE_ID_MIN)
 		return false;
drivers/net/mlx5/mlx5_rxq.c +8 -8
index f67aaa6178..d6c84b84e4 100644
@@ -2133,10 +2133,10 @@ mlx5_rxq_data_get(struct rte_eth_dev *dev, uint16_t idx)
  * @return
  *   A pointer to the queue if it exists, NULL otherwise.
  */
-struct mlx5_external_rxq *
+struct mlx5_external_q *
 mlx5_ext_rxq_ref(struct rte_eth_dev *dev, uint16_t idx)
 {
-	struct mlx5_external_rxq *rxq = mlx5_ext_rxq_get(dev, idx);
+	struct mlx5_external_q *rxq = mlx5_ext_rxq_get(dev, idx);
 
 	rte_atomic_fetch_add_explicit(&rxq->refcnt, 1, rte_memory_order_relaxed);
 	return rxq;
@@ -2156,7 +2156,7 @@ mlx5_ext_rxq_ref(struct rte_eth_dev *dev, uint16_t idx)
 uint32_t
 mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx)
 {
-	struct mlx5_external_rxq *rxq = mlx5_ext_rxq_get(dev, idx);
+	struct mlx5_external_q *rxq = mlx5_ext_rxq_get(dev, idx);
 
 	return rte_atomic_fetch_sub_explicit(&rxq->refcnt, 1, rte_memory_order_relaxed) - 1;
 }
@@ -2172,7 +2172,7 @@ mlx5_ext_rxq_deref(struct rte_eth_dev *dev, uint16_t idx)
  * @return
  *   A pointer to the queue if it exists, NULL otherwise.
  */
-struct mlx5_external_rxq *
+struct mlx5_external_q *
 mlx5_ext_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -2336,7 +2336,7 @@ int
 mlx5_ext_rxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_external_rxq *rxq;
+	struct mlx5_external_q *rxq;
 	uint32_t i;
 	int ret = 0;
 
@@ -3206,7 +3206,7 @@ mlx5_rxq_timestamp_set(struct rte_eth_dev *dev)
  *   Pointer to concurrent external RxQ on success,
  *   NULL otherwise and rte_errno is set.
  */
-static struct mlx5_external_rxq *
+static struct mlx5_external_q *
 mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx)
 {
 	struct rte_eth_dev *dev;
@@ -3252,7 +3252,7 @@ int
 rte_pmd_mlx5_external_rx_queue_id_map(uint16_t port_id, uint16_t dpdk_idx,
 				      uint32_t hw_idx)
 {
-	struct mlx5_external_rxq *ext_rxq;
+	struct mlx5_external_q *ext_rxq;
 	uint32_t unmapped = 0;
 
 	ext_rxq = mlx5_external_rx_queue_get_validate(port_id, dpdk_idx);
@@ -3284,7 +3284,7 @@ rte_pmd_mlx5_external_rx_queue_id_map(uint16_t port_id, uint16_t dpdk_idx,
 int
 rte_pmd_mlx5_external_rx_queue_id_unmap(uint16_t port_id, uint16_t dpdk_idx)
 {
-	struct mlx5_external_rxq *ext_rxq;
+	struct mlx5_external_q *ext_rxq;
 	uint32_t mapped = 1;
 
 	ext_rxq = mlx5_external_rx_queue_get_validate(port_id, dpdk_idx);
Signature · 1 line
2.34.1
#9 lore raw

PATCH v2 3/3 net/mlx5: add external Tx queue map and unmap

In reply to Suanming Mou

For using external created Tx queues in RTE_FLOW_ITEM_TX_QUEUE, this commit provides the map and unmap functions to convert the external created SQ's devx ID to DPDK flow item Tx queue ID.

---

v2: add feature and release notes.

---
 doc/guides/nics/mlx5.rst               |   1 +
 doc/guides/rel_notes/release_24_07.rst |   1 +
 drivers/net/mlx5/linux/mlx5_os.c       |  12 +-
 drivers/net/mlx5/mlx5.c                |   5 +
 drivers/net/mlx5/mlx5.h                |   7 ++
 drivers/net/mlx5/mlx5_defs.h           |   3 +
 drivers/net/mlx5/mlx5_devx.c           |  40 +++++++
 drivers/net/mlx5/mlx5_devx.h           |   1 +
 drivers/net/mlx5/mlx5_ethdev.c         |   8 ++
 drivers/net/mlx5/mlx5_flow.h           |   6 +
 drivers/net/mlx5/mlx5_rx.h             |   6 -
 drivers/net/mlx5/mlx5_rxq.c            |  22 +---
 drivers/net/mlx5/mlx5_tx.h             |  25 ++++
 drivers/net/mlx5/mlx5_txq.c            | 152 +++++++++++++++++++++++++
 drivers/net/mlx5/rte_pmd_mlx5.h        |  48 ++++++++
 drivers/net/mlx5/version.map           |   3 +
 16 files changed, 314 insertions(+), 26 deletions(-)
doc/guides/nics/mlx5.rst +1 -0
index b5928d40b2..5cd41d3c7f 100644
@@ -169,6 +169,7 @@ Features
 - Sub-Function.
 - Matching on represented port.
 - Matching on aggregated affinity.
+- Matching on external Tx queue.
 
 
 Limitations
doc/guides/rel_notes/release_24_07.rst +1 -0
index 46efc04eac..3a3257fcd5 100644
@@ -84,6 +84,7 @@ New Features
 * **Updated NVIDIA mlx5 driver.**
 
   * Added match with Tx queue.
+  * Added match with external Tx queue.
 
 
 Removed Items
drivers/net/mlx5/linux/mlx5_os.c +11 -1
index 99de52936a..f887501a9b 100644
@@ -1224,7 +1224,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOMEM;
 			goto error;
 		}
-		DRV_LOG(DEBUG, "External RxQ is supported.");
+		priv->ext_txqs = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
+					     sizeof(struct mlx5_external_q) *
+					     MLX5_MAX_EXT_TX_QUEUES, 0,
+					     SOCKET_ID_ANY);
+		if (priv->ext_txqs == NULL) {
+			DRV_LOG(ERR, "Fail to allocate external TxQ array.");
+			err = ENOMEM;
+			goto error;
+		}
+		DRV_LOG(DEBUG, "External queue is supported.");
 	}
 	priv->sh = sh;
 	priv->dev_port = spawn->phys_port;
@@ -1762,6 +1771,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		if (eth_dev && priv->flex_item_map)
 			mlx5_flex_item_port_cleanup(eth_dev);
 		mlx5_free(priv->ext_rxqs);
+		mlx5_free(priv->ext_txqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +5 -0
index d15302d00d..e41b1e82d7 100644
@@ -2436,6 +2436,10 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain",
 			dev->data->port_id);
+	ret = mlx5_ext_txq_verify(dev);
+	if (ret)
+		DRV_LOG(WARNING, "Port %u some external TxQ still remain.",
+			dev->data->port_id);
 	ret = mlx5_txq_verify(dev);
 	if (ret)
 		DRV_LOG(WARNING, "port %u some Tx queues still remain",
@@ -2447,6 +2451,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (priv->hrxqs)
 		mlx5_list_destroy(priv->hrxqs);
 	mlx5_free(priv->ext_rxqs);
+	mlx5_free(priv->ext_txqs);
 	sh->port[priv->dev_port - 1].nl_ih_port_id = RTE_MAX_ETHPORTS;
 	/*
 	 * The interrupt handler port id must be reset before priv is reset
drivers/net/mlx5/mlx5.h +7 -0
index e85308f6e0..91ceceb34a 100644
@@ -381,6 +381,12 @@ struct mlx5_lb_ctx {
 	RTE_ATOMIC(uint16_t) refcnt; /* Reference count for representors. */
 };
 
+/* External queue descriptor. */
+struct mlx5_external_q {
+	uint32_t hw_id; /* Queue index in the Hardware. */
+	RTE_ATOMIC(uint32_t) refcnt; /* Reference counter. */
+};
+
 /* HW steering queue job descriptor type. */
 enum mlx5_hw_job_type {
 	MLX5_HW_Q_JOB_TYPE_CREATE, /* Flow create job type. */
@@ -1883,6 +1889,7 @@ struct mlx5_priv {
 	unsigned int rxqs_n; /* RX queues array size. */
 	unsigned int txqs_n; /* TX queues array size. */
 	struct mlx5_external_q *ext_rxqs; /* External RX queues array. */
+	struct mlx5_external_q *ext_txqs; /* External TX queues array. */
 	struct mlx5_rxq_priv *(*rxq_privs)[]; /* RX queue non-shared data. */
 	struct mlx5_txq_data *(*txqs)[]; /* TX queues. */
 	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */
drivers/net/mlx5/mlx5_defs.h +3 -0
index dc5216cb24..9c454983be 100644
@@ -183,6 +183,9 @@
 /* Maximum number of external Rx queues supported by rte_flow */
 #define MLX5_MAX_EXT_RX_QUEUES (UINT16_MAX - RTE_PMD_MLX5_EXTERNAL_RX_QUEUE_ID_MIN + 1)
 
+/* Maximum number of external Tx queues supported by rte_flow */
+#define MLX5_MAX_EXT_TX_QUEUES (UINT16_MAX - MLX5_EXTERNAL_TX_QUEUE_ID_MIN + 1)
+
 /*
  * Linux definition of static_assert is found in /usr/include/assert.h.
  * Windows does not require a redefinition.
drivers/net/mlx5/mlx5_devx.c +40 -0
index cae9d578ab..f23eb1def6 100644
@@ -27,6 +27,46 @@
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
 
+/**
+ * Validate given external queue's port is valid or not.
+ *
+ * @param[in] port_id
+ *   The port identifier of the Ethernet device.
+ *
+ * @return
+ *   0 on success, non-0 otherwise
+ */
+int
+mlx5_devx_extq_port_validate(uint16_t port_id)
+{
+	struct rte_eth_dev *dev;
+	struct mlx5_priv *priv;
+
+	if (rte_eth_dev_is_valid_port(port_id) < 0) {
+		DRV_LOG(ERR, "There is no Ethernet device for port %u.",
+			port_id);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	dev = &rte_eth_devices[port_id];
+	priv = dev->data->dev_private;
+	if (!mlx5_imported_pd_and_ctx(priv->sh->cdev)) {
+		DRV_LOG(ERR, "Port %u "
+			"external queue isn't supported on local PD and CTX.",
+			port_id);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	if (!mlx5_devx_obj_ops_en(priv->sh)) {
+		DRV_LOG(ERR,
+			"Port %u external queue isn't supported by Verbs API.",
+			port_id);
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	return 0;
+}
+
 /**
  * Modify RQ vlan stripping offload
  *
drivers/net/mlx5/mlx5_devx.h +1 -0
index ebd1da455a..4ab8cfbd22 100644
@@ -12,6 +12,7 @@ int mlx5_txq_devx_modify(struct mlx5_txq_obj *obj,
 			 enum mlx5_txq_modify_type type, uint8_t dev_port);
 void mlx5_txq_devx_obj_release(struct mlx5_txq_obj *txq_obj);
 int mlx5_devx_modify_rq(struct mlx5_rxq_priv *rxq, uint8_t type);
+int mlx5_devx_extq_port_validate(uint16_t port_id);
 
 extern struct mlx5_obj_ops devx_obj_ops;
 
drivers/net/mlx5/mlx5_ethdev.c +8 -0
index aea799341c..1b721cda5e 100644
@@ -123,6 +123,14 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
 			dev->data->port_id, priv->txqs_n, txqs_n);
 		priv->txqs_n = txqs_n;
 	}
+	if (priv->ext_txqs && txqs_n >= MLX5_EXTERNAL_TX_QUEUE_ID_MIN) {
+		DRV_LOG(ERR, "port %u cannot handle this many Tx queues (%u), "
+			"the maximal number of internal Tx queues is %u",
+			dev->data->port_id, txqs_n,
+			MLX5_EXTERNAL_TX_QUEUE_ID_MIN - 1);
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
 	if (rxqs_n > priv->sh->dev_cap.ind_table_max_size) {
 		DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)",
 			dev->data->port_id, rxqs_n);
drivers/net/mlx5/mlx5_flow.h +6 -0
index 357267e0c3..ba75b99139 100644
@@ -1954,12 +1954,18 @@ static __rte_always_inline int
 flow_hw_get_sqn(struct rte_eth_dev *dev, uint16_t tx_queue, uint32_t *sqn)
 {
 	struct mlx5_txq_ctrl *txq;
+	struct mlx5_external_q *ext_txq;
 
 	/* Means Tx queue is PF0. */
 	if (tx_queue == UINT16_MAX) {
 		*sqn = 0;
 		return 0;
 	}
+	if (mlx5_is_external_txq(dev, tx_queue)) {
+		ext_txq = mlx5_ext_txq_get(dev, tx_queue);
+		*sqn = ext_txq->hw_id;
+		return 0;
+	}
 	txq = mlx5_txq_get(dev, tx_queue);
 	if (unlikely(!txq))
 		return -ENOENT;
drivers/net/mlx5/mlx5_rx.h +0 -6
index decb14e708..1485556d89 100644
@@ -185,12 +185,6 @@ struct mlx5_rxq_priv {
 	uint32_t lwm_devx_subscribed:1;
 };
 
-/* External RX queue descriptor. */
-struct mlx5_external_q {
-	uint32_t hw_id; /* Queue index in the Hardware. */
-	RTE_ATOMIC(uint32_t) refcnt; /* Reference counter. */
-};
-
 /* mlx5_rxq.c */
 
 extern uint8_t rss_hash_default_key[];
drivers/net/mlx5/mlx5_rxq.c +3 -19
index d6c84b84e4..f13fc3b353 100644
@@ -3211,6 +3211,7 @@ mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx)
 {
 	struct rte_eth_dev *dev;
 	struct mlx5_priv *priv;
+	int ret;
 
 	if (dpdk_idx < RTE_PMD_MLX5_EXTERNAL_RX_QUEUE_ID_MIN) {
 		DRV_LOG(ERR, "Queue index %u should be in range: [%u, %u].",
@@ -3218,28 +3219,11 @@ mlx5_external_rx_queue_get_validate(uint16_t port_id, uint16_t dpdk_idx)
 		rte_errno = EINVAL;
 		return NULL;
 	}
-	if (rte_eth_dev_is_valid_port(port_id) < 0) {
-		DRV_LOG(ERR, "There is no Ethernet device for port %u.",
-			port_id);
-		rte_errno = ENODEV;
+	ret = mlx5_devx_extq_port_validate(port_id);
+	if (unlikely(ret))
 		return NULL;
-	}
 	dev = &rte_eth_devices[port_id];
 	priv = dev->data->dev_private;
-	if (!mlx5_imported_pd_and_ctx(priv->sh->cdev)) {
-		DRV_LOG(ERR, "Port %u "
-			"external RxQ isn't supported on local PD and CTX.",
-			port_id);
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
-	if (!mlx5_devx_obj_ops_en(priv->sh)) {
-		DRV_LOG(ERR,
-			"Port %u external RxQ isn't supported by Verbs API.",
-			port_id);
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	/*
 	 * When user configures remote PD and CTX and device creates RxQ by
 	 * DevX, external RxQs array is allocated.
drivers/net/mlx5/mlx5_tx.h +25 -0
index 0d77ff89de..983913faa2 100644
@@ -227,6 +227,8 @@ void mlx5_txq_dynf_timestamp_set(struct rte_eth_dev *dev);
 int mlx5_count_aggr_ports(struct rte_eth_dev *dev);
 int mlx5_map_aggr_tx_affinity(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 			      uint8_t affinity);
+int mlx5_ext_txq_verify(struct rte_eth_dev *dev);
+struct mlx5_external_q *mlx5_ext_txq_get(struct rte_eth_dev *dev, uint16_t idx);
 
 /* mlx5_tx.c */
 
@@ -3788,4 +3790,27 @@ mlx5_tx_burst_tmpl(struct mlx5_txq_data *__rte_restrict txq,
 	return loc.pkts_sent;
 }
 
+/**
+ * Check whether given TxQ is external.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param queue_idx
+ *   Tx queue index.
+ *
+ * @return
+ *   True if is external TxQ, otherwise false.
+ */
+static __rte_always_inline bool
+mlx5_is_external_txq(struct rte_eth_dev *dev, uint16_t queue_idx)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_external_q *txq;
+
+	if (!priv->ext_txqs || queue_idx < MLX5_EXTERNAL_TX_QUEUE_ID_MIN)
+		return false;
+	txq = &priv->ext_txqs[queue_idx - MLX5_EXTERNAL_TX_QUEUE_ID_MIN];
+	return !!rte_atomic_load_explicit(&txq->refcnt, rte_memory_order_relaxed);
+}
+
 #endif /* RTE_PMD_MLX5_TX_H_ */
drivers/net/mlx5/mlx5_txq.c +152 -0

167 lines not on this page · raw message

drivers/net/mlx5/rte_pmd_mlx5.h +48 -0

60 lines not on this page · raw message

drivers/net/mlx5/version.map +3 -0
index 8fb0e07303..8a78d14786 100644
@@ -20,4 +20,7 @@ EXPERIMENTAL {
 	# added in 24.03
 	rte_pmd_mlx5_create_geneve_tlv_parser;
 	rte_pmd_mlx5_destroy_geneve_tlv_parser;
+	# added in 24.07
+	rte_pmd_mlx5_external_tx_queue_id_map;
+	rte_pmd_mlx5_external_tx_queue_id_unmap;
 };
Signature · 1 line
2.34.1
Acked-by on v2 3/3
#10 lore raw

RE: [PATCH v2 3/3] net/mlx5: add external Tx queue map and unmap

Quoted 9 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Wednesday, June 5, 2024 11:32
To: Dariusz Sosnowski <[email protected]>; Slava Ovsiienko <[email protected]>; Ori Kam <[email protected]>; Matan Azrad <[email protected]>
Cc: [email protected]; Raslan Darawsheh <[email protected]>
Subject: [PATCH v2 3/3] net/mlx5: add external Tx queue map and unmap

Quoted 5 lines

For using external created Tx queues in RTE_FLOW_ITEM_TX_QUEUE, this commit provides the map and unmap functions to convert the external created SQ's devx ID to DPDK flow item Tx queue ID.

Best regards,
Dariusz Sosnowski

Acked-by Dariusz Sosnowski on v2 1/3
#11 lore raw
In reply to Suanming Mou

Hi,

From: Suanming Mou <[email protected]>
Sent: Wednesday, June 5, 2024 12:31 PM
To: Dariusz Sosnowski; Slava Ovsiienko; Ori Kam; Matan Azrad
Cc: [email protected]; Raslan Darawsheh
Subject: [PATCH v2 1/3] net/mlx5: add match with Tx queue item

With the item RTE_FLOW_ITEM_TYPE_TX_QUEUE, user will be able to set the Tx queue index and create flow match with that queue index.

This commit adds match with RTE_FLOW_ITEM_TX_QUEUE item.

Series rebased and applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh