common/mlx5: add device duplication function

From: Michael Baum <[email protected]>

The GENEVE parser is shared across ports on the same physical device. It is created once for the first port and increments a reference counter for each additional port. The parser is created using the InfiniBand
(IBV) context (CTX) of the first port, and cleanup should use the same context.

Previously, if the port owning the context closed while another port still used the parser, the port closure would fail due to the shared parser dependency.

This patch addresses the issue by changing the approach: the physical device now creates its own distinct context by importing the context from the first initialized port. This ensures that parser creation and cleanup use a consistent context, independent of individual port lifecycles.

Fixes f5177bdc8b76 ("net/mlx5: add GENEVE TLV options parser API")

Cc [email protected]

Signed-off-by Michael Baum <[email protected]>

---
 drivers/net/mlx5/mlx5.c             | 21 ++++++------
 drivers/net/mlx5/mlx5.h             |  2 +-
 drivers/net/mlx5/mlx5_flow.h        |  2 --
 drivers/net/mlx5/mlx5_flow_geneve.c | 50 +++--------------------------
 4 files changed, 18 insertions(+), 57 deletions(-)

drivers/net/mlx5/mlx5.c index renamed from drivers/net/mlx5/mlx5.c +0 -0

0f49cb5e5b..a16568ff22 100644

drivers/net/mlx5/mlx5.c +2 -0

@@ -1724,6 +1724,16 @@ mlx5_get_physical_device(struct
 mlx5_common_device *cdev)
 		rte_errno = ENOMEM;
 		return NULL;
 	}
+	/*
+	 * The same CTX create the physical device objects should destroy

them.

drivers/net/mlx5/mlx5.c +8 -0

+	 * Since we can't be sure it will be done by same CTX, we prepare for
+	 * the physical device a special CTX used by objects creation.
+	 */
+	phdev->ctx = mlx5_os_get_physical_device_ctx(cdev);
+	if (!phdev->ctx) {
+		mlx5_free(phdev);
+		return NULL;
+	}
 	phdev->guid = attr->system_image_guid;
 	phdev->refcnt = 1;
 	LIST_INSERT_HEAD(&phdev_list, phdev, next); @@ -1767,6 +1777,8

@@ mlx5_physical_device_destroy(struct mlx5_physical_device *phdev)

drivers/net/mlx5/mlx5.c +2 -9

 		return;
 	/* Remove physical device from the global device list. */
 	LIST_REMOVE(phdev, next);
+	MLX5_ASSERT(phdev->ctx);
+	claim_zero(mlx5_glue->close_device(phdev->ctx));
 	mlx5_free(phdev);
 }

@@ -2323,15 +2335,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 		rte_errno = EBUSY;
 		return -EBUSY;
 	}
-#ifdef HAVE_MLX5_HWS_SUPPORT
-	/* Check if shared GENEVE options created on context being closed. */
-	ret = mlx5_geneve_tlv_options_check_busy(priv);
-	if (ret) {
-		DRV_LOG(ERR, "port %u maintains shared GENEVE TLV
 options",
-			dev->data->port_id);
-		return ret;
-	}
-#endif
 	DRV_LOG(DEBUG, "port %u closing device \"%s\"",
 		dev->data->port_id, sh->ibdev_name);

drivers/net/mlx5/mlx5.c +0 -0

 	/*

drivers/net/mlx5/mlx5.h index renamed from drivers/net/mlx5/mlx5.h +0 -0

6df99c25e2..0194887a8b 100644

drivers/net/mlx5/mlx5.h +1 -1

@@ -1527,7 +1527,7 @@ struct mlx5_common_nic_config {
  */
 struct mlx5_physical_device {
 	LIST_ENTRY(mlx5_physical_device) next;
-	struct mlx5_dev_ctx_shared *sh; /* Created on sherd context. */
+	void *ctx; /* CTX for creation of options. */
 	uint64_t guid; /* System image guid, the uniq ID of physical device. */
 	struct mlx5_geneve_tlv_options *tlv_options;
 	struct mlx5_common_nic_config config;

drivers/net/mlx5/mlx5_flow.h +0 -0

index a5bde158ca..9816ed9238 100644
@@ -3469,8 +3469,6 @@ flow_hw_resource_release(struct rte_eth_dev
 *dev);  int  mlx5_geneve_tlv_options_destroy(struct mlx5_geneve_tlv_options
 *options,
 				struct mlx5_physical_device *phdev); -int -

mlx5_geneve_tlv_options_check_busy(struct mlx5_priv *priv); void flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable); int flow_dv_action_validate(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_geneve.c b/drivers/net/mlx5/mlx5_flow_geneve.c
index 6bf53e1270..4d57bb763f 100644

drivers/net/mlx5/mlx5_flow_geneve.c +2 -2

@@ -590,8 +590,8 @@ mlx5_geneve_tlv_option_copy(struct
 rte_pmd_mlx5_geneve_tlv *dst,
 /**
  * Create list of GENEVE TLV options according to user configuration list.
  *
- * @param sh
- *   Shared context the options are being created on.
+ * @param ctx
+ *   Context returned from mlx5 open_device() glue function.
  * @param tlv_list
  *   A list of GENEVE TLV options to create parser for them.

* @param nb_options

drivers/net/mlx5/mlx5_flow_geneve.c +1 -0

@@ -604,7 +604,7 @@ mlx5_geneve_tlv_option_copy(struct
 rte_pmd_mlx5_geneve_tlv *dst,
  *   NULL otherwise and rte_errno is set.
  */
 static struct mlx5_geneve_tlv_options * -
 mlx5_geneve_tlv_options_create(struct mlx5_dev_ctx_shared *sh,
+mlx5_geneve_tlv_options_create(void *ctx,
 			       const struct rte_pmd_mlx5_geneve_tlv tlv_list[],
 			       uint8_t nb_options, uint8_t sample_id)  { @@ -

625,7 +625,7 @@ mlx5_geneve_tlv_options_create(struct mlx5_dev_ctx_shared *sh,

drivers/net/mlx5/mlx5_flow_geneve.c +1 -1

 	}
 	for (i = 0; i < nb_options; ++i) {
 		spec = &tlv_list[i];
-		ret = mlx5_geneve_tlv_option_create(sh->cdev->ctx, spec,
+		ret = mlx5_geneve_tlv_option_create(ctx, spec,
 						    &options->options[i],

sample_id);

drivers/net/mlx5/mlx5_flow_geneve.c +0 -2

 		if (ret < 0)
 			goto error;

@@ -633,8 +633,6 @@ mlx5_geneve_tlv_options_create(struct
 mlx5_dev_ctx_shared *sh,
 		data_mask = options->buffer + i *
 MAX_GENEVE_OPTION_DATA_SIZE;
 		mlx5_geneve_tlv_option_copy(&options->spec[i], spec,
 data_mask);
 	}
-	MLX5_ASSERT(sh->phdev->sh == NULL);
-	sh->phdev->sh = sh;

drivers/net/mlx5/mlx5_flow_geneve.c +2 -39

 	options->nb_options = nb_options;
 	options->refcnt = 1;
 	return options;

@@ -676,41 +674,9 @@ mlx5_geneve_tlv_options_destroy(struct
 mlx5_geneve_tlv_options *options,
 	}
 	mlx5_free(options);
 	phdev->tlv_options = NULL;
-	phdev->sh = NULL;
 	return 0;
 }
 
-/**
- * Check if GENEVE TLV options are hosted on the current port
- * and the port can be closed
- *
- * @param priv
- *   Device private data.
- *
- * @return
- *   0 on success, a negative EBUSY and rte_errno is set.
- */
-int
-mlx5_geneve_tlv_options_check_busy(struct mlx5_priv *priv) -{
-	struct mlx5_physical_device *phdev =
 mlx5_get_locked_physical_device(priv);
-	struct mlx5_dev_ctx_shared *sh = priv->sh;
-
-	if (!phdev || phdev->sh != sh) {
-		mlx5_unlock_physical_device();
-		return 0;
-	}
-	if (!sh->phdev->tlv_options || sh->phdev->tlv_options->refcnt == 1) {
-		/* Mark port as being closed one */
-		sh->phdev->sh = NULL;
-		mlx5_unlock_physical_device();
-		return 0;
-	}
-	mlx5_unlock_physical_device();
-	rte_errno = EBUSY;
-	return -EBUSY;
-}
-
 /**
  * Validate GENEVE TLV option user request structure.
  *

@@ -955,18 +921,12 @@ mlx5_geneve_tlv_parser_create(uint16_t port_id,
 			rte_errno = EEXIST;
 			return NULL;
 		}
-		if (phdev->sh == NULL) {
-			mlx5_unlock_physical_device();
-			DRV_LOG(ERR, "GENEVE TLV options are hosted on
 port being closed.");
-			rte_errno = EBUSY;
-			return NULL;
-		}
 		/* Use existing options. */
 		options->refcnt++;
 		goto exit;
 	}
 	/* Create GENEVE TLV options for this physical device. */
-	options = mlx5_geneve_tlv_options_create(priv->sh, tlv_list,
 nb_options, sample_id);
+	options = mlx5_geneve_tlv_options_create(phdev->ctx, tlv_list,
+nb_options, sample_id);
 	if (!options) {
 		mlx5_unlock_physical_device();
 		return NULL;
--

2.45.2

From: Michael Baum <[email protected]>

Add function for creating additional CTX for same device base on existing CTX.

Signed-off-by Michael Baum <[email protected]>

---
 drivers/common/mlx5/linux/mlx5_common_os.c   | 52
++++++++++++++++++++
 drivers/common/mlx5/mlx5_common.h            |  4 ++
 drivers/common/mlx5/version.map              |  1 +
 drivers/common/mlx5/windows/mlx5_common_os.c | 31 ++++++++++++
 4 files changed, 88 insertions(+)

drivers/common/mlx5/linux/mlx5_common_os.c +2 -0

index 41345e1597..2f5032f0da 100644
@@ -873,6 +873,58 @@ mlx5_os_open_device(struct mlx5_common_device
 *cdev, uint32_t classes)
 	return 0;
 }
 
+/**
+ * API function to obtain a new InfiniBand (IB) context for a given common

device.
+ *
+ * This function provides a port-agnostic IB context for a physical

drivers/common/mlx5/linux/mlx5_common_os.c +12 -0

+device, enabling the
+ * device to create and manage resources that can be initialized when a
+port starts and
+ * released when another port stops.
+ *
+ * For Linux, it imports new context from the existing context.
+ *
+ * @param cdev
+ *   Pointer to the mlx5 device structure.
+ *
+ * @return
+ *   Pointer to an `ibv_context` on success, or NULL on failure, with `rte_errno`

set.
+ */

drivers/common/mlx5/linux/mlx5_common_os.c +8 -0

+void *
+mlx5_os_get_physical_device_ctx(struct mlx5_common_device *cdev) {
+	struct ibv_context *ctx = NULL;
+	int cmd_fd = ((struct ibv_context *)cdev->ctx)->cmd_fd;
+	int new_cmd_fd;
+
+	/*
+	 * Duplicate the command FD to pass it as input to the import device

function.

drivers/common/mlx5/linux/mlx5_common_os.c +1 -0

+	 * If the import function succeeds, the new device context takes

ownership of

drivers/common/mlx5/linux/mlx5_common_os.c +13 -0

+	 * this FD, which will be freed when the new device is closed.
+	 * If the import function fails, we are responsible for closing this FD.
+	 */
+	new_cmd_fd = dup(cmd_fd);
+	if (new_cmd_fd < 0) {
+		DRV_LOG(ERR,
+			"Failed to duplicate FD %d for IB device \"%s\": %s",
+			cmd_fd, mlx5_os_get_ctx_device_name(cdev->ctx),
+			rte_strerror(errno));
+		rte_errno = errno;
+		return NULL;
+	}
+	/* Attempt to import the duplicated FD to create a new device context.

*/

drivers/common/mlx5/linux/mlx5_common_os.c +10 -0

+	ctx = mlx5_glue->import_device(new_cmd_fd);
+	if (!ctx) {
+		DRV_LOG(ERR, "Failed to import IB device \"%s\": %s",
+			mlx5_os_get_ctx_device_name(cdev->ctx),
+			rte_strerror(errno));
+		close(new_cmd_fd);
+		rte_errno = errno;
+		return NULL;
+	}
+	DRV_LOG(INFO, "IB device \"%s\" successfully imported, old_fd=%d,

new_fd=%d",

drivers/common/mlx5/linux/mlx5_common_os.c +1 -0

+		mlx5_os_get_ctx_device_name(cdev->ctx), cmd_fd,

new_cmd_fd);

drivers/common/mlx5/linux/mlx5_common_os.c +3 -0

+	return (void *)ctx;
+}
+
 int
 mlx5_get_device_guid(const struct rte_pci_addr *dev, uint8_t *guid, size_t

len) { diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index e7bd4c6ec4..bea1382911 100644

drivers/common/mlx5/mlx5_common.h +4 -0

@@ -651,6 +651,10 @@ mlx5_devx_uar_release(struct mlx5_uar *uar);
 
 /* mlx5_common_os.c */
 
+__rte_internal
+void *
+mlx5_os_get_physical_device_ctx(struct mlx5_common_device *cdev);
+
 int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t

classes); int mlx5_os_pd_prepare(struct mlx5_common_device *cdev); int mlx5_os_pd_release(struct mlx5_common_device *cdev); diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map index 8301485839..819e6b96cb 100644

drivers/common/mlx5/version.map +1 -0

@@ -150,6 +150,7 @@ INTERNAL {
 	mlx5_nl_rdma_monitor_info_get; # WINDOWS_NO_EXPORT
 	mlx5_nl_rdma_monitor_cap_get; # WINDOWS_NO_EXPORT
 
+	mlx5_os_get_physical_device_ctx;
 	mlx5_os_umem_dereg;
 	mlx5_os_umem_reg;
 

drivers/common/mlx5/windows/mlx5_common_os.c +10 -0

index 2b6058eb3e..68f1981193 100644
@@ -266,6 +266,37 @@ mlx5_os_open_device(struct mlx5_common_device
 *cdev, uint32_t classes)
 	return -rte_errno;
 }
 
+/**
+ * API function to obtain a new MLX5 context for a given common device.
+ *
+ * This function provides a port-agnostic context for a physical
+device, enabling the
+ * device to create and manage resources that can be initialized when a
+port starts and
+ * released when another port stops.
+ *
+ * For Windows, it creates a new context for the device regardless to existing

context.
+ *
+ * @param cdev
+ * Pointer to the mlx5 device structure.
+ *
+ * @return
+ * Pointer to an ibv_context on success, or NULL on failure, with rte_errno
set.
+ */

drivers/common/mlx5/windows/mlx5_common_os.c +15 -1

+void *
+mlx5_os_get_physical_device_ctx(struct mlx5_common_device *cdev) {
+	struct mlx5_common_device temp = {
+		.dev = cdev->dev,
+	};
+
+	if (mlx5_os_open_device(&temp, MLX5_CLASS_ETH) < 0) {
+		DRV_LOG(ERR, "Failed to duplicate DevX device \"%s\": %s",
+			mlx5_os_get_ctx_device_name(cdev->ctx),
+			rte_strerror(rte_errno));
+		return NULL;
+	}
+	return (void *)temp.ctx;
+}
+
 /**
  * Register umem.
  *
--

2.45.2