mlx5: sharing global MR cache between drivers

index ec246c15f9..17a54acf1e 100644
@@ -354,6 +354,16 @@ mlx5_dev_hw_global_prepare(struct mlx5_common_device *cdev, uint32_t classes)
 	ret = mlx5_os_pd_create(cdev);
 	if (ret)
 		goto error;
+	/* All actions taken below are relevant only when DevX is supported */
+	if (cdev->config.devx == 0)
+		return 0;
+	/* Query HCA attributes. */
+	ret = mlx5_devx_cmd_query_hca_attr(cdev->ctx, &cdev->config.hca_attr);
+	if (ret) {
+		DRV_LOG(ERR, "Unable to read HCA capabilities.");
+		rte_errno = ENOTSUP;
+		goto error;
+	}
 	return 0;
 error:
 	mlx5_dev_hw_global_release(cdev);

index d72002ca3c..a863fb2b26 100644
@@ -332,6 +332,7 @@ void mlx5_common_init(void);
  *  - User device parameters disabled features.
  */
 struct mlx5_common_dev_config {
+	struct mlx5_hca_attr hca_attr; /* HCA attributes. */
 	int dbnc; /* Skip doorbell register write barrier. */
 	unsigned int devx:1; /* Whether devx interface is available or not. */
 	unsigned int sys_mem_en:1; /* The default memory allocator. */

index 7dc670482a..f1d3416237 100644
@@ -39,7 +39,6 @@ struct mlx5_compress_priv {
 	struct mlx5_common_device *cdev; /* Backend mlx5 device. */
 	void *uar;
 	uint8_t min_block_size;
-	uint8_t qp_ts_format; /* Whether SQ supports timestamp formats. */
 	/* Minimum huffman block size supported by the device. */
 	struct rte_compressdev_config dev_config;
 	LIST_HEAD(xform_list, mlx5_compress_xform) xform_list;

@@ -243,7 +242,8 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 		goto err;
 	}
 	qp_attr.cqn = qp->cq.cq->id;
-	qp_attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format);
+	qp_attr.ts_format =
+		mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 	qp_attr.rq_size = 0;
 	qp_attr.sq_size = RTE_BIT32(log_ops_n);
 	qp_attr.mmo = priv->mmo_decomp_qp && priv->mmo_comp_qp

@@ -755,7 +755,7 @@ mlx5_compress_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct rte_compressdev *compressdev;
 	struct mlx5_compress_priv *priv;
-	struct mlx5_hca_attr att = { 0 };
+	struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
 	struct rte_compressdev_pmd_init_params init_params = {
 		.name = "",
 		.socket_id = cdev->dev->numa_node,

@@ -767,10 +767,9 @@ mlx5_compress_dev_probe(struct mlx5_common_device *cdev)
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (mlx5_devx_cmd_query_hca_attr(cdev->ctx, &att) != 0 ||
-	    ((att.mmo_compress_sq_en == 0 || att.mmo_decompress_sq_en == 0 ||
-		att.mmo_dma_sq_en == 0) && (att.mmo_compress_qp_en == 0 ||
-		att.mmo_decompress_qp_en == 0 || att.mmo_dma_qp_en == 0))) {
+	if ((attr->mmo_compress_sq_en == 0 || attr->mmo_decompress_sq_en == 0 ||
+	    attr->mmo_dma_sq_en == 0) && (attr->mmo_compress_qp_en == 0 ||
+	    attr->mmo_decompress_qp_en == 0 || attr->mmo_dma_qp_en == 0)) {
 		DRV_LOG(ERR, "Not enough capabilities to support compress "
 			"operations, maybe old FW/OFED version?");
 		rte_errno = ENOTSUP;

@@ -789,16 +788,15 @@ mlx5_compress_dev_probe(struct mlx5_common_device *cdev)
 	compressdev->enqueue_burst = mlx5_compress_enqueue_burst;
 	compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
 	priv = compressdev->data->dev_private;
-	priv->mmo_decomp_sq = att.mmo_decompress_sq_en;
-	priv->mmo_decomp_qp = att.mmo_decompress_qp_en;
-	priv->mmo_comp_sq = att.mmo_compress_sq_en;
-	priv->mmo_comp_qp = att.mmo_compress_qp_en;
-	priv->mmo_dma_sq = att.mmo_dma_sq_en;
-	priv->mmo_dma_qp = att.mmo_dma_qp_en;
+	priv->mmo_decomp_sq = attr->mmo_decompress_sq_en;
+	priv->mmo_decomp_qp = attr->mmo_decompress_qp_en;
+	priv->mmo_comp_sq = attr->mmo_compress_sq_en;
+	priv->mmo_comp_qp = attr->mmo_compress_qp_en;
+	priv->mmo_dma_sq = attr->mmo_dma_sq_en;
+	priv->mmo_dma_qp = attr->mmo_dma_qp_en;
 	priv->cdev = cdev;
 	priv->compressdev = compressdev;
-	priv->min_block_size = att.compress_min_block_size;
-	priv->qp_ts_format = att.qp_ts_format;
+	priv->min_block_size = attr->compress_min_block_size;
 	if (mlx5_compress_hw_global_prepare(priv) != 0) {
 		rte_compressdev_pmd_destroy(priv->compressdev);
 		return -1;

index 1edd9d61d5..30e23e7b00 100644
@@ -669,7 +669,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.cqn = qp->cq_obj.cq->id;
 	attr.rq_size = 0;
 	attr.sq_size = RTE_BIT32(log_nb_desc);
-	attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format);
+	attr.ts_format =
+		mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 	ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj, log_nb_desc,
 				  &attr, socket_id);
 	if (ret) {

@@ -920,7 +921,6 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
-	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
 		.private_data_size = sizeof(struct mlx5_crypto_priv),

@@ -937,8 +937,7 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (mlx5_devx_cmd_query_hca_attr(cdev->ctx, &attr) != 0 ||
-	    attr.crypto == 0 || attr.aes_xts == 0) {
+	if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
 		DRV_LOG(ERR, "Not enough capabilities to support crypto "
 			"operations, maybe old FW/OFED version?");
 		rte_errno = ENOTSUP;

@@ -972,7 +971,6 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 	priv->cdev = cdev;
 	priv->login_obj = login;
 	priv->crypto_dev = crypto_dev;
-	priv->qp_ts_format = attr.qp_ts_format;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		return -1;

index 27ae9cff2c..030f369423 100644
@@ -24,7 +24,6 @@ struct mlx5_crypto_priv {
 	void *uar; /* User Access Region. */
 	volatile uint64_t *uar_addr;
 	uint32_t max_segs_num; /* Maximum supported data segs. */
-	uint8_t qp_ts_format; /* Whether QP supports timestamp formats. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */

index 36927e0dcc..06bde2669c 100644
@@ -132,8 +132,8 @@ mlx5_os_set_nonblock_channel_fd(int fd)
  * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5
  * device attributes from the glue out parameter.
  *
- * @param dev
- *   Pointer to ibv context.
+ * @param cdev
+ *   Pointer to mlx5 device.
  *
  * @param device_attr
  *   Pointer to mlx5 device attributes.

@@ -142,15 +142,17 @@ mlx5_os_set_nonblock_channel_fd(int fd)
  *   0 on success, non zero error number otherwise
  */
 int
-mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr)
+mlx5_os_get_dev_attr(struct mlx5_common_device *cdev,
+		     struct mlx5_dev_attr *device_attr)
 {
 	int err;
+	struct ibv_context *ctx = cdev->ctx;
 	struct ibv_device_attr_ex attr_ex;
+
 	memset(device_attr, 0, sizeof(*device_attr));
 	err = mlx5_glue->query_device_ex(ctx, NULL, &attr_ex);
 	if (err)
 		return err;
-
 	device_attr->device_cap_flags_ex = attr_ex.device_cap_flags_ex;
 	device_attr->max_qp_wr = attr_ex.orig_attr.max_qp_wr;
 	device_attr->max_sge = attr_ex.orig_attr.max_sge;

@@ -1326,27 +1328,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		config->mps == MLX5_MPW ? "legacy " : "",
 		config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled");
 	if (sh->devx) {
-		err = mlx5_devx_cmd_query_hca_attr(sh->cdev->ctx,
-						   &config->hca_attr);
-		if (err) {
-			err = -err;
-			goto error;
-		}
-		/* Check relax ordering support. */
-		if (!haswell_broadwell_cpu) {
-			sh->cmng.relaxed_ordering_write =
-				config->hca_attr.relaxed_ordering_write;
-			sh->cmng.relaxed_ordering_read =
-				config->hca_attr.relaxed_ordering_read;
-		} else {
-			sh->cmng.relaxed_ordering_read = 0;
-			sh->cmng.relaxed_ordering_write = 0;
-		}
-		sh->rq_ts_format = config->hca_attr.rq_ts_format;
-		sh->sq_ts_format = config->hca_attr.sq_ts_format;
+		config->hca_attr = sh->cdev->config.hca_attr;
 		sh->steering_format_version =
 			config->hca_attr.steering_format_version;
-		sh->qp_ts_format = config->hca_attr.qp_ts_format;
 		/* Check for LRO support. */
 		if (config->dest_tir && config->hca_attr.lro_cap &&
 		    config->dv_flow_en) {

index 1bb490c5e7..788c701292 100644
@@ -520,6 +520,7 @@ mlx5_flow_aging_init(struct mlx5_dev_ctx_shared *sh)
 static void
 mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh)
 {
+	struct mlx5_hca_attr *attr = &sh->cdev->config.hca_attr;
 	int i;
 
 	memset(&sh->cmng, 0, sizeof(sh->cmng));

@@ -532,6 +533,10 @@ mlx5_flow_counters_mng_init(struct mlx5_dev_ctx_shared *sh)
 		TAILQ_INIT(&sh->cmng.counters[i]);
 		rte_spinlock_init(&sh->cmng.csl[i]);
 	}
+	if (sh->devx && !haswell_broadwell_cpu) {
+		sh->cmng.relaxed_ordering_write = attr->relaxed_ordering_write;
+		sh->cmng.relaxed_ordering_read = attr->relaxed_ordering_read;
+	}
 }
 
 /**

@@ -1287,7 +1292,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	sh->devx = sh->cdev->config.devx;
 	if (spawn->bond_info)
 		sh->bond = *spawn->bond_info;
-	err = mlx5_os_get_dev_attr(sh->cdev->ctx, &sh->device_attr);
+	err = mlx5_os_get_dev_attr(sh->cdev, &sh->device_attr);
 	if (err) {
 		DRV_LOG(DEBUG, "mlx5_os_get_dev_attr() failed");
 		goto error;

index 7048f7bd1c..96e2cbc644 100644
@@ -1127,11 +1127,8 @@ struct mlx5_dev_ctx_shared {
 	uint32_t refcnt;
 	uint32_t devx:1; /* Opened with DV. */
 	uint32_t flow_hit_aso_en:1; /* Flow Hit ASO is supported. */
-	uint32_t rq_ts_format:2; /* RQ timestamp formats supported. */
-	uint32_t sq_ts_format:2; /* SQ timestamp formats supported. */
 	uint32_t steering_format_version:4;
 	/* Indicates the device steering logic format. */
-	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
 	uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */

@@ -1765,7 +1762,8 @@ void mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev);
 
 /* mlx5_os.c */
 struct rte_pci_driver;
-int mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *dev_attr);
+int mlx5_os_get_dev_attr(struct mlx5_common_device *dev,
+			 struct mlx5_dev_attr *dev_attr);
 void mlx5_os_free_shared_dr(struct mlx5_priv *priv);
 int mlx5_os_net_probe(struct mlx5_common_device *cdev);
 void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh);

index b98b82bf79..6b6b9c77ae 100644
@@ -236,6 +236,7 @@ static int
 mlx5_rxq_create_devx_rq_resources(struct rte_eth_dev *dev, uint16_t idx)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_common_device *cdev = priv->sh->cdev;
 	struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
 	struct mlx5_rxq_ctrl *rxq_ctrl =
 		container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);

@@ -249,7 +250,8 @@ mlx5_rxq_create_devx_rq_resources(struct rte_eth_dev *dev, uint16_t idx)
 	rq_attr.vsd = (rxq_data->vlan_strip) ? 0 : 1;
 	rq_attr.cqn = rxq_ctrl->obj->cq_obj.cq->id;
 	rq_attr.scatter_fcs = (rxq_data->crc_present) ? 1 : 0;
-	rq_attr.ts_format = mlx5_ts_format_conv(priv->sh->rq_ts_format);
+	rq_attr.ts_format =
+			mlx5_ts_format_conv(cdev->config.hca_attr.rq_ts_format);
 	/* Fill WQ attributes for this RQ. */
 	if (mlx5_rxq_mprq_enabled(rxq_data)) {
 		rq_attr.wq_attr.wq_type = MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ;

@@ -276,12 +278,11 @@ mlx5_rxq_create_devx_rq_resources(struct rte_eth_dev *dev, uint16_t idx)
 	rq_attr.wq_attr.end_padding_mode = priv->config.hw_padding ?
 						MLX5_WQ_END_PAD_MODE_ALIGN :
 						MLX5_WQ_END_PAD_MODE_NONE;
-	rq_attr.wq_attr.pd = priv->sh->cdev->pdn;
+	rq_attr.wq_attr.pd = cdev->pdn;
 	rq_attr.counter_set_id = priv->counter_set_id;
 	/* Create RQ using DevX API. */
-	return mlx5_devx_rq_create(priv->sh->cdev->ctx, &rxq_ctrl->obj->rq_obj,
-				   wqe_size, log_desc_n, &rq_attr,
-				   rxq_ctrl->socket);
+	return mlx5_devx_rq_create(cdev->ctx, &rxq_ctrl->obj->rq_obj, wqe_size,
+				   log_desc_n, &rq_attr, rxq_ctrl->socket);
 }
 
 /**

@@ -981,6 +982,7 @@ mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx,
 				  uint16_t log_desc_n)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_common_device *cdev = priv->sh->cdev;
 	struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
 	struct mlx5_txq_ctrl *txq_ctrl =
 			container_of(txq_data, struct mlx5_txq_ctrl, txq);

@@ -994,14 +996,15 @@ mlx5_txq_create_devx_sq_resources(struct rte_eth_dev *dev, uint16_t idx,
 		.tis_lst_sz = 1,
 		.tis_num = priv->sh->tis->id,
 		.wq_attr = (struct mlx5_devx_wq_attr){
-			.pd = priv->sh->cdev->pdn,
+			.pd = cdev->pdn,
 			.uar_page =
 				 mlx5_os_get_devx_uar_page_id(priv->sh->tx_uar),
 		},
-		.ts_format = mlx5_ts_format_conv(priv->sh->sq_ts_format),
+		.ts_format =
+			mlx5_ts_format_conv(cdev->config.hca_attr.sq_ts_format),
 	};
 	/* Create Send Queue object with DevX. */
-	return mlx5_devx_sq_create(priv->sh->cdev->ctx, &txq_obj->sq_obj,
+	return mlx5_devx_sq_create(cdev->ctx, &txq_obj->sq_obj,
 				   log_desc_n, &sq_attr, priv->sh->numa_node);
 }
 #endif

index 17e3f2a300..8f3d2ffc2c 100644
@@ -319,7 +319,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 		if (mlx5_aso_sq_create(cdev->ctx, &sh->aso_age_mng->aso_sq, 0,
 				       sh->tx_uar, cdev->pdn,
 				       MLX5_ASO_QUEUE_LOG_DESC,
-				       sh->sq_ts_format)) {
+				       cdev->config.hca_attr.sq_ts_format)) {
 			mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
 			return -1;
 		}

@@ -329,7 +329,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 		if (mlx5_aso_sq_create(cdev->ctx, &sh->mtrmng->pools_mng.sq, 0,
 				       sh->tx_uar, cdev->pdn,
 				       MLX5_ASO_QUEUE_LOG_DESC,
-				       sh->sq_ts_format))
+				       cdev->config.hca_attr.sq_ts_format))
 			return -1;
 		mlx5_aso_mtr_init_sq(&sh->mtrmng->pools_mng.sq);
 		break;

@@ -341,7 +341,7 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 		if (mlx5_aso_sq_create(cdev->ctx, &sh->ct_mng->aso_sq, 0,
 				       sh->tx_uar, cdev->pdn,
 				       MLX5_ASO_QUEUE_LOG_DESC,
-				       sh->sq_ts_format)) {
+				       cdev->config.hca_attr.sq_ts_format)) {
 			mlx5_aso_dereg_mr(sh, &sh->ct_mng->aso_sq.mr);
 			return -1;
 		}

index 190fd8cc35..b8d3970e48 100644
@@ -235,7 +235,8 @@ mlx5_txpp_create_rearm_queue(struct mlx5_dev_ctx_shared *sh)
 			.pd = sh->cdev->pdn,
 			.uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar),
 		},
-		.ts_format = mlx5_ts_format_conv(sh->sq_ts_format),
+		.ts_format = mlx5_ts_format_conv
+				       (sh->cdev->config.hca_attr.sq_ts_format),
 	};
 	struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
 	struct mlx5_devx_cq_attr cq_attr = {

@@ -445,7 +446,8 @@ mlx5_txpp_create_clock_queue(struct mlx5_dev_ctx_shared *sh)
 	sq_attr.wq_attr.cd_slave = 1;
 	sq_attr.wq_attr.uar_page = mlx5_os_get_devx_uar_page_id(sh->tx_uar);
 	sq_attr.wq_attr.pd = sh->cdev->pdn;
-	sq_attr.ts_format = mlx5_ts_format_conv(sh->sq_ts_format);
+	sq_attr.ts_format =
+		mlx5_ts_format_conv(sh->cdev->config.hca_attr.sq_ts_format);
 	ret = mlx5_devx_sq_create(sh->cdev->ctx, &wq->sq_obj,
 				  log2above(wq->sq_size),
 				  &sq_attr, sh->numa_node);

100 lines not on this page · raw message

index 54d3e64f43..91fb931062 100644
@@ -125,18 +125,13 @@ static int
 mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct mlx5_regex_priv *priv = NULL;
-	struct mlx5_hca_attr attr;
+	struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
 	char name[RTE_REGEXDEV_NAME_MAX_LEN];
 	int ret;
 	uint32_t val;
 
-	ret = mlx5_devx_cmd_query_hca_attr(cdev->ctx, &attr);
-	if (ret) {
-		DRV_LOG(ERR, "Unable to read HCA capabilities.");
-		rte_errno = ENOTSUP;
-		return -rte_errno;
-	} else if (((!attr.regex) && (!attr.mmo_regex_sq_en) &&
-		(!attr.mmo_regex_qp_en)) || attr.regexp_num_of_engines == 0) {
+	if ((!attr->regex && !attr->mmo_regex_sq_en && !attr->mmo_regex_qp_en)
+	    || attr->regexp_num_of_engines == 0) {
 		DRV_LOG(ERR, "Not enough capabilities to support RegEx, maybe "
 			"old FW/OFED version?");
 		rte_errno = ENOTSUP;

@@ -154,9 +149,8 @@ mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
-	priv->mmo_regex_qp_cap = attr.mmo_regex_qp_en;
-	priv->mmo_regex_sq_cap = attr.mmo_regex_sq_en;
-	priv->qp_ts_format = attr.qp_ts_format;
+	priv->mmo_regex_qp_cap = attr->mmo_regex_qp_en;
+	priv->mmo_regex_sq_cap = attr->mmo_regex_sq_en;
 	priv->cdev = cdev;
 	priv->nb_engines = 2; /* attr.regexp_num_of_engines */
 	ret = mlx5_devx_regex_register_read(priv->cdev->ctx, 0,

@@ -190,8 +184,8 @@ mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
 	priv->regexdev->dev_ops = &mlx5_regexdev_ops;
 	priv->regexdev->enqueue = mlx5_regexdev_enqueue;
 #ifdef HAVE_MLX5_UMR_IMKEY
-	if (!attr.umr_indirect_mkey_disabled &&
-	    !attr.umr_modify_entity_size_disabled)
+	if (!attr->umr_indirect_mkey_disabled &&
+	    !attr->umr_modify_entity_size_disabled)
 		priv->has_umr = 1;
 	if (priv->has_umr)
 		priv->regexdev->enqueue = mlx5_regexdev_enqueue_gga;

index c128b7acbb..be81931b3a 100644
@@ -72,7 +72,6 @@ struct mlx5_regex_priv {
 	/**< Called by memory event callback. */
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	uint8_t is_bf2; /* The device is BF2 device. */
-	uint8_t qp_ts_format; /* Whether SQ supports timestamp formats. */
 	uint8_t has_umr; /* The device supports UMR. */
 	uint32_t mmo_regex_qp_cap:1;
 	uint32_t mmo_regex_sq_cap:1;

index 1136de1d7e..545bbbcf89 100644
@@ -139,7 +139,8 @@ regex_ctrl_create_hw_qp(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 		.cqn = qp->cq.cq_obj.cq->id,
 		.uar_index = priv->uar->page_id,
 		.pd = priv->cdev->pdn,
-		.ts_format = mlx5_ts_format_conv(priv->qp_ts_format),
+		.ts_format = mlx5_ts_format_conv
+				     (priv->cdev->config.hca_attr.qp_ts_format),
 		.user_index = q_ind,
 	};
 	struct mlx5_regex_hw_qp *qp_obj = &qp->qps[q_ind];

index fe68ab0252..3971f2e335 100644
@@ -505,36 +505,29 @@ static int
 mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct mlx5_vdpa_priv *priv = NULL;
-	struct mlx5_hca_attr attr;
-	int ret;
+	struct mlx5_hca_attr *attr = &cdev->config.hca_attr;
 
-	ret = mlx5_devx_cmd_query_hca_attr(cdev->ctx, &attr);
-	if (ret) {
-		DRV_LOG(ERR, "Unable to read HCA capabilities.");
-		rte_errno = ENOTSUP;
-		return -rte_errno;
-	} else if (!attr.vdpa.valid || !attr.vdpa.max_num_virtio_queues) {
+	if (!attr->vdpa.valid || !attr->vdpa.max_num_virtio_queues) {
 		DRV_LOG(ERR, "Not enough capabilities to support vdpa, maybe "
 			"old FW/OFED version?");
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	if (!attr.vdpa.queue_counters_valid)
+	if (!attr->vdpa.queue_counters_valid)
 		DRV_LOG(DEBUG, "No capability to support virtq statistics.");
 	priv = rte_zmalloc("mlx5 vDPA device private", sizeof(*priv) +
 			   sizeof(struct mlx5_vdpa_virtq) *
-			   attr.vdpa.max_num_virtio_queues * 2,
+			   attr->vdpa.max_num_virtio_queues * 2,
 			   RTE_CACHE_LINE_SIZE);
 	if (!priv) {
 		DRV_LOG(ERR, "Failed to allocate private memory.");
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
-	priv->caps = attr.vdpa;
-	priv->log_max_rqt_size = attr.log_max_rqt_size;
-	priv->num_lag_ports = attr.num_lag_ports;
-	priv->qp_ts_format = attr.qp_ts_format;
-	if (attr.num_lag_ports == 0)
+	priv->caps = attr->vdpa;
+	priv->log_max_rqt_size = attr->log_max_rqt_size;
+	priv->num_lag_ports = attr->num_lag_ports;
+	if (attr->num_lag_ports == 0)
 		priv->num_lag_ports = 1;
 	priv->cdev = cdev;
 	priv->var = mlx5_glue->dv_alloc_var(priv->cdev->ctx, 0);

index d9a68e701e..5045fea773 100644
@@ -142,7 +142,6 @@ struct mlx5_vdpa_priv {
 	struct mlx5_devx_obj *tiss[16]; /* TIS list for each LAG port. */
 	uint16_t nr_virtqs;
 	uint8_t num_lag_ports;
-	uint8_t qp_ts_format;
 	uint64_t features; /* Negotiated features. */
 	uint16_t log_max_rqt_size;
 	struct mlx5_vdpa_steer steer;

index 47f9afe855..19497597e6 100644
@@ -594,7 +594,8 @@ mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
 	if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
 		return -1;
 	attr.pd = priv->cdev->pdn;
-	attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format);
+	attr.ts_format =
+		mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 	eqp->fw_qp = mlx5_devx_cmd_create_qp(priv->cdev->ctx, &attr);
 	if (!eqp->fw_qp) {
 		DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno);

@@ -605,7 +606,8 @@ mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
 	attr.rq_size = RTE_BIT32(log_desc_n);
 	attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE);
 	attr.sq_size = 0; /* No need SQ. */
-	attr.ts_format = mlx5_ts_format_conv(priv->qp_ts_format);
+	attr.ts_format =
+		mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
 	ret = mlx5_devx_qp_create(priv->cdev->ctx, &(eqp->sw_qp), log_desc_n,
 				  &attr, SOCKET_ID_ANY);
 	if (ret) {

index 4d884f7295..9ce973d95c 100644
@@ -17,19 +17,6 @@
 #include "mlx5_rx.h"
 #include "mlx5_tx.h"
 
-struct mr_find_contig_memsegs_data {
-	uintptr_t addr;
-	uintptr_t start;
-	uintptr_t end;
-	const struct rte_memseg_list *msl;
-};
-
-struct mr_update_mp_data {
-	struct rte_eth_dev *dev;
-	struct mlx5_mr_ctrl *mr_ctrl;
-	int ret;
-};
-
 /**
  * Callback for memory event. This can be called from both primary and secondary
  * process.

@@ -134,70 +121,7 @@ mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
 		}
 		/* Fallback for generic mechanism in corner cases. */
 	}
-	lkey = mlx5_tx_addr2mr_bh(txq, addr);
-	if (lkey == UINT32_MAX && rte_errno == ENXIO) {
-		/* Mempool may have externally allocated memory. */
-		return mlx5_tx_update_ext_mp(txq, addr, mlx5_mb2mp(mb));
-	}
-	return lkey;
-}
-
-/**
- * Called during rte_mempool_mem_iter() by mlx5_mr_update_ext_mp().
- *
- * Externally allocated chunk is registered and a MR is created for the chunk.
- * The MR object is added to the global list. If memseg list of a MR object
- * (mr->msl) is null, the MR object can be regarded as externally allocated
- * memory.
- *
- * Once external memory is registered, it should be static. If the memory is
- * freed and the virtual address range has different physical memory mapped
- * again, it may cause crash on device due to the wrong translation entry. PMD
- * can't track the free event of the external memory for now.
- */
-static void
-mlx5_mr_update_ext_mp_cb(struct rte_mempool *mp, void *opaque,
-			 struct rte_mempool_memhdr *memhdr,
-			 unsigned mem_idx __rte_unused)
-{
-	struct mr_update_mp_data *data = opaque;
-	struct rte_eth_dev *dev = data->dev;
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_mr_ctrl *mr_ctrl = data->mr_ctrl;
-	struct mlx5_mr *mr = NULL;
-	uintptr_t addr = (uintptr_t)memhdr->addr;
-	size_t len = memhdr->len;
-	struct mr_cache_entry entry;
-	uint32_t lkey;
-
-	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
-	/* If already registered, it should return. */
-	rte_rwlock_read_lock(&sh->share_cache.rwlock);
-	lkey = mlx5_mr_lookup_cache(&sh->share_cache, &entry, addr);
-	rte_rwlock_read_unlock(&sh->share_cache.rwlock);
-	if (lkey != UINT32_MAX)
-		return;
-	DRV_LOG(DEBUG, "port %u register MR for chunk #%d of mempool (%s)",
-		dev->data->port_id, mem_idx, mp->name);
-	mr = mlx5_create_mr_ext(sh->cdev->pd, addr, len, mp->socket_id,
-				sh->share_cache.reg_mr_cb);
-	if (!mr) {
-		DRV_LOG(WARNING,
-			"port %u unable to allocate a new MR of"
-			" mempool (%s).",
-			dev->data->port_id, mp->name);
-		data->ret = -1;
-		return;
-	}
-	rte_rwlock_write_lock(&sh->share_cache.rwlock);
-	LIST_INSERT_HEAD(&sh->share_cache.mr_list, mr, mr);
-	/* Insert to the global cache table. */
-	mlx5_mr_insert_cache(&sh->share_cache, mr);
-	rte_rwlock_write_unlock(&sh->share_cache.rwlock);
-	/* Insert to the local cache table */
-	mlx5_mr_addr2mr_bh(sh->cdev->pd, &priv->mp_id, &sh->share_cache,
-			   mr_ctrl, addr, sh->cdev->config.mr_ext_memseg_en);
+	return mlx5_tx_addr2mr_bh(txq, addr);
 }
 
 /**

@@ -331,67 +255,3 @@ mlx5_net_dma_unmap(struct rte_device *rte_dev, void *addr,
 	rte_rwlock_write_unlock(&sh->share_cache.rwlock);
 	return 0;
 }
-
-/**
- * Register MR for entire memory chunks in a Mempool having externally allocated
- * memory and fill in local cache.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param mr_ctrl
- *   Pointer to per-queue MR control structure.
- * @param mp
- *   Pointer to registering Mempool.
- *
- * @return
- *   0 on success, -1 on failure.
- */
-static uint32_t
-mlx5_mr_update_ext_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
-		      struct rte_mempool *mp)
-{
-	struct mr_update_mp_data data = {
-		.dev = dev,
-		.mr_ctrl = mr_ctrl,
-		.ret = 0,
-	};
-
-	rte_mempool_mem_iter(mp, mlx5_mr_update_ext_mp_cb, &data);
-	return data.ret;
-}
-
-/**
- * Register MR entire memory chunks in a Mempool having externally allocated
- * memory and search LKey of the address to return.
- *
- * @param dev
- *   Pointer to Ethernet device.
- * @param addr
- *   Search key.
- * @param mp
- *   Pointer to registering Mempool where addr belongs.
- *
- * @return
- *   LKey for address on success, UINT32_MAX on failure.
- */
-uint32_t
-mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
-		      struct rte_mempool *mp)
-{
-	struct mlx5_txq_ctrl *txq_ctrl =
-		container_of(txq, struct mlx5_txq_ctrl, txq);
-	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
-	struct mlx5_priv *priv = txq_ctrl->priv;
-
-	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
-		DRV_LOG(WARNING,
-			"port %u using address (%p) from unregistered mempool"
-			" having externally allocated memory"
-			" in secondary process, please create mempool"
-			" prior to rte_eth_dev_start()",
-			PORT_ID(priv), (void *)addr);
-		return UINT32_MAX;
-	}
-	mlx5_mr_update_ext_mp(ETH_DEV(priv), mr_ctrl, mp);
-	return mlx5_tx_addr2mr_bh(txq, addr);
-}

index 77d6069755..a6aa34084c 100644
@@ -239,8 +239,6 @@ int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 /* mlx5_mr.c */
 
 uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
-uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
-			       struct rte_mempool *mp);
 
 /* mlx5_tx_empw.c */
 

index 2e039a4e70..8fd65484cf 100644
@@ -271,6 +271,34 @@ mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)
 #endif
 }
 
+/**
+ * Initialize per-queue MR control descriptor.
+ *
+ * @param mr_ctrl
+ *   Pointer to MR control structure.
+ * @param dev_gen_ptr
+ *   Pointer to generation number of global cache.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
+		  int socket)
+{
+	if (mr_ctrl == NULL) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	/* Save pointer of global generation number to check memory event. */
+	mr_ctrl->dev_gen_ptr = dev_gen_ptr;
+	/* Initialize B-tree and allocate memory for bottom-half cache table. */
+	return mlx5_mr_btree_init(&mr_ctrl->cache_bh, MLX5_MR_BTREE_CACHE_N,
+				  socket);
+}
+
 /**
  * Find virtually contiguous memory chunk in a given MR.
  *

index 15489cd399..1392d9b55a 100644
@@ -124,6 +124,9 @@ mlx5_mr_lookup_lkey(struct mr_cache_entry *lkp_tbl, uint16_t *cached_idx,
 	return UINT32_MAX;
 }
 
+__rte_internal
+int mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
+		      int socket);
 __rte_internal
 int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
 __rte_internal

index f1d3416237..90d1684e00 100644
@@ -206,8 +206,8 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 		return -rte_errno;
 	}
 	dev->data->queue_pairs[qp_id] = qp;
-	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
-			       priv->dev_config.socket_id)) {
+	if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->mr_scache.dev_gen,
+			      priv->dev_config.socket_id)) {
 		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 			(uint32_t)qp_id);
 		rte_errno = ENOMEM;

@@ -258,8 +258,6 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 	ret = mlx5_devx_qp2rts(&qp->qp, 0);
 	if (ret)
 		goto err;
-	/* Save pointer of global generation number to check memory event. */
-	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	DRV_LOG(INFO, "QP %u: SQN=0x%X CQN=0x%X entries num = %u",
 		(uint32_t)qp_id, qp->qp.qp->id, qp->cq.cq->id, qp->entries_n);
 	return 0;

index 30e23e7b00..acfb856ef8 100644
@@ -677,14 +677,13 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to create QP.");
 		goto error;
 	}
-	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
-			       priv->dev_config.socket_id) != 0) {
+	if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->mr_scache.dev_gen,
+			      priv->dev_config.socket_id) != 0) {
 		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 			(uint32_t)qp_id);
 		rte_errno = ENOMEM;
 		goto error;
 	}
-	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	/*
 	 * In Order to configure self loopback, when calling devx qp2rts the
 	 * remote QP id that is used is the id of the same QP.

index f7f71f933e..479ea1324c 100644
@@ -1452,13 +1452,11 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		goto error;
 	}
 	tmpl->type = MLX5_RXQ_TYPE_STANDARD;
-	if (mlx5_mr_btree_init(&tmpl->rxq.mr_ctrl.cache_bh,
-			       MLX5_MR_BTREE_CACHE_N, socket)) {
+	if (mlx5_mr_ctrl_init(&tmpl->rxq.mr_ctrl,
+			      &priv->sh->share_cache.dev_gen, socket)) {
 		/* rte_errno is already set. */
 		goto error;
 	}
-	/* Rx queues don't use this pointer, but we want a valid structure. */
-	tmpl->rxq.mr_ctrl.dev_gen_ptr = &priv->sh->share_cache.dev_gen;
 	tmpl->socket = socket;
 	if (dev->data->dev_conf.intr_conf.rxq)
 		tmpl->irq = 1;

index a9c8ce52c9..4b027d6807 100644
@@ -1124,13 +1124,11 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		rte_errno = ENOMEM;
 		return NULL;
 	}
-	if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh,
-			       MLX5_MR_BTREE_CACHE_N, socket)) {
+	if (mlx5_mr_ctrl_init(&tmpl->txq.mr_ctrl,
+			      &priv->sh->share_cache.dev_gen, socket)) {
 		/* rte_errno is already set. */
 		goto error;
 	}
-	/* Save pointer of global generation number to check memory event. */
-	tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->share_cache.dev_gen;
 	MLX5_ASSERT(desc > MLX5_TX_COMP_THRESH);
 	tmpl->txq.offloads = conf->offloads |
 			     dev->data->dev_conf.txmode.offloads;

index 545bbbcf89..6735e51976 100644
@@ -242,10 +242,8 @@ mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind,
 		nb_sq_config++;
 	}
 
-	/* Save pointer of global generation number to check memory event. */
-	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
-	ret = mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
-				 rte_socket_id());
+	ret = mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->mr_scache.dev_gen,
+				rte_socket_id());
 	if (ret) {
 		DRV_LOG(ERR, "Error setting up mr btree");
 		goto err_btree;

index 519cb8d056..cf2f7ecbf3 100644
@@ -142,3 +142,18 @@ mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
 		memset(pmd_mr, 0, sizeof(*pmd_mr));
 	}
 }
+
+/**
+ * Set the reg_mr and dereg_mr callbacks.
+ *
+ * @param[out] reg_mr_cb
+ *   Pointer to reg_mr func
+ * @param[out] dereg_mr_cb
+ *   Pointer to dereg_mr func
+ */
+void
+mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
+{
+	*reg_mr_cb = mlx5_common_verbs_reg_mr;
+	*dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+}

index 8fd65484cf..43dc3d88ce 100644
@@ -199,7 +199,7 @@ mr_btree_insert(struct mlx5_mr_btree *bt, struct mr_cache_entry *entry)
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
-int
+static int
 mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
 {
 	if (bt == NULL) {

@@ -1044,6 +1044,29 @@ mlx5_mr_release_cache(struct mlx5_mr_share_cache *share_cache)
 	mlx5_mr_garbage_collect(share_cache);
 }
 
+/**
+ * Initialize global MR cache of a device.
+ *
+ * @param share_cache
+ *   Pointer to a global shared MR cache.
+ * @param socket
+ *   NUMA socket on which memory must be allocated.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket)
+{
+	/* Set the reg_mr and dereg_mr callback functions */
+	mlx5_os_set_reg_mr_cb(&share_cache->reg_mr_cb,
+			      &share_cache->dereg_mr_cb);
+	rte_rwlock_init(&share_cache->rwlock);
+	/* Initialize B-tree and allocate memory for global MR cache table. */
+	return mlx5_mr_btree_init(&share_cache->cache,
+				  MLX5_MR_BTREE_CACHE_N * 2, socket);
+}
+
 /**
  * Flush all of the local cache entries.
  *

index 1392d9b55a..36689dfb54 100644
@@ -128,8 +128,6 @@ __rte_internal
 int mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
 		      int socket);
 __rte_internal
-int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
-__rte_internal
 void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
 __rte_internal
 void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused);

@@ -145,6 +143,8 @@ uint32_t mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
 __rte_internal
 void mlx5_mr_release_cache(struct mlx5_mr_share_cache *mr_cache);
 __rte_internal
+int mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket);
+__rte_internal
 void mlx5_mr_dump_cache(struct mlx5_mr_share_cache *share_cache __rte_unused);
 __rte_internal
 void mlx5_mr_rebuild_cache(struct mlx5_mr_share_cache *share_cache);

@@ -183,6 +183,10 @@ __rte_internal
 void
 mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
 
+__rte_internal
+void
+mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
+
 __rte_internal
 void
 mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);

index 4d0f1e92e3..44e8ebec2b 100644
@@ -317,7 +317,7 @@ mlx5_os_umem_dereg(void *pumem)
  * @return
  *   0 on successful registration, -1 otherwise
  */
-int
+static int
 mlx5_os_reg_mr(void *pd,
 	       void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr)
 {

@@ -365,7 +365,7 @@ mlx5_os_reg_mr(void *pd,
  * @param[in] pmd_mr
  *  Pointer to PMD mr object
  */
-void
+static void
 mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
 {
 	if (pmd_mr && pmd_mr->mkey)

@@ -374,3 +374,19 @@ mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr)
 		claim_zero(mlx5_os_umem_dereg(pmd_mr->obj));
 	memset(pmd_mr, 0, sizeof(*pmd_mr));
 }
+
+/**
+ * Set the reg_mr and dereg_mr callbacks.
+ *
+ * @param[out] reg_mr_cb
+ *   Pointer to reg_mr func
+ * @param[out] dereg_mr_cb
+ *   Pointer to dereg_mr func
+ *
+ */
+void
+mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb)
+{
+	*reg_mr_cb = mlx5_os_reg_mr;
+	*dereg_mr_cb = mlx5_os_dereg_mr;
+}

index c99645aefd..3afce56cd9 100644
@@ -253,9 +253,5 @@ __rte_internal
 void *mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access);
 __rte_internal
 int mlx5_os_umem_dereg(void *pumem);
-__rte_internal
-int mlx5_os_reg_mr(void *pd,
-		   void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr);
-__rte_internal
-void mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
+
 #endif /* RTE_PMD_MLX5_COMMON_OS_H_ */

index 90d1684e00..bf3586a354 100644
@@ -799,16 +799,13 @@ mlx5_compress_dev_probe(struct mlx5_common_device *cdev)
 		rte_compressdev_pmd_destroy(priv->compressdev);
 		return -1;
 	}
-	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
-			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+	if (mlx5_mr_create_cache(&priv->mr_scache, rte_socket_id()) != 0) {
 		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
 		mlx5_compress_hw_global_release(priv);
 		rte_compressdev_pmd_destroy(priv->compressdev);
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
-	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
-	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_compress_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",

index acfb856ef8..eeb4dde9a9 100644
@@ -974,16 +974,13 @@ mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		return -1;
 	}
-	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
-			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+	if (mlx5_mr_create_cache(&priv->mr_scache, rte_socket_id()) != 0) {
 		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
-	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
-	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	priv->max_segs_num = devarg_prms.max_segs_num;
 	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +

index 06bde2669c..9e445f2f9b 100644
@@ -2831,23 +2831,6 @@ mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name,
 	return 1;
 }
 
-/**
- * Set the reg_mr and dereg_mr call backs
- *
- * @param reg_mr_cb[out]
- *   Pointer to reg_mr func
- * @param dereg_mr_cb[out]
- *   Pointer to dereg_mr func
- *
- */
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
-		      mlx5_dereg_mr_t *dereg_mr_cb)
-{
-	*reg_mr_cb = mlx5_mr_verbs_ops.reg_mr;
-	*dereg_mr_cb = mlx5_mr_verbs_ops.dereg_mr;
-}
-
 /**
  * Remove a MAC address from device
  *

48 lines not on this page · raw message

index f7e8e2fe98..829d9fa738 100644
@@ -12,7 +12,5 @@ void mlx5_txq_ibv_obj_release(struct mlx5_txq_obj *txq_obj);
 int mlx5_rxq_ibv_obj_dummy_lb_create(struct rte_eth_dev *dev);
 void mlx5_rxq_ibv_obj_dummy_lb_release(struct rte_eth_dev *dev);
 
-/* Verbs ops struct */
-extern const struct mlx5_mr_ops mlx5_mr_verbs_ops;
 extern struct mlx5_obj_ops ibv_obj_ops;
 #endif /* RTE_PMD_MLX5_VERBS_H_ */

index 788c701292..a6c196b368 100644
@@ -1351,15 +1351,11 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	 * At this point the device is not added to the memory
 	 * event list yet, context is just being created.
 	 */
-	err = mlx5_mr_btree_init(&sh->share_cache.cache,
-				 MLX5_MR_BTREE_CACHE_N * 2,
-				 sh->numa_node);
+	err = mlx5_mr_create_cache(&sh->share_cache, sh->numa_node);
 	if (err) {
 		err = rte_errno;
 		goto error;
 	}
-	mlx5_os_set_reg_mr_cb(&sh->share_cache.reg_mr_cb,
-			      &sh->share_cache.dereg_mr_cb);
 	mlx5_os_dev_shared_handler_install(sh);
 	sh->cnt_id_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 	if (!sh->cnt_id_tbl) {

index 96e2cbc644..5c25b94f36 100644
@@ -1363,12 +1363,6 @@ struct mlx5_obj_ops {
 
 #define MLX5_RSS_HASH_FIELDS_LEN RTE_DIM(mlx5_rss_hash_fields)
 
-/* MR operations structure. */
-struct mlx5_mr_ops {
-	mlx5_reg_mr_t reg_mr;
-	mlx5_dereg_mr_t dereg_mr;
-};
-
 struct mlx5_priv {
 	struct rte_eth_dev_data *dev_data;  /* Pointer to device data. */
 	struct mlx5_dev_ctx_shared *sh; /* Shared device context. */

@@ -1768,8 +1762,6 @@ void mlx5_os_free_shared_dr(struct mlx5_priv *priv);
 int mlx5_os_net_probe(struct mlx5_common_device *cdev);
 void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh);
 void mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh);
-void mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
-			   mlx5_dereg_mr_t *dereg_mr_cb);
 void mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
 int mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
 			 uint32_t index);

index 3477f018c1..c3d4b90946 100644
@@ -926,21 +926,4 @@ mlx5_os_net_probe(struct mlx5_common_device *cdev)
 	return 0;
 }
 
-/**
- * Set the reg_mr and dereg_mr call backs
- *
- * @param reg_mr_cb[out]
- *   Pointer to reg_mr func
- * @param dereg_mr_cb[out]
- *   Pointer to dereg_mr func
- *
- */
-void
-mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
-		      mlx5_dereg_mr_t *dereg_mr_cb)
-{
-	*reg_mr_cb = mlx5_os_reg_mr;
-	*dereg_mr_cb = mlx5_os_dereg_mr;
-}
-
 const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {0};

index 91fb931062..b39181ebb5 100644
@@ -194,11 +194,7 @@ mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
 	priv->regexdev->device = cdev->dev;
 	priv->regexdev->data->dev_private = priv;
 	priv->regexdev->state = RTE_REGEXDEV_READY;
-	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
-	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
-	ret = mlx5_mr_btree_init(&priv->mr_scache.cache,
-				 MLX5_MR_BTREE_CACHE_N * 2,
-				 rte_socket_id());
+	ret = mlx5_mr_create_cache(&priv->mr_scache, rte_socket_id());
 	if (ret) {
 		DRV_LOG(ERR, "MR init tree failed.");
 	    rte_errno = ENOMEM;

index a863fb2b26..8df4f32aa2 100644
@@ -22,6 +22,7 @@
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
 #include "mlx5_common_os.h"
+#include "mlx5_common_mr.h"
 
 /* Reported driver name. */
 #define MLX5_PCI_DRIVER_NAME "mlx5_pci"

@@ -447,6 +448,14 @@ __rte_internal
 bool
 mlx5_dev_is_pci(const struct rte_device *dev);
 
+/* mlx5_common_mr.c */
+
+__rte_internal
+uint32_t
+mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
+	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
+	      struct mlx5_mr_share_cache *share_cache);
+
 /* mlx5_common_os.c */
 
 int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);

97 lines not on this page · raw message

index bf3586a354..03ba8b7a7d 100644
@@ -435,40 +435,6 @@ static struct rte_compressdev_ops mlx5_compress_ops = {
 	.stream_free		= NULL,
 };
 
-/**
- * Query LKey from a packet buffer for QP. If not found, add the mempool.
- *
- * @param priv
- *   Pointer to the priv object.
- * @param addr
- *   Search key.
- * @param mr_ctrl
- *   Pointer to per-queue MR control structure.
- * @param ol_flags
- *   Mbuf offload features.
- *
- * @return
- *   Searched LKey on success, UINT32_MAX on no match.
- */
-static __rte_always_inline uint32_t
-mlx5_compress_addr2mr(struct mlx5_compress_priv *priv, uintptr_t addr,
-		      struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
-{
-	uint32_t lkey;
-
-	/* Check generation bit to see if there's any change on existing MRs. */
-	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
-		mlx5_mr_flush_local_cache(mr_ctrl);
-	/* Linear search on MR cache array. */
-	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
-				   MLX5_MR_CACHE_N, addr);
-	if (likely(lkey != UINT32_MAX))
-		return lkey;
-	/* Take slower bottom-half on miss. */
-	return mlx5_mr_addr2mr_bh(priv->cdev->pd, 0, &priv->mr_scache, mr_ctrl,
-				  addr, !!(ol_flags & EXT_ATTACHED_MBUF));
-}
-
 static __rte_always_inline uint32_t
 mlx5_compress_dseg_set(struct mlx5_compress_qp *qp,
 		       volatile struct mlx5_wqe_dseg *restrict dseg,

@@ -478,8 +444,8 @@ mlx5_compress_dseg_set(struct mlx5_compress_qp *qp,
 	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 
 	dseg->bcount = rte_cpu_to_be_32(len);
-	dseg->lkey = mlx5_compress_addr2mr(qp->priv, addr, &qp->mr_ctrl,
-					   mbuf->ol_flags);
+	dseg->lkey = mlx5_mr_mb2mr(qp->priv->cdev, 0, &qp->mr_ctrl, mbuf,
+				   &qp->priv->mr_scache);
 	dseg->pbuf = rte_cpu_to_be_64(addr);
 	return dseg->lkey;
 }

index eeb4dde9a9..f2e8b3d64c 100644
@@ -303,40 +303,6 @@ mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 	}
 }
 
-/**
- * Query LKey from a packet buffer for QP. If not found, add the mempool.
- *
- * @param priv
- *   Pointer to the priv object.
- * @param addr
- *   Search key.
- * @param mr_ctrl
- *   Pointer to per-queue MR control structure.
- * @param ol_flags
- *   Mbuf offload features.
- *
- * @return
- *   Searched LKey on success, UINT32_MAX on no match.
- */
-static __rte_always_inline uint32_t
-mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
-		    struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
-{
-	uint32_t lkey;
-
-	/* Check generation bit to see if there's any change on existing MRs. */
-	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
-		mlx5_mr_flush_local_cache(mr_ctrl);
-	/* Linear search on MR cache array. */
-	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
-				   MLX5_MR_CACHE_N, addr);
-	if (likely(lkey != UINT32_MAX))
-		return lkey;
-	/* Take slower bottom-half on miss. */
-	return mlx5_mr_addr2mr_bh(priv->cdev->pd, 0, &priv->mr_scache, mr_ctrl,
-				  addr, !!(ol_flags & EXT_ATTACHED_MBUF));
-}
-
 static __rte_always_inline uint32_t
 mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,

@@ -350,8 +316,8 @@ mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 	*remain -= data_len;
 	klm->bcount = rte_cpu_to_be_32(data_len);
 	klm->pbuf = rte_cpu_to_be_64(addr);
-	klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
-					mbuf->ol_flags);
+	klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf,
+				  &priv->mr_scache);
 	return klm->lkey;
 
 }

index 575b639752..8817e2e074 100644
@@ -123,26 +123,12 @@ set_wqe_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode,
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 static inline uint32_t
-mlx5_regex_addr2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
-		   struct rte_mbuf *mbuf)
+mlx5_regex_mb2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
+		 struct rte_mbuf *mbuf)
 {
-	uintptr_t addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
-	uint32_t lkey;
-
-	/* Check generation bit to see if there's any change on existing MRs. */
-	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
-		mlx5_mr_flush_local_cache(mr_ctrl);
-	/* Linear search on MR cache array. */
-	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
-				   MLX5_MR_CACHE_N, addr);
-	if (likely(lkey != UINT32_MAX))
-		return lkey;
-	/* Take slower bottom-half on miss. */
-	return mlx5_mr_addr2mr_bh(priv->cdev->pd, 0, &priv->mr_scache, mr_ctrl,
-				  addr, !!(mbuf->ol_flags & EXT_ATTACHED_MBUF));
+	return mlx5_mr_mb2mr(priv->cdev, 0, mr_ctrl, mbuf, &priv->mr_scache);
 }
 
-
 static inline void
 __prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp_obj,
 	   struct rte_regex_ops *op, struct mlx5_regex_job *job,

@@ -194,7 +180,7 @@ prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 	struct mlx5_klm klm;
 
 	klm.byte_count = rte_pktmbuf_data_len(op->mbuf);
-	klm.mkey = mlx5_regex_addr2mr(priv, &qp->mr_ctrl, op->mbuf);
+	klm.mkey = mlx5_regex_mb2mr(priv, &qp->mr_ctrl, op->mbuf);
 	klm.address = rte_pktmbuf_mtod(op->mbuf, uintptr_t);
 	__prep_one(priv, qp_obj, op, job, qp_obj->pi, &klm);
 	qp_obj->db_pi = qp_obj->pi;

@@ -317,6 +303,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 	uint32_t len = 0;
 	struct mlx5_klm *mkey_klm = NULL;
 	struct mlx5_klm klm;
+	uintptr_t addr;
 
 	while (left_ops--)
 		rte_prefetch0(op[left_ops]);

@@ -360,11 +347,12 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 			klm.mkey = rte_cpu_to_be_32
 					(qp->jobs[mkey_job_id].imkey->id);
 			while (mbuf) {
+				addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
 				/* Build indirect mkey seg's KLM. */
-				mkey_klm->mkey = mlx5_regex_addr2mr
-						(priv, &qp->mr_ctrl, mbuf);
-				mkey_klm->address = rte_cpu_to_be_64
-					(rte_pktmbuf_mtod(mbuf, uintptr_t));
+				mkey_klm->mkey = mlx5_regex_mb2mr(priv,
+								  &qp->mr_ctrl,
+								  mbuf);
+				mkey_klm->address = rte_cpu_to_be_64(addr);
 				mkey_klm->byte_count = rte_cpu_to_be_32
 						(rte_pktmbuf_data_len(mbuf));
 				/*

@@ -380,7 +368,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 			klm.byte_count = scatter_size;
 		} else {
 			/* The single mubf case. Build the KLM directly. */
-			klm.mkey = mlx5_regex_addr2mr(priv, &qp->mr_ctrl, mbuf);
+			klm.mkey = mlx5_regex_mb2mr(priv, &qp->mr_ctrl, mbuf);
 			klm.address = rte_pktmbuf_mtod(mbuf, uintptr_t);
 			klm.byte_count = rte_pktmbuf_data_len(mbuf);
 		}

index 17a54acf1e..d6acf87493 100644
@@ -308,6 +308,41 @@ mlx5_dev_to_pci_str(const struct rte_device *dev, char *addr, size_t size)
 #endif
 }
 
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+static void
+mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+		     size_t len, void *arg __rte_unused)
+{
+	struct mlx5_common_device *cdev;
+
+	/* Must be called from the primary process. */
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		pthread_mutex_lock(&devices_list_lock);
+		/* Iterate all the existing mlx5 devices. */
+		TAILQ_FOREACH(cdev, &devices_list, next)
+			mlx5_free_mr_by_addr(&cdev->mr_scache,
+					     mlx5_os_get_ctx_device_name
+								    (cdev->ctx),
+					     addr, len);
+		pthread_mutex_unlock(&devices_list_lock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
 /**
  * Uninitialize all HW global of device context.
  *

@@ -376,8 +411,13 @@ mlx5_common_dev_release(struct mlx5_common_device *cdev)
 	pthread_mutex_lock(&devices_list_lock);
 	TAILQ_REMOVE(&devices_list, cdev, next);
 	pthread_mutex_unlock(&devices_list_lock);
-	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		if (TAILQ_EMPTY(&devices_list))
+			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
+							  NULL);
+		mlx5_mr_release_cache(&cdev->mr_scache);
 		mlx5_dev_hw_global_release(cdev);
+	}
 	rte_free(cdev);
 }
 

@@ -412,6 +452,18 @@ mlx5_common_dev_create(struct rte_device *eal_dev, uint32_t classes)
 		rte_free(cdev);
 		return NULL;
 	}
+	/* Initialize global MR cache resources and update its functions. */
+	ret = mlx5_mr_create_cache(&cdev->mr_scache, eal_dev->numa_node);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to initialize global MR share cache.");
+		mlx5_dev_hw_global_release(cdev);
+		rte_free(cdev);
+		return NULL;
+	}
+	/* Register callback function for global shared MR cache management. */
+	if (TAILQ_EMPTY(&devices_list))
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_mr_mem_event_cb, NULL);
 exit:
 	pthread_mutex_lock(&devices_list_lock);
 	TAILQ_INSERT_HEAD(&devices_list, cdev, next);

index 8df4f32aa2..1a6b8c0f52 100644
@@ -350,6 +350,7 @@ struct mlx5_common_device {
 	void *ctx; /* Verbs/DV/DevX context. */
 	void *pd; /* Protection Domain. */
 	uint32_t pdn; /* Protection Domain Number. */
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_common_dev_config config; /* Device configuration. */
 };
 

@@ -453,8 +454,7 @@ mlx5_dev_is_pci(const struct rte_device *dev);
 __rte_internal
 uint32_t
 mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
-	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
-	      struct mlx5_mr_share_cache *share_cache);
+	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf);
 
 /* mlx5_common_os.c */
 

index 4de1c25f2a..d63e973b60 100644
@@ -1848,16 +1848,13 @@ mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
  *   Pointer to per-queue MR control structure.
  * @param mbuf
  *   Pointer to mbuf.
- * @param share_cache
- *   Pointer to a global shared MR cache.
  *
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 uint32_t
 mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
-	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
-	      struct mlx5_mr_share_cache *share_cache)
+	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf)
 {
 	uint32_t lkey;
 	uintptr_t addr = (uintptr_t)mbuf->buf_addr;

@@ -1871,6 +1868,6 @@ mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
 	if (likely(lkey != UINT32_MAX))
 		return lkey;
 	/* Take slower bottom-half on miss. */
-	return mlx5_mr_addr2mr_bh(cdev->pd, mp_id, share_cache, mr_ctrl,
+	return mlx5_mr_addr2mr_bh(cdev->pd, mp_id, &cdev->mr_scache, mr_ctrl,
 				  addr, cdev->config.mr_ext_memseg_en);
 }

index 36689dfb54..0bc3519fd9 100644
@@ -140,9 +140,7 @@ __rte_internal
 uint32_t mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
 			       struct mlx5_mr_ctrl *mr_ctrl,
 			       struct rte_mempool *mp, uintptr_t addr);
-__rte_internal
 void mlx5_mr_release_cache(struct mlx5_mr_share_cache *mr_cache);
-__rte_internal
 int mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket);
 __rte_internal
 void mlx5_mr_dump_cache(struct mlx5_mr_share_cache *share_cache __rte_unused);

@@ -150,7 +148,6 @@ __rte_internal
 void mlx5_mr_rebuild_cache(struct mlx5_mr_share_cache *share_cache);
 __rte_internal
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
-__rte_internal
 void mlx5_free_mr_by_addr(struct mlx5_mr_share_cache *share_cache,
 			  const char *ibdev_name, const void *addr, size_t len);
 __rte_internal

@@ -183,7 +180,6 @@ __rte_internal
 void
 mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
 
-__rte_internal
 void
 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
 

93 lines not on this page · raw message

92 lines not on this page · raw message

index 030f369423..69cef81d77 100644
@@ -26,7 +26,6 @@ struct mlx5_crypto_priv {
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
-	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
 	uint16_t wqe_set_size;

index 286a7caf36..c3b6495d9e 100644
@@ -91,7 +91,7 @@ mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
 	case MLX5_MP_REQ_CREATE_MR:
 		mp_init_msg(&priv->mp_id, &mp_res, param->type);
 		lkey = mlx5_mr_create_primary(cdev->pd,
-					      &priv->sh->share_cache,
+					      &priv->sh->cdev->mr_scache,
 					      &entry, param->args.addr,
 					      cdev->config.mr_ext_memseg_en);
 		if (lkey == UINT32_MAX)

index 9e445f2f9b..61c4870d8c 100644
@@ -44,7 +44,6 @@
 #include "mlx5_rx.h"
 #include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 #include "mlx5_flow.h"
 #include "rte_pmd_mlx5.h"
 #include "mlx5_verbs.h"

@@ -623,10 +622,6 @@ mlx5_init_once(void)
 	case RTE_PROC_PRIMARY:
 		if (sd->init_done)
 			break;
-		LIST_INIT(&sd->mem_event_cb_list);
-		rte_rwlock_init(&sd->mem_event_rwlock);
-		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
-						mlx5_mr_mem_event_cb, NULL);
 		ret = mlx5_mp_init_primary(MLX5_MP_NAME,
 					   mlx5_mp_os_primary_handle);
 		if (ret)

index a6c196b368..91aa5c0c75 100644
@@ -36,7 +36,6 @@
 #include "mlx5_rx.h"
 #include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
 #include "rte_pmd_mlx5.h"

@@ -1112,7 +1111,7 @@ mlx5_dev_ctx_shared_mempool_unregister(struct mlx5_dev_ctx_shared *sh,
 	struct mlx5_mp_id mp_id;
 
 	mlx5_mp_id_init(&mp_id, 0);
-	if (mlx5_mr_mempool_unregister(&sh->share_cache, mp, &mp_id) < 0)
+	if (mlx5_mr_mempool_unregister(&sh->cdev->mr_scache, mp, &mp_id) < 0)
 		DRV_LOG(WARNING, "Failed to unregister mempool %s for PD %p: %s",
 			mp->name, sh->cdev->pd, rte_strerror(rte_errno));
 }

@@ -1134,7 +1133,7 @@ mlx5_dev_ctx_shared_mempool_register_cb(struct rte_mempool *mp, void *arg)
 	int ret;
 
 	mlx5_mp_id_init(&mp_id, 0);
-	ret = mlx5_mr_mempool_register(&sh->share_cache, sh->cdev->pd, mp,
+	ret = mlx5_mr_mempool_register(&sh->cdev->mr_scache, sh->cdev->pd, mp,
 				       &mp_id);
 	if (ret < 0 && rte_errno != EEXIST)
 		DRV_LOG(ERR, "Failed to register existing mempool %s for PD %p: %s",

@@ -1177,8 +1176,8 @@ mlx5_dev_ctx_shared_mempool_event_cb(enum rte_mempool_event event,
 	switch (event) {
 	case RTE_MEMPOOL_EVENT_READY:
 		mlx5_mp_id_init(&mp_id, 0);
-		if (mlx5_mr_mempool_register(&sh->share_cache, sh->cdev->pd, mp,
-					     &mp_id) < 0)
+		if (mlx5_mr_mempool_register(&sh->cdev->mr_scache, sh->cdev->pd,
+					     mp, &mp_id) < 0)
 			DRV_LOG(ERR, "Failed to register new mempool %s for PD %p: %s",
 				mp->name, sh->cdev->pd,
 				rte_strerror(rte_errno));

@@ -1342,20 +1341,6 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++)
 		rte_spinlock_init(&sh->uar_lock[i]);
 #endif
-	/*
-	 * Once the device is added to the list of memory event
-	 * callback, its global MR cache table cannot be expanded
-	 * on the fly because of deadlock. If it overflows, lookup
-	 * should be done by searching MR list linearly, which is slow.
-	 *
-	 * At this point the device is not added to the memory
-	 * event list yet, context is just being created.
-	 */
-	err = mlx5_mr_create_cache(&sh->share_cache, sh->numa_node);
-	if (err) {
-		err = rte_errno;
-		goto error;
-	}
 	mlx5_os_dev_shared_handler_install(sh);
 	sh->cnt_id_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 	if (!sh->cnt_id_tbl) {

@@ -1370,11 +1355,6 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	mlx5_flow_aging_init(sh);
 	mlx5_flow_counters_mng_init(sh);
 	mlx5_flow_ipool_create(sh, config);
-	/* Add device to memory callback list. */
-	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
-	LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
-			 sh, mem_event_cb);
-	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
 	/* Add context to the global device list. */
 	LIST_INSERT_HEAD(&mlx5_dev_ctx_list, sh, next);
 	rte_spinlock_init(&sh->geneve_tlv_opt_sl);

@@ -1387,8 +1367,6 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	MLX5_ASSERT(sh);
 	if (sh->cnt_id_tbl)
 		mlx5_l3t_destroy(sh->cnt_id_tbl);
-	if (sh->share_cache.cache.table)
-		mlx5_mr_btree_free(&sh->share_cache.cache);
 	if (sh->tis)
 		claim_zero(mlx5_devx_cmd_destroy(sh->tis));
 	if (sh->td)

@@ -1444,12 +1422,6 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
 	if (ret == 0)
 		rte_mempool_walk(mlx5_dev_ctx_shared_mempool_unregister_cb,
 				 sh);
-	/* Remove from memory callback device list. */
-	rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
-	LIST_REMOVE(sh, mem_event_cb);
-	rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
-	/* Release created Memory Regions. */
-	mlx5_mr_release_cache(&sh->share_cache);
 	/* Remove context from the global device list. */
 	LIST_REMOVE(sh, next);
 	/* Release flow workspaces objects on the last device. */

index 5c25b94f36..4f823baa6d 100644
@@ -1142,9 +1142,6 @@ struct mlx5_dev_ctx_shared {
 	char ibdev_path[MLX5_FS_PATH_MAX]; /* SYSFS dev path for secondary */
 	struct mlx5_dev_attr device_attr; /* Device properties. */
 	int numa_node; /* Numa node of backing physical device. */
-	LIST_ENTRY(mlx5_dev_ctx_shared) mem_event_cb;
-	/**< Called by memory event callback. */
-	struct mlx5_mr_share_cache share_cache;
 	/* Packet pacing related structure. */
 	struct mlx5_dev_txpp txpp;
 	/* Shared DV/DR flow data section. */

index 8f3d2ffc2c..1fc1000b01 100644
@@ -60,17 +60,17 @@ mlx5_aso_cq_create(void *ctx, struct mlx5_aso_cq *cq, uint16_t log_desc_n,
 /**
  * Free MR resources.
  *
- * @param[in] sh
- *   Pointer to shared device context.
+ * @param[in] cdev
+ *   Pointer to the mlx5 common device.
  * @param[in] mr
  *   MR to free.
  */
 static void
-mlx5_aso_dereg_mr(struct mlx5_dev_ctx_shared *sh, struct mlx5_pmd_mr *mr)
+mlx5_aso_dereg_mr(struct mlx5_common_device *cdev, struct mlx5_pmd_mr *mr)
 {
 	void *addr = mr->addr;
 
-	sh->share_cache.dereg_mr_cb(mr);
+	cdev->mr_scache.dereg_mr_cb(mr);
 	mlx5_free(addr);
 	memset(mr, 0, sizeof(*mr));
 }

@@ -78,8 +78,8 @@ mlx5_aso_dereg_mr(struct mlx5_dev_ctx_shared *sh, struct mlx5_pmd_mr *mr)
 /**
  * Register Memory Region.
  *
- * @param[in] sh
- *   Pointer to shared device context.
+ * @param[in] cdev
+ *   Pointer to the mlx5 common device.
  * @param[in] length
  *   Size of MR buffer.
  * @param[in/out] mr

@@ -91,7 +91,7 @@ mlx5_aso_dereg_mr(struct mlx5_dev_ctx_shared *sh, struct mlx5_pmd_mr *mr)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx5_aso_reg_mr(struct mlx5_dev_ctx_shared *sh, size_t length,
+mlx5_aso_reg_mr(struct mlx5_common_device *cdev, size_t length,
 		struct mlx5_pmd_mr *mr, int socket)
 {
 

@@ -103,7 +103,7 @@ mlx5_aso_reg_mr(struct mlx5_dev_ctx_shared *sh, size_t length,
 		DRV_LOG(ERR, "Failed to create ASO bits mem for MR.");
 		return -1;
 	}
-	ret = sh->share_cache.reg_mr_cb(sh->cdev->pd, mr->addr, length, mr);
+	ret = cdev->mr_scache.reg_mr_cb(cdev->pd, mr->addr, length, mr);
 	if (ret) {
 		DRV_LOG(ERR, "Failed to create direct Mkey.");
 		mlx5_free(mr->addr);

@@ -313,14 +313,14 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 
 	switch (aso_opc_mod) {
 	case ASO_OPC_MOD_FLOW_HIT:
-		if (mlx5_aso_reg_mr(sh, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
+		if (mlx5_aso_reg_mr(cdev, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) *
 				    sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0))
 			return -1;
 		if (mlx5_aso_sq_create(cdev->ctx, &sh->aso_age_mng->aso_sq, 0,
 				       sh->tx_uar, cdev->pdn,
 				       MLX5_ASO_QUEUE_LOG_DESC,
 				       cdev->config.hca_attr.sq_ts_format)) {
-			mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
+			mlx5_aso_dereg_mr(cdev, &sh->aso_age_mng->aso_sq.mr);
 			return -1;
 		}
 		mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq);

@@ -335,14 +335,14 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh,
 		break;
 	case ASO_OPC_MOD_CONNECTION_TRACKING:
 		/* 64B per object for query. */
-		if (mlx5_aso_reg_mr(sh, 64 * sq_desc_n,
+		if (mlx5_aso_reg_mr(cdev, 64 * sq_desc_n,
 				    &sh->ct_mng->aso_sq.mr, 0))
 			return -1;
 		if (mlx5_aso_sq_create(cdev->ctx, &sh->ct_mng->aso_sq, 0,
 				       sh->tx_uar, cdev->pdn,
 				       MLX5_ASO_QUEUE_LOG_DESC,
 				       cdev->config.hca_attr.sq_ts_format)) {
-			mlx5_aso_dereg_mr(sh, &sh->ct_mng->aso_sq.mr);
+			mlx5_aso_dereg_mr(cdev, &sh->ct_mng->aso_sq.mr);
 			return -1;
 		}
 		mlx5_aso_ct_init_sq(&sh->ct_mng->aso_sq);

@@ -370,14 +370,14 @@ mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh,
 
 	switch (aso_opc_mod) {
 	case ASO_OPC_MOD_FLOW_HIT:
-		mlx5_aso_dereg_mr(sh, &sh->aso_age_mng->aso_sq.mr);
+		mlx5_aso_dereg_mr(sh->cdev, &sh->aso_age_mng->aso_sq.mr);
 		sq = &sh->aso_age_mng->aso_sq;
 		break;
 	case ASO_OPC_MOD_POLICER:
 		sq = &sh->mtrmng->pools_mng.sq;
 		break;
 	case ASO_OPC_MOD_CONNECTION_TRACKING:
-		mlx5_aso_dereg_mr(sh, &sh->ct_mng->aso_sq.mr);
+		mlx5_aso_dereg_mr(sh->cdev, &sh->ct_mng->aso_sq.mr);
 		sq = &sh->ct_mng->aso_sq;
 		break;
 	default:

141 lines not on this page · raw message

deleted file mode 100644
index c984e777b5..0000000000
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2018 6WIND S.A.
- * Copyright 2018 Mellanox Technologies, Ltd
- */
-
-#ifndef RTE_PMD_MLX5_MR_H_
-#define RTE_PMD_MLX5_MR_H_
-
-#include <stddef.h>
-#include <stdint.h>
-#include <sys/queue.h>
-
-#include <rte_ethdev.h>
-#include <rte_rwlock.h>
-#include <rte_bitmap.h>
-#include <rte_memory.h>
-
-#include <mlx5_common_mr.h>
-
-/* First entry must be NULL for comparison. */
-#define mlx5_mr_btree_len(bt) ((bt)->len - 1)
-
-void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
-			  size_t len, void *arg);
-
-#endif /* RTE_PMD_MLX5_MR_H_ */

index e3b1051ba4..c83c7f4a39 100644
@@ -22,7 +22,6 @@
 #include "mlx5_autoconf.h"
 #include "mlx5_defs.h"
 #include "mlx5.h"
-#include "mlx5_mr.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"

index 3952e64d50..42a12151fc 100644
@@ -18,11 +18,13 @@
 
 #include "mlx5.h"
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 
 /* Support tunnel matching. */
 #define MLX5_FLOW_TUNNEL 10
 
+/* First entry must be NULL for comparison. */
+#define mlx5_mr_btree_len(bt) ((bt)->len - 1)
+
 struct mlx5_rxq_stats {
 #ifdef MLX5_PMD_SOFT_COUNTERS
 	uint64_t ipackets; /**< Total of successfully received packets. */

@@ -309,7 +311,7 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
 	 */
 	rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 	mp = mlx5_rxq_mprq_enabled(rxq) ? rxq->mprq_mp : rxq->mp;
-	return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->share_cache,
+	return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
 				     mr_ctrl, mp, addr);
 }
 

index 479ea1324c..174899e661 100644
@@ -1245,7 +1245,7 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev)
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
-	ret = mlx5_mr_mempool_register(&priv->sh->share_cache,
+	ret = mlx5_mr_mempool_register(&priv->sh->cdev->mr_scache,
 				       priv->sh->cdev->pd, mp, &priv->mp_id);
 	if (ret < 0 && rte_errno != EEXIST) {
 		ret = rte_errno;

@@ -1453,7 +1453,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	}
 	tmpl->type = MLX5_RXQ_TYPE_STANDARD;
 	if (mlx5_mr_ctrl_init(&tmpl->rxq.mr_ctrl,
-			      &priv->sh->share_cache.dev_gen, socket)) {
+			      &priv->sh->cdev->mr_scache.dev_gen, socket)) {
 		/* rte_errno is already set. */
 		goto error;
 	}

index 7b984eff35..ed1f2d2c8c 100644
@@ -22,7 +22,6 @@
 #include "mlx5_autoconf.h"
 #include "mlx5_defs.h"
 #include "mlx5.h"
-#include "mlx5_mr.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_rx.h"

index ad1144e218..b400295e7d 100644
@@ -24,7 +24,6 @@
 #include "mlx5_utils.h"
 #include "mlx5.h"
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 
 struct mlx5_priv;
 

index 93b4f517bb..1aec72817e 100644
@@ -12,7 +12,6 @@
 #include <mlx5_prm.h>
 
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 
 /* HW checksum offload capabilities of vectorized Tx. */
 #define MLX5_VEC_TX_CKSUM_OFFLOAD_CAP \

index cf4fbd3c9f..54c2893437 100644
@@ -14,7 +14,6 @@
 #include <mlx5_malloc.h>
 
 #include "mlx5.h"
-#include "mlx5_mr.h"
 #include "mlx5_rx.h"
 #include "mlx5_tx.h"
 #include "mlx5_utils.h"

@@ -148,7 +147,7 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
 	}
 	for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
 		mp = rxq_ctrl->rxq.rxseg[s].mp;
-		ret = mlx5_mr_mempool_register(&priv->sh->share_cache,
+		ret = mlx5_mr_mempool_register(&priv->sh->cdev->mr_scache,
 					       priv->sh->cdev->pd, mp,
 					       &priv->mp_id);
 		if (ret < 0 && rte_errno != EEXIST)

index df671379e4..2cc9ae6772 100644
@@ -22,7 +22,6 @@
 #include "mlx5_autoconf.h"
 #include "mlx5_defs.h"
 #include "mlx5.h"
-#include "mlx5_mr.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
 #include "mlx5_tx.h"

index a6aa34084c..1f124b92e6 100644
@@ -18,7 +18,6 @@
 
 #include "mlx5.h"
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 
 /* TX burst subroutines return codes. */
 enum mlx5_txcmp_code {

index 4b027d6807..2c3c134f6f 100644
@@ -1125,7 +1125,7 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		return NULL;
 	}
 	if (mlx5_mr_ctrl_init(&tmpl->txq.mr_ctrl,
-			      &priv->sh->share_cache.dev_gen, socket)) {
+			      &priv->sh->cdev->mr_scache.dev_gen, socket)) {
 		/* rte_errno is already set. */
 		goto error;
 	}

index c3d4b90946..afdfff8b36 100644
@@ -26,7 +26,6 @@
 #include "mlx5_rx.h"
 #include "mlx5_tx.h"
 #include "mlx5_autoconf.h"
-#include "mlx5_mr.h"
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 

@@ -122,21 +121,8 @@ mlx5_init_shared_data(void)
 static int
 mlx5_init_once(void)
 {
-	struct mlx5_shared_data *sd;
-
 	if (mlx5_init_shared_data())
 		return -rte_errno;
-	sd = mlx5_shared_data;
-	rte_spinlock_lock(&sd->lock);
-	MLX5_ASSERT(sd);
-	if (!sd->init_done) {
-		LIST_INIT(&sd->mem_event_cb_list);
-		rte_rwlock_init(&sd->mem_event_rwlock);
-		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
-						mlx5_mr_mem_event_cb, NULL);
-		sd->init_done = true;
-	}
-	rte_spinlock_unlock(&sd->lock);
 	return 0;
 }
 

87 lines not on this page · raw message

index be81931b3a..eb59cc38a6 100644
@@ -68,9 +68,6 @@ struct mlx5_regex_priv {
 				MLX5_RXP_EM_COUNT];
 	uint32_t nb_engines; /* Number of RegEx engines. */
 	struct mlx5dv_devx_uar *uar; /* UAR object. */
-	TAILQ_ENTRY(mlx5_regex_priv) mem_event_cb;
-	/**< Called by memory event callback. */
-	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	uint8_t is_bf2; /* The device is BF2 device. */
 	uint8_t has_umr; /* The device supports UMR. */
 	uint32_t mmo_regex_qp_cap:1;

index 6735e51976..50c966a022 100644
@@ -242,7 +242,7 @@ mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind,
 		nb_sq_config++;
 	}
 
-	ret = mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->mr_scache.dev_gen,
+	ret = mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
 				rte_socket_id());
 	if (ret) {
 		DRV_LOG(ERR, "Error setting up mr btree");

index 8817e2e074..adb5343a46 100644
@@ -126,7 +126,7 @@ static inline uint32_t
 mlx5_regex_mb2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
 		 struct rte_mbuf *mbuf)
 {
-	return mlx5_mr_mb2mr(priv->cdev, 0, mr_ctrl, mbuf, &priv->mr_scache);
+	return mlx5_mr_mb2mr(priv->cdev, 0, mr_ctrl, mbuf);
 }
 
 static inline void

index d6acf87493..0ed1477eb8 100644
@@ -258,12 +258,6 @@ is_valid_class_combination(uint32_t user_classes)
 	return 0;
 }
 
-static bool
-device_class_enabled(const struct mlx5_common_device *device, uint32_t class)
-{
-	return (device->classes_loaded & class) > 0;
-}
-
 static bool
 mlx5_bus_match(const struct mlx5_class_driver *drv,
 	       const struct rte_device *dev)

@@ -597,62 +591,106 @@ mlx5_common_dev_remove(struct rte_device *eal_dev)
 	return ret;
 }
 
+/**
+ * Callback to DMA map external memory to a device.
+ *
+ * @param rte_dev
+ *   Pointer to the generic device.
+ * @param addr
+ *   Starting virtual address of memory to be mapped.
+ * @param iova
+ *   Starting IOVA address of memory to be mapped.
+ * @param len
+ *   Length of memory segment being mapped.
+ *
+ * @return
+ *   0 on success, negative value on error.
+ */
 int
-mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova,
-			size_t len)
+mlx5_common_dev_dma_map(struct rte_device *rte_dev, void *addr,
+			uint64_t iova __rte_unused, size_t len)
 {
-	struct mlx5_class_driver *driver = NULL;
-	struct mlx5_class_driver *temp;
-	struct mlx5_common_device *mdev;
-	int ret = -EINVAL;
-
-	mdev = to_mlx5_device(dev);
-	if (!mdev)
-		return -ENODEV;
-	TAILQ_FOREACH(driver, &drivers_list, next) {
-		if (!device_class_enabled(mdev, driver->drv_class) ||
-		    driver->dma_map == NULL)
-			continue;
-		ret = driver->dma_map(dev, addr, iova, len);
-		if (ret)
-			goto map_err;
+	struct mlx5_common_device *dev;
+	struct mlx5_mr *mr;
+
+	dev = to_mlx5_device(rte_dev);
+	if (!dev) {
+		DRV_LOG(WARNING,
+			"Unable to find matching mlx5 device to device %s",
+			rte_dev->name);
+		rte_errno = ENODEV;
+		return -1;
 	}
-	return ret;
-map_err:
-	TAILQ_FOREACH(temp, &drivers_list, next) {
-		if (temp == driver)
-			break;
-		if (device_class_enabled(mdev, temp->drv_class) &&
-		    temp->dma_map && temp->dma_unmap)
-			temp->dma_unmap(dev, addr, iova, len);
+	mr = mlx5_create_mr_ext(dev->pd, (uintptr_t)addr, len,
+				SOCKET_ID_ANY, dev->mr_scache.reg_mr_cb);
+	if (!mr) {
+		DRV_LOG(WARNING, "Device %s unable to DMA map", rte_dev->name);
+		rte_errno = EINVAL;
+		return -1;
 	}
-	return ret;
+	rte_rwlock_write_lock(&dev->mr_scache.rwlock);
+	LIST_INSERT_HEAD(&dev->mr_scache.mr_list, mr, mr);
+	/* Insert to the global cache table. */
+	mlx5_mr_insert_cache(&dev->mr_scache, mr);
+	rte_rwlock_write_unlock(&dev->mr_scache.rwlock);
+	return 0;
 }
 
+/**
+ * Callback to DMA unmap external memory to a device.
+ *
+ * @param rte_dev
+ *   Pointer to the generic device.
+ * @param addr
+ *   Starting virtual address of memory to be unmapped.
+ * @param iova
+ *   Starting IOVA address of memory to be unmapped.
+ * @param len
+ *   Length of memory segment being unmapped.
+ *
+ * @return
+ *   0 on success, negative value on error.
+ */
 int
-mlx5_common_dev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova,
-			  size_t len)
+mlx5_common_dev_dma_unmap(struct rte_device *rte_dev, void *addr,
+			  uint64_t iova __rte_unused, size_t len __rte_unused)
 {
-	struct mlx5_class_driver *driver;
-	struct mlx5_common_device *mdev;
-	int local_ret = -EINVAL;
-	int ret = 0;
-
-	mdev = to_mlx5_device(dev);
-	if (!mdev)
-		return -ENODEV;
-	/* There is no unmap error recovery in current implementation. */
-	TAILQ_FOREACH_REVERSE(driver, &drivers_list, mlx5_drivers, next) {
-		if (!device_class_enabled(mdev, driver->drv_class) ||
-		    driver->dma_unmap == NULL)
-			continue;
-		local_ret = driver->dma_unmap(dev, addr, iova, len);
-		if (local_ret && (ret == 0))
-			ret = local_ret;
+	struct mlx5_common_device *dev;
+	struct mr_cache_entry entry;
+	struct mlx5_mr *mr;
+
+	dev = to_mlx5_device(rte_dev);
+	if (!dev) {
+		DRV_LOG(WARNING,
+			"Unable to find matching mlx5 device to device %s.",
+			rte_dev->name);
+		rte_errno = ENODEV;
+		return -1;
 	}
-	if (local_ret)
-		ret = local_ret;
-	return ret;
+	rte_rwlock_read_lock(&dev->mr_scache.rwlock);
+	mr = mlx5_mr_lookup_list(&dev->mr_scache, &entry, (uintptr_t)addr);
+	if (!mr) {
+		rte_rwlock_read_unlock(&dev->mr_scache.rwlock);
+		DRV_LOG(WARNING,
+			"Address 0x%" PRIxPTR " wasn't registered to device %s",
+			(uintptr_t)addr, rte_dev->name);
+		rte_errno = EINVAL;
+		return -1;
+	}
+	LIST_REMOVE(mr, mr);
+	DRV_LOG(DEBUG, "MR(%p) is removed from list.", (void *)mr);
+	mlx5_mr_free(mr, dev->mr_scache.dereg_mr_cb);
+	mlx5_mr_rebuild_cache(&dev->mr_scache);
+	/*
+	 * No explicit wmb is needed after updating dev_gen due to
+	 * store-release ordering in unlock that provides the
+	 * implicit barrier at the software visible level.
+	 */
+	++dev->mr_scache.dev_gen;
+	DRV_LOG(DEBUG, "Broadcasting local cache flush, gen=%d.",
+		dev->mr_scache.dev_gen);
+	rte_rwlock_read_unlock(&dev->mr_scache.rwlock);
+	return 0;
 }
 
 void

53 lines not on this page · raw message

index d63e973b60..5bfddac08e 100644
@@ -455,7 +455,7 @@ mlx5_mr_lookup_list(struct mlx5_mr_share_cache *share_cache,
  * @return
  *   Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
  */
-uint32_t
+static uint32_t
 mlx5_mr_lookup_cache(struct mlx5_mr_share_cache *share_cache,
 		     struct mr_cache_entry *entry, uintptr_t addr)
 {

index 0bc3519fd9..8a7af05ca5 100644
@@ -124,12 +124,13 @@ mlx5_mr_lookup_lkey(struct mr_cache_entry *lkp_tbl, uint16_t *cached_idx,
 	return UINT32_MAX;
 }
 
+/* mlx5_common_mr.c */
+
 __rte_internal
 int mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
 		      int socket);
 __rte_internal
 void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
-__rte_internal
 void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused);
 __rte_internal
 uint32_t mlx5_mr_addr2mr_bh(void *pd, struct mlx5_mp_id *mp_id,

@@ -142,36 +143,30 @@ uint32_t mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
 			       struct rte_mempool *mp, uintptr_t addr);
 void mlx5_mr_release_cache(struct mlx5_mr_share_cache *mr_cache);
 int mlx5_mr_create_cache(struct mlx5_mr_share_cache *share_cache, int socket);
-__rte_internal
 void mlx5_mr_dump_cache(struct mlx5_mr_share_cache *share_cache __rte_unused);
-__rte_internal
 void mlx5_mr_rebuild_cache(struct mlx5_mr_share_cache *share_cache);
 __rte_internal
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
 void mlx5_free_mr_by_addr(struct mlx5_mr_share_cache *share_cache,
 			  const char *ibdev_name, const void *addr, size_t len);
-__rte_internal
-int
-mlx5_mr_insert_cache(struct mlx5_mr_share_cache *share_cache,
-		     struct mlx5_mr *mr);
-__rte_internal
-uint32_t
-mlx5_mr_lookup_cache(struct mlx5_mr_share_cache *share_cache,
-		     struct mr_cache_entry *entry, uintptr_t addr);
-__rte_internal
+int mlx5_mr_insert_cache(struct mlx5_mr_share_cache *share_cache,
+			 struct mlx5_mr *mr);
 struct mlx5_mr *
 mlx5_mr_lookup_list(struct mlx5_mr_share_cache *share_cache,
 		    struct mr_cache_entry *entry, uintptr_t addr);
-__rte_internal
 struct mlx5_mr *
 mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
 		   mlx5_reg_mr_t reg_mr_cb);
+void mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
 __rte_internal
 uint32_t
 mlx5_mr_create_primary(void *pd,
 		       struct mlx5_mr_share_cache *share_cache,
 		       struct mr_cache_entry *entry, uintptr_t addr,
 		       unsigned int mr_ext_memseg_en);
+
+/* mlx5_common_verbs.c */
+
 __rte_internal
 int
 mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,

@@ -183,10 +178,6 @@ mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
 void
 mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
 
-__rte_internal
-void
-mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
-
 __rte_internal
 int
 mlx5_mr_mempool_register(struct mlx5_mr_share_cache *share_cache, void *pd,

index 91aa5c0c75..17113be873 100644
@@ -2589,8 +2589,6 @@ static struct mlx5_class_driver mlx5_net_driver = {
 	.id_table = mlx5_pci_id_map,
 	.probe = mlx5_os_net_probe,
 	.remove = mlx5_net_remove,
-	.dma_map = mlx5_net_dma_map,
-	.dma_unmap = mlx5_net_dma_unmap,
 	.probe_again = 1,
 	.intr_lsc = 1,
 	.intr_rmv = 1,

135 lines not on this page · raw message

index 0ed1477eb8..e6ff045c95 100644
@@ -13,6 +13,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
+#include "mlx5_common_mp.h"
 #include "mlx5_common_log.h"
 #include "mlx5_common_defs.h"
 #include "mlx5_common_private.h"

@@ -302,6 +303,152 @@ mlx5_dev_to_pci_str(const struct rte_device *dev, char *addr, size_t size)
 #endif
 }
 
+/**
+ * Register the mempool for the protection domain.
+ *
+ * @param cdev
+ *   Pointer to the mlx5 common device.
+ * @param mp
+ *   Mempool being registered.
+ *
+ * @return
+ *   0 on success, (-1) on failure and rte_errno is set.
+ */
+static int
+mlx5_dev_mempool_register(struct mlx5_common_device *cdev,
+			  struct rte_mempool *mp)
+{
+	struct mlx5_mp_id mp_id;
+
+	mlx5_mp_id_init(&mp_id, 0);
+	return mlx5_mr_mempool_register(&cdev->mr_scache, cdev->pd, mp, &mp_id);
+}
+
+/**
+ * Unregister the mempool from the protection domain.
+ *
+ * @param cdev
+ *   Pointer to the mlx5 common device.
+ * @param mp
+ *   Mempool being unregistered.
+ */
+void
+mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
+			    struct rte_mempool *mp)
+{
+	struct mlx5_mp_id mp_id;
+
+	mlx5_mp_id_init(&mp_id, 0);
+	if (mlx5_mr_mempool_unregister(&cdev->mr_scache, mp, &mp_id) < 0)
+		DRV_LOG(WARNING, "Failed to unregister mempool %s for PD %p: %s",
+			mp->name, cdev->pd, rte_strerror(rte_errno));
+}
+
+/**
+ * rte_mempool_walk() callback to register mempools for the protection domain.
+ *
+ * @param mp
+ *   The mempool being walked.
+ * @param arg
+ *   Pointer to the device shared context.
+ */
+static void
+mlx5_dev_mempool_register_cb(struct rte_mempool *mp, void *arg)
+{
+	struct mlx5_common_device *cdev = arg;
+	int ret;
+
+	ret = mlx5_dev_mempool_register(cdev, mp);
+	if (ret < 0 && rte_errno != EEXIST)
+		DRV_LOG(ERR,
+			"Failed to register existing mempool %s for PD %p: %s",
+			mp->name, cdev->pd, rte_strerror(rte_errno));
+}
+
+/**
+ * rte_mempool_walk() callback to unregister mempools
+ * from the protection domain.
+ *
+ * @param mp
+ *   The mempool being walked.
+ * @param arg
+ *   Pointer to the device shared context.
+ */
+static void
+mlx5_dev_mempool_unregister_cb(struct rte_mempool *mp, void *arg)
+{
+	mlx5_dev_mempool_unregister((struct mlx5_common_device *)arg, mp);
+}
+
+/**
+ * Mempool life cycle callback for mlx5 common devices.
+ *
+ * @param event
+ *   Mempool life cycle event.
+ * @param mp
+ *   Associated mempool.
+ * @param arg
+ *   Pointer to a device shared context.
+ */
+static void
+mlx5_dev_mempool_event_cb(enum rte_mempool_event event, struct rte_mempool *mp,
+			  void *arg)
+{
+	struct mlx5_common_device *cdev = arg;
+
+	switch (event) {
+	case RTE_MEMPOOL_EVENT_READY:
+		if (mlx5_dev_mempool_register(cdev, mp) < 0)
+			DRV_LOG(ERR,
+				"Failed to register new mempool %s for PD %p: %s",
+				mp->name, cdev->pd, rte_strerror(rte_errno));
+		break;
+	case RTE_MEMPOOL_EVENT_DESTROY:
+		mlx5_dev_mempool_unregister(cdev, mp);
+		break;
+	}
+}
+
+int
+mlx5_dev_mempool_subscribe(struct mlx5_common_device *cdev)
+{
+	int ret = 0;
+
+	if (!cdev->config.mr_mempool_reg_en)
+		return 0;
+	rte_rwlock_write_lock(&cdev->mr_scache.mprwlock);
+	if (cdev->mr_scache.mp_cb_registered)
+		goto exit;
+	/* Callback for this device may be already registered. */
+	ret = rte_mempool_event_callback_register(mlx5_dev_mempool_event_cb,
+						  cdev);
+	if (ret != 0 && rte_errno != EEXIST)
+		goto exit;
+	/* Register mempools only once for this device. */
+	if (ret == 0)
+		rte_mempool_walk(mlx5_dev_mempool_register_cb, cdev);
+	ret = 0;
+	cdev->mr_scache.mp_cb_registered = 1;
+exit:
+	rte_rwlock_write_unlock(&cdev->mr_scache.mprwlock);
+	return ret;
+}
+
+static void
+mlx5_dev_mempool_unsubscribe(struct mlx5_common_device *cdev)
+{
+	int ret;
+
+	if (!cdev->mr_scache.mp_cb_registered ||
+	    !cdev->config.mr_mempool_reg_en)
+		return;
+	/* Stop watching for mempool events and unregister all mempools. */
+	ret = rte_mempool_event_callback_unregister(mlx5_dev_mempool_event_cb,
+						    cdev);
+	if (ret == 0)
+		rte_mempool_walk(mlx5_dev_mempool_unregister_cb, cdev);
+}
+
 /**
  * Callback for memory event.
  *

@@ -409,6 +556,7 @@ mlx5_common_dev_release(struct mlx5_common_device *cdev)
 		if (TAILQ_EMPTY(&devices_list))
 			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
 							  NULL);
+		mlx5_dev_mempool_unsubscribe(cdev);
 		mlx5_mr_release_cache(&cdev->mr_scache);
 		mlx5_dev_hw_global_release(cdev);
 	}

index 72ff0ff809..744c6a72b3 100644
@@ -408,6 +408,15 @@ __rte_internal
 bool
 mlx5_dev_is_pci(const struct rte_device *dev);
 
+__rte_internal
+int
+mlx5_dev_mempool_subscribe(struct mlx5_common_device *cdev);
+
+__rte_internal
+void
+mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
+			    struct rte_mempool *mp);
+
 /* mlx5_common_mr.c */
 
 __rte_internal

index 527bf3cad8..2276dc921c 100644
@@ -64,6 +64,17 @@ struct mlx5_mp_id {
 	uint16_t port_id;
 };
 
+/** Key string for IPC. */
+#define MLX5_MP_NAME "common_mlx5_mp"
+
+/** Initialize a multi-process ID. */
+static inline void
+mlx5_mp_id_init(struct mlx5_mp_id *mp_id, uint16_t port_id)
+{
+	mp_id->port_id = port_id;
+	strlcpy(mp_id->name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
+}
+
 /** Request timeout for IPC. */
 #define MLX5_MP_REQ_TIMEOUT_SEC 5
 

160 lines not on this page · raw message

59 lines not on this page · raw message

index 91f5ffdf87..bc42b2f755 100644
@@ -386,8 +386,9 @@ mlx5_compress_dev_stop(struct rte_compressdev *dev)
 static int
 mlx5_compress_dev_start(struct rte_compressdev *dev)
 {
-	RTE_SET_USED(dev);
-	return 0;
+	struct mlx5_compress_priv *priv = dev->data->dev_private;
+
+	return mlx5_dev_mempool_subscribe(priv->cdev);
 }
 
 static void

index 064501ba8c..4cf4c0ec8e 100644
@@ -142,8 +142,9 @@ mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
 static int
 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
 {
-	RTE_SET_USED(dev);
-	return 0;
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	return mlx5_dev_mempool_subscribe(priv->cdev);
 }
 
 static int

index c3b6495d9e..017a731b3f 100644
@@ -90,8 +90,7 @@ mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
 	switch (param->type) {
 	case MLX5_MP_REQ_CREATE_MR:
 		mp_init_msg(&priv->mp_id, &mp_res, param->type);
-		lkey = mlx5_mr_create_primary(cdev->pd,
-					      &priv->sh->cdev->mr_scache,
+		lkey = mlx5_mr_create_primary(cdev->pd, &cdev->mr_scache,
 					      &entry, param->args.addr,
 					      cdev->config.mr_ext_memseg_en);
 		if (lkey == UINT32_MAX)

141 lines not on this page · raw message

index 4f823baa6d..059d400384 100644
@@ -153,17 +153,6 @@ struct mlx5_flow_dump_ack {
 	int rc; /**< Return code. */
 };
 
-/** Key string for IPC. */
-#define MLX5_MP_NAME "net_mlx5_mp"
-
-/** Initialize a multi-process ID. */
-static inline void
-mlx5_mp_id_init(struct mlx5_mp_id *mp_id, uint16_t port_id)
-{
-	mp_id->port_id = port_id;
-	strlcpy(mp_id->name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN);
-}
-
 LIST_HEAD(mlx5_dev_list, mlx5_dev_ctx_shared);
 
 /* Shared data between primary and secondary processes. */

@@ -172,8 +161,6 @@ struct mlx5_shared_data {
 	/* Global spinlock for primary and secondary processes. */
 	int init_done; /* Whether primary has done initialization. */
 	unsigned int secondary_cnt; /* Number of secondary processes init'd. */
-	struct mlx5_dev_list mem_event_cb_list;
-	rte_rwlock_t mem_event_rwlock;
 };
 
 /* Per-process data structure, not visible to other processes. */

89 lines not on this page · raw message

index c83c7f4a39..8fa15e9820 100644
@@ -18,6 +18,7 @@
 
 #include <mlx5_prm.h>
 #include <mlx5_common.h>
+#include <mlx5_common_mr.h>
 
 #include "mlx5_autoconf.h"
 #include "mlx5_defs.h"

@@ -1027,20 +1028,6 @@ mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
 	mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type);
 }
 
-void
-mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
-{
-	struct mlx5_mprq_buf *buf = opaque;
-
-	if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) {
-		rte_mempool_put(buf->mp, buf);
-	} else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1,
-					       __ATOMIC_RELAXED) == 0)) {
-		__atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED);
-		rte_mempool_put(buf->mp, buf);
-	}
-}
-
 void
 mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf)
 {

index 42a12151fc..84a21fbfb9 100644
@@ -43,19 +43,6 @@ struct rxq_zip {
 	uint32_t cqe_cnt; /* Number of CQEs. */
 };
 
-/* Multi-Packet RQ buffer header. */
-struct mlx5_mprq_buf {
-	struct rte_mempool *mp;
-	uint16_t refcnt; /* Atomically accessed refcnt. */
-	uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */
-	struct rte_mbuf_ext_shared_info shinfos[];
-	/*
-	 * Shared information per stride.
-	 * More memory will be allocated for the first stride head-room and for
-	 * the strides data.
-	 */
-} __rte_cache_aligned;
-
 /* Get pointer to the first stride. */
 #define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \
 				sizeof(struct mlx5_mprq_buf) + \

@@ -255,7 +242,6 @@ int mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hxrq_idx,
 uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);
 void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);
 __rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec);
-void mlx5_mprq_buf_free_cb(void *addr, void *opaque);
 void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf);
 uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts,
 			    uint16_t pkts_n);

index 174899e661..e1a4ded688 100644
@@ -21,6 +21,7 @@
 
 #include <mlx5_glue.h>
 #include <mlx5_malloc.h>
+#include <mlx5_common_mr.h>
 
 #include "mlx5_defs.h"
 #include "mlx5.h"

index b400295e7d..876aa14ae6 100644
@@ -43,30 +43,4 @@ int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
 int mlx5_queue_state_modify(struct rte_eth_dev *dev,
 			    struct mlx5_mp_arg_queue_state_modify *sm);
 
-/* mlx5_mr.c */
-
-void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
-int mlx5_net_dma_map(struct rte_device *rte_dev, void *addr, uint64_t iova,
-		     size_t len);
-int mlx5_net_dma_unmap(struct rte_device *rte_dev, void *addr, uint64_t iova,
-		       size_t len);
-
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which the
- * cloned mbuf is allocated is returned instead.
- *
- * @param buf
- *   Pointer to mbuf.
- *
- * @return
- *   Memory pool where data is located for given mbuf.
- */
-static inline struct rte_mempool *
-mlx5_mb2mp(struct rte_mbuf *buf)
-{
-	if (unlikely(RTE_MBUF_CLONED(buf)))
-		return rte_mbuf_from_indirect(buf)->pool;
-	return buf->pool;
-}
-
 #endif /* RTE_PMD_MLX5_RXTX_H_ */

index 1f124b92e6..de2e284929 100644
@@ -235,10 +235,6 @@ void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 			   struct rte_eth_burst_mode *mode);
 
-/* mlx5_mr.c */
-
-uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf *mb);
-
 /* mlx5_tx_empw.c */
 
 MLX5_TXOFF_PRE_DECL(full_empw);

@@ -356,12 +352,12 @@ __mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
 #endif
 
 /**
- * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ * Query LKey from a packet buffer for Tx.
  *
  * @param txq
  *   Pointer to Tx queue structure.
- * @param addr
- *   Address to search.
+ * @param mb
+ *   Pointer to mbuf.
  *
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.

@@ -370,19 +366,12 @@ static __rte_always_inline uint32_t
 mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
 {
 	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
-	uintptr_t addr = (uintptr_t)mb->buf_addr;
-	uint32_t lkey;
-
-	/* Check generation bit to see if there's any change on existing MRs. */
-	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
-		mlx5_mr_flush_local_cache(mr_ctrl);
-	/* Linear search on MR cache array. */
-	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
-				   MLX5_MR_CACHE_N, addr);
-	if (likely(lkey != UINT32_MAX))
-		return lkey;
+	struct mlx5_txq_ctrl *txq_ctrl =
+			container_of(txq, struct mlx5_txq_ctrl, txq);
+	struct mlx5_priv *priv = txq_ctrl->priv;
+
 	/* Take slower bottom-half on miss. */
-	return mlx5_tx_mb2mr_bh(txq, mb);
+	return mlx5_mr_mb2mr(priv->sh->cdev, &priv->mp_id, mr_ctrl, mb);
 }
 
 /**

index 7f900b67ee..b8a513e1fa 100644
@@ -36,9 +36,11 @@ const struct rte_regexdev_ops mlx5_regexdev_ops = {
 };
 
 int
-mlx5_regex_start(struct rte_regexdev *dev __rte_unused)
+mlx5_regex_start(struct rte_regexdev *dev)
 {
-	return 0;
+	struct mlx5_regex_priv *priv = dev->data->dev_private;
+
+	return mlx5_dev_mempool_subscribe(priv->cdev);
 }
 
 int

index 149253d174..459414d5c2 100644
@@ -122,8 +122,21 @@ mlx5_init_shared_data(void)
 static int
 mlx5_init_once(void)
 {
+	struct mlx5_shared_data *sd;
+
 	if (mlx5_init_shared_data())
 		return -rte_errno;
+	sd = mlx5_shared_data;
+	rte_spinlock_lock(&sd->lock);
+	MLX5_ASSERT(sd);
+	if (!sd->init_done) {
+		LIST_INIT(&sd->mem_event_cb_list);
+		rte_rwlock_init(&sd->mem_event_rwlock);
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_mr_mem_event_cb, NULL);
+		sd->init_done = true;
+	}
+	rte_spinlock_unlock(&sd->lock);
 	return 0;
 }
 

index 6885bbb1d2..91de7b3e2c 100644
@@ -241,7 +241,7 @@ drivers_remove(struct mlx5_common_device *dev, uint32_t enabled_classes)
 	while (enabled_classes) {
 		driver = driver_get(RTE_BIT64(i));
 		if (driver != NULL) {
-			local_ret = driver->remove(dev->dev);
+			local_ret = driver->remove(dev);
 			if (local_ret == 0)
 				dev->classes_loaded &= ~RTE_BIT64(i);
 			else if (ret == 0)

@@ -275,7 +275,7 @@ drivers_probe(struct mlx5_common_device *dev, uint32_t user_classes)
 			ret = -EEXIST;
 			goto probe_err;
 		}
-		ret = driver->probe(dev->dev);
+		ret = driver->probe(dev);
 		if (ret < 0) {
 			DRV_LOG(ERR, "Failed to load driver %s",
 				driver->name);

index a772371200..b7e2ad1a82 100644
@@ -324,15 +324,21 @@ void mlx5_common_init(void);
  *   from devargs, locating target RDMA device and probing with it.
  */
 
+struct mlx5_common_device {
+	struct rte_device *dev;
+	TAILQ_ENTRY(mlx5_common_device) next;
+	uint32_t classes_loaded;
+};
+
 /**
  * Initialization function for the driver called during device probing.
  */
-typedef int (mlx5_class_driver_probe_t)(struct rte_device *dev);
+typedef int (mlx5_class_driver_probe_t)(struct mlx5_common_device *dev);
 
 /**
  * Uninitialization function for the driver called during hot-unplugging.
  */
-typedef int (mlx5_class_driver_remove_t)(struct rte_device *dev);
+typedef int (mlx5_class_driver_remove_t)(struct mlx5_common_device *dev);
 
 /**
  * Driver-specific DMA mapping. After a successful call the device

index a038330375..04c0af3763 100644
@@ -16,12 +16,6 @@ extern "C" {
 
 /* Common bus driver: */
 
-struct mlx5_common_device {
-	struct rte_device *dev;
-	TAILQ_ENTRY(mlx5_common_device) next;
-	uint32_t classes_loaded;
-};
-
 int mlx5_common_dev_probe(struct rte_device *eal_dev);
 int mlx5_common_dev_remove(struct rte_device *eal_dev);
 int mlx5_common_dev_dma_map(struct rte_device *dev, void *addr, uint64_t iova,

index 5c5aa87a18..75a07e5380 100644
@@ -36,7 +36,7 @@ struct mlx5_compress_xform {
 struct mlx5_compress_priv {
 	TAILQ_ENTRY(mlx5_compress_priv) next;
 	struct ibv_context *ctx; /* Device context. */
-	struct rte_compressdev *cdev;
+	struct rte_compressdev *compressdev;
 	void *uar;
 	uint32_t pdn; /* Protection Domain number. */
 	uint8_t min_block_size;

@@ -790,16 +790,16 @@ mlx5_compress_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 }
 
 static int
-mlx5_compress_dev_probe(struct rte_device *dev)
+mlx5_compress_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct ibv_device *ibv;
-	struct rte_compressdev *cdev;
+	struct rte_compressdev *compressdev;
 	struct ibv_context *ctx;
 	struct mlx5_compress_priv *priv;
 	struct mlx5_hca_attr att = { 0 };
 	struct rte_compressdev_pmd_init_params init_params = {
 		.name = "",
-		.socket_id = dev->numa_node,
+		.socket_id = cdev->dev->numa_node,
 	};
 
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {

@@ -807,7 +807,7 @@ mlx5_compress_dev_probe(struct rte_device *dev)
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	ibv = mlx5_os_get_ibv_dev(dev);
+	ibv = mlx5_os_get_ibv_dev(cdev->dev);
 	if (ibv == NULL)
 		return -rte_errno;
 	ctx = mlx5_glue->dv_open_device(ibv);

@@ -826,20 +826,20 @@ mlx5_compress_dev_probe(struct rte_device *dev)
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	cdev = rte_compressdev_pmd_create(ibv->name, dev,
-					  sizeof(*priv), &init_params);
-	if (cdev == NULL) {
+	compressdev = rte_compressdev_pmd_create(ibv->name, cdev->dev,
+						 sizeof(*priv), &init_params);
+	if (compressdev == NULL) {
 		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
 		claim_zero(mlx5_glue->close_device(ctx));
 		return -ENODEV;
 	}
 	DRV_LOG(INFO,
 		"Compress device %s was created successfully.", ibv->name);
-	cdev->dev_ops = &mlx5_compress_ops;
-	cdev->dequeue_burst = mlx5_compress_dequeue_burst;
-	cdev->enqueue_burst = mlx5_compress_enqueue_burst;
-	cdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
-	priv = cdev->data->dev_private;
+	compressdev->dev_ops = &mlx5_compress_ops;
+	compressdev->dequeue_burst = mlx5_compress_dequeue_burst;
+	compressdev->enqueue_burst = mlx5_compress_enqueue_burst;
+	compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
+	priv = compressdev->data->dev_private;
 	priv->mmo_decomp_sq = att.mmo_decompress_sq_en;
 	priv->mmo_decomp_qp = att.mmo_decompress_qp_en;
 	priv->mmo_comp_sq = att.mmo_compress_sq_en;

@@ -847,11 +847,11 @@ mlx5_compress_dev_probe(struct rte_device *dev)
 	priv->mmo_dma_sq = att.mmo_dma_sq_en;
 	priv->mmo_dma_qp = att.mmo_dma_qp_en;
 	priv->ctx = ctx;
-	priv->cdev = cdev;
+	priv->compressdev = compressdev;
 	priv->min_block_size = att.compress_min_block_size;
 	priv->qp_ts_format = att.qp_ts_format;
 	if (mlx5_compress_hw_global_prepare(priv) != 0) {
-		rte_compressdev_pmd_destroy(priv->cdev);
+		rte_compressdev_pmd_destroy(priv->compressdev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}

@@ -859,7 +859,7 @@ mlx5_compress_dev_probe(struct rte_device *dev)
 			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
 		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
 		mlx5_compress_hw_global_release(priv);
-		rte_compressdev_pmd_destroy(priv->cdev);
+		rte_compressdev_pmd_destroy(priv->compressdev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		rte_errno = ENOMEM;
 		return -rte_errno;

@@ -878,13 +878,13 @@ mlx5_compress_dev_probe(struct rte_device *dev)
 }
 
 static int
-mlx5_compress_dev_remove(struct rte_device *dev)
+mlx5_compress_dev_remove(struct mlx5_common_device *cdev)
 {
 	struct mlx5_compress_priv *priv = NULL;
 
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_FOREACH(priv, &mlx5_compress_priv_list, next)
-		if (priv->cdev->device == dev)
+		if (priv->compressdev->device == cdev->dev)
 			break;
 	if (priv)
 		TAILQ_REMOVE(&mlx5_compress_priv_list, priv, next);

@@ -895,7 +895,7 @@ mlx5_compress_dev_remove(struct rte_device *dev)
 							  NULL);
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_compress_hw_global_release(priv);
-		rte_compressdev_pmd_destroy(priv->cdev);
+		rte_compressdev_pmd_destroy(priv->compressdev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;

index 6a2f8b6ac6..d9762a964d 100644
@@ -951,7 +951,7 @@ mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 }
 
 static int
-mlx5_crypto_dev_probe(struct rte_device *dev)
+mlx5_crypto_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;

@@ -963,7 +963,7 @@ mlx5_crypto_dev_probe(struct rte_device *dev)
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
 		.private_data_size = sizeof(struct mlx5_crypto_priv),
-		.socket_id = dev->numa_node,
+		.socket_id = cdev->dev->numa_node,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};

@@ -975,7 +975,7 @@ mlx5_crypto_dev_probe(struct rte_device *dev)
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	ibv = mlx5_os_get_ibv_dev(dev);
+	ibv = mlx5_os_get_ibv_dev(cdev->dev);
 	if (ibv == NULL)
 		return -rte_errno;
 	ctx = mlx5_glue->dv_open_device(ibv);

@@ -992,7 +992,7 @@ mlx5_crypto_dev_probe(struct rte_device *dev)
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	ret = mlx5_crypto_parse_devargs(dev->devargs, &devarg_prms);
+	ret = mlx5_crypto_parse_devargs(cdev->dev->devargs, &devarg_prms);
 	if (ret) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		claim_zero(mlx5_glue->close_device(ctx));

@@ -1005,8 +1005,8 @@ mlx5_crypto_dev_probe(struct rte_device *dev)
 		claim_zero(mlx5_glue->close_device(ctx));
 		return -rte_errno;
 	}
-	crypto_dev = rte_cryptodev_pmd_create(ibv->name, dev,
-					&init_params);
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, cdev->dev,
+					      &init_params);
 	if (crypto_dev == NULL) {
 		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
 		claim_zero(mlx5_glue->close_device(ctx));

@@ -1065,13 +1065,13 @@ mlx5_crypto_dev_probe(struct rte_device *dev)
 }
 
 static int
-mlx5_crypto_dev_remove(struct rte_device *dev)
+mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
 {
 	struct mlx5_crypto_priv *priv = NULL;
 
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
-		if (priv->crypto_dev->device == dev)
+		if (priv->crypto_dev->device == cdev->dev)
 			break;
 	if (priv)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);

154 lines not on this page · raw message

index 1e1b8b736b..e39e77aa9d 100644
@@ -1279,7 +1279,8 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 		rte_errno  = ENOMEM;
 		goto exit;
 	}
-	sh->numa_node = spawn->numa_node;
+	sh->numa_node = spawn->cdev->dev->numa_node;
+	sh->cdev = spawn->cdev;
 	if (spawn->bond_info)
 		sh->bond = *spawn->bond_info;
 	err = mlx5_os_open_device(spawn, config, sh);

@@ -2537,19 +2538,19 @@ mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev)
  *
  * This function removes all Ethernet devices belong to a given device.
  *
- * @param[in] dev
+ * @param[in] cdev
  *   Pointer to the generic device.
  *
  * @return
  *   0 on success, the function cannot fail.
  */
 int
-mlx5_net_remove(struct rte_device *dev)
+mlx5_net_remove(struct mlx5_common_device *cdev)
 {
 	uint16_t port_id;
 	int ret = 0;
 
-	RTE_ETH_FOREACH_DEV_OF(port_id, dev) {
+	RTE_ETH_FOREACH_DEV_OF(port_id, cdev->dev) {
 		/*
 		 * mlx5_dev_close() is not registered to secondary process,
 		 * call the close function explicitly for secondary process.

index fe533fcc81..f441352a63 100644
@@ -134,11 +134,11 @@ struct mlx5_dev_spawn_data {
 	uint32_t max_port; /**< Device maximal port index. */
 	uint32_t phys_port; /**< Device physical port index. */
 	int pf_bond; /**< bonding device PF index. < 0 - no bonding */
-	int numa_node; /**< Device numa node. */
 	struct mlx5_switch_info info; /**< Switch information. */
 	void *phys_dev; /**< Associated physical device. */
 	struct rte_eth_dev *eth_dev; /**< Associated Ethernet device. */
 	struct rte_pci_device *pci_dev; /**< Backend PCI device. */
+	struct mlx5_common_device *cdev; /**< Backend common device. */
 	struct mlx5_bond_info *bond_info;
 };
 

@@ -1148,6 +1148,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
+	struct mlx5_common_device *cdev; /* Backend mlx5 device. */
 	void *ctx; /* Verbs/DV/DevX context. */
 	void *pd; /* Protection Domain. */
 	uint32_t pdn; /* Protection Domain number. */

@@ -1492,7 +1493,7 @@ int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
 			      struct rte_eth_udp_tunnel *udp_tunnel);
 uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_device *odev);
 int mlx5_dev_close(struct rte_eth_dev *dev);
-int mlx5_net_remove(struct rte_device *dev);
+int mlx5_net_remove(struct mlx5_common_device *cdev);
 bool mlx5_is_hpf(struct rte_eth_dev *dev);
 bool mlx5_is_sf_repr(struct rte_eth_dev *dev);
 void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh);

@@ -1782,7 +1783,7 @@ int mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
 			 const struct mlx5_dev_config *config,
 			 struct mlx5_dev_ctx_shared *sh);
 int mlx5_os_get_pdn(void *pd, uint32_t *pdn);
-int mlx5_os_net_probe(struct rte_device *dev);
+int mlx5_os_net_probe(struct mlx5_common_device *cdev);
 void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh);
 void mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh);
 void mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,

index 459414d5c2..8ffbb9ff54 100644
@@ -337,7 +337,7 @@ mlx5_flow_counter_mode_config(struct rte_eth_dev *dev __rte_unused)
 }
 
 /**
- * Spawn an Ethernet device from Verbs information.
+ * Spawn an Ethernet device from DevX information.
  *
  * @param dpdk_dev
  *   Backing DPDK device.

@@ -982,15 +982,15 @@ mlx5_match_devx_devices_to_addr(struct devx_device_bdf *devx_bdf,
  * This function spawns Ethernet devices out of a given device.
  *
  * @param[in] dev
- *   Pointer to the generic device.
+ *   Pointer to the common device.
  *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_os_net_probe(struct rte_device *dev)
+mlx5_os_net_probe(struct mlx5_common_device *cdev)
 {
-	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev);
+	struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(cdev->dev);
 	struct devx_device_bdf *devx_bdf_devs, *orig_devx_bdf_devs;
 	/*
 	 * Number of found IB Devices matching with requested PCI BDF.

@@ -1092,6 +1092,7 @@ mlx5_os_net_probe(struct rte_device *dev)
 	list[ns].phys_dev = devx_bdf_match[ns];
 	list[ns].eth_dev = NULL;
 	list[ns].pci_dev = pci_dev;
+	list[ns].cdev = cdev;
 	list[ns].pf_bond = bd;
 	list[ns].ifindex = -1; /* Spawn will assign */
 	list[ns].info =

@@ -1136,10 +1137,7 @@ mlx5_os_net_probe(struct rte_device *dev)
 	dev_config.dv_flow_en = 1;
 	dev_config.decap_en = 0;
 	dev_config.log_hp_size = MLX5_ARG_UNSET;
-	list[ns].numa_node = pci_dev->device.numa_node;
-	list[ns].eth_dev = mlx5_dev_spawn(&pci_dev->device,
-					  &list[ns],
-					  &dev_config);
+	list[ns].eth_dev = mlx5_dev_spawn(cdev->dev, &list[ns], &dev_config);
 	if (!list[ns].eth_dev)
 		goto exit;
 	restore = list[ns].eth_dev->data->dev_flags;

index 5aa988be6d..399c24a20c 100644
@@ -121,7 +121,7 @@ mlx5_regex_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
 }
 
 static int
-mlx5_regex_dev_probe(struct rte_device *rte_dev)
+mlx5_regex_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct ibv_device *ibv;
 	struct mlx5_regex_priv *priv = NULL;

@@ -131,7 +131,7 @@ mlx5_regex_dev_probe(struct rte_device *rte_dev)
 	int ret;
 	uint32_t val;
 
-	ibv = mlx5_os_get_ibv_dev(rte_dev);
+	ibv = mlx5_os_get_ibv_dev(cdev->dev);
 	if (ibv == NULL)
 		return -rte_errno;
 	DRV_LOG(INFO, "Probe device \"%s\".", ibv->name);

@@ -180,7 +180,7 @@ mlx5_regex_dev_probe(struct rte_device *rte_dev)
 		priv->is_bf2 = 1;
 	/* Default RXP programming mode to Shared. */
 	priv->prog_mode = MLX5_RXP_SHARED_PROG_MODE;
-	mlx5_regex_get_name(name, rte_dev);
+	mlx5_regex_get_name(name, cdev->dev);
 	priv->regexdev = rte_regexdev_register(name);
 	if (priv->regexdev == NULL) {
 		DRV_LOG(ERR, "Failed to register RegEx device.");

@@ -214,7 +214,7 @@ mlx5_regex_dev_probe(struct rte_device *rte_dev)
 		priv->regexdev->enqueue = mlx5_regexdev_enqueue_gga;
 #endif
 	priv->regexdev->dequeue = mlx5_regexdev_dequeue;
-	priv->regexdev->device = rte_dev;
+	priv->regexdev->device = cdev->dev;
 	priv->regexdev->data->dev_private = priv;
 	priv->regexdev->state = RTE_REGEXDEV_READY;
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;

@@ -256,13 +256,13 @@ mlx5_regex_dev_probe(struct rte_device *rte_dev)
 }
 
 static int
-mlx5_regex_dev_remove(struct rte_device *rte_dev)
+mlx5_regex_dev_remove(struct mlx5_common_device *cdev)
 {
 	char name[RTE_REGEXDEV_NAME_MAX_LEN];
 	struct rte_regexdev *dev;
 	struct mlx5_regex_priv *priv = NULL;
 
-	mlx5_regex_get_name(name, rte_dev);
+	mlx5_regex_get_name(name, cdev->dev);
 	dev = rte_regexdev_get_device_by_name(name);
 	if (!dev)
 		return 0;

index 6d17d7a6f3..d7ef303cfe 100644
@@ -630,7 +630,7 @@ mlx5_vdpa_config_get(struct rte_devargs *devargs, struct mlx5_vdpa_priv *priv)
 }
 
 static int
-mlx5_vdpa_dev_probe(struct rte_device *dev)
+mlx5_vdpa_dev_probe(struct mlx5_common_device *cdev)
 {
 	struct ibv_device *ibv;
 	struct mlx5_vdpa_priv *priv = NULL;

@@ -639,14 +639,14 @@ mlx5_vdpa_dev_probe(struct rte_device *dev)
 	int retry;
 	int ret;
 
-	if (mlx5_vdpa_roce_disable(dev) != 0) {
+	if (mlx5_vdpa_roce_disable(cdev->dev) != 0) {
 		DRV_LOG(WARNING, "Failed to disable ROCE for \"%s\".",
-			dev->name);
+			cdev->dev->name);
 		return -rte_errno;
 	}
 	/* Wait for the IB device to appear again after reload. */
 	for (retry = MLX5_VDPA_MAX_RETRIES; retry > 0; --retry) {
-		ibv = mlx5_os_get_ibv_dev(dev);
+		ibv = mlx5_os_get_ibv_dev(cdev->dev);
 		if (ibv != NULL)
 			break;
 		usleep(MLX5_VDPA_USEC);

@@ -654,7 +654,7 @@ mlx5_vdpa_dev_probe(struct rte_device *dev)
 	if (ibv == NULL) {
 		DRV_LOG(ERR, "Cannot get IB device after disabling RoCE for "
 				"\"%s\", retries exceed %d.",
-				dev->name, MLX5_VDPA_MAX_RETRIES);
+				cdev->dev->name, MLX5_VDPA_MAX_RETRIES);
 		rte_errno = EAGAIN;
 		return -rte_errno;
 	}

@@ -698,13 +698,13 @@ mlx5_vdpa_dev_probe(struct rte_device *dev)
 		DRV_LOG(ERR, "Failed to allocate VAR %u.", errno);
 		goto error;
 	}
-	priv->vdev = rte_vdpa_register_device(dev, &mlx5_vdpa_ops);
+	priv->vdev = rte_vdpa_register_device(cdev->dev, &mlx5_vdpa_ops);
 	if (priv->vdev == NULL) {
 		DRV_LOG(ERR, "Failed to register vDPA device.");
 		rte_errno = rte_errno ? rte_errno : EINVAL;
 		goto error;
 	}
-	mlx5_vdpa_config_get(dev->devargs, priv);
+	mlx5_vdpa_config_get(cdev->dev->devargs, priv);
 	SLIST_INIT(&priv->mr_list);
 	pthread_mutex_init(&priv->vq_config_lock, NULL);
 	pthread_mutex_lock(&priv_list_lock);

@@ -724,14 +724,14 @@ mlx5_vdpa_dev_probe(struct rte_device *dev)
 }
 
 static int
-mlx5_vdpa_dev_remove(struct rte_device *dev)
+mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
 {
 	struct mlx5_vdpa_priv *priv = NULL;
 	int found = 0;
 
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_FOREACH(priv, &priv_list, next) {
-		if (priv->vdev->device == dev) {
+		if (priv->vdev->device == cdev->dev) {
 			found = 1;
 			break;
 		}