mlx5: fix performance degradation

index 744c6a72b3..e16f3aa713 100644
@@ -417,13 +417,6 @@ void
 mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
 			    struct rte_mempool *mp);
 
-/* mlx5_common_mr.c */
-
-__rte_internal
-uint32_t
-mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
-	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf);
-
 /* mlx5_common_os.c */
 
 int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);

60 lines not on this page · raw message

79 lines not on this page · raw message

index 903ed0652c..003d358f96 100644
@@ -292,8 +292,8 @@ mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)
  *
  * @param mr_ctrl
  *   Pointer to MR control structure.
- * @param dev_gen_ptr
- *   Pointer to generation number of global cache.
+ * @param cdev
+ *   Pointer to the mlx5 device structure.
  * @param socket
  *   NUMA socket on which memory must be allocated.
  *

@@ -301,15 +301,16 @@ mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused)
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
+mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, struct mlx5_common_device *cdev,
 		  int socket)
 {
 	if (mr_ctrl == NULL) {
 		rte_errno = EINVAL;
 		return -rte_errno;
 	}
+	mr_ctrl->cdev = cdev;
 	/* Save pointer of global generation number to check memory event. */
-	mr_ctrl->dev_gen_ptr = dev_gen_ptr;
+	mr_ctrl->dev_gen_ptr = &cdev->mr_scache.dev_gen;
 	/* Initialize B-tree and allocate memory for bottom-half cache table. */
 	return mlx5_mr_btree_init(&mr_ctrl->cache_bh, MLX5_MR_BTREE_CACHE_N,
 				  socket);

@@ -1860,11 +1861,12 @@ mlx5_mr_mempool2mr_bh(struct mlx5_mr_share_cache *share_cache,
 }
 
 uint32_t
-mlx5_mr_mb2mr_bh(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
-		 struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mb)
+mlx5_mr_mb2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mb,
+		 struct mlx5_mp_id *mp_id)
 {
 	uint32_t lkey;
 	uintptr_t addr = (uintptr_t)mb->buf_addr;
+	struct mlx5_common_device *cdev = mr_ctrl->cdev;
 
 	if (cdev->config.mr_mempool_reg_en) {
 		struct rte_mempool *mp = NULL;

index 8771c7d02b..f65974b8a9 100644
@@ -66,6 +66,7 @@ struct mlx5_common_device;
 
 /* Per-queue MR control descriptor. */
 struct mlx5_mr_ctrl {
+	struct mlx5_common_device *cdev; /* Pointer to the mlx5 common device.*/
 	uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
 	uint32_t cur_gen; /* Generation number saved to flush caches. */
 	uint16_t mru; /* Index of last hit entry in top-half cache. */

@@ -169,41 +170,36 @@ void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
  * Bottom-half of LKey search on. If supported, lookup for the address from
  * the mempool. Otherwise, search in old mechanism caches.
  *
- * @param cdev
- *   Pointer to mlx5 device.
- * @param mp_id
- *   Multi-process identifier, may be NULL for the primary process.
  * @param mr_ctrl
  *   Pointer to per-queue MR control structure.
  * @param mb
  *   Pointer to mbuf.
+ * @param mp_id
+ *   Multi-process identifier, may be NULL for the primary process.
  *
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 __rte_internal
-uint32_t mlx5_mr_mb2mr_bh(struct mlx5_common_device *cdev,
-			  struct mlx5_mp_id *mp_id,
-			  struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mb);
+uint32_t mlx5_mr_mb2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
+			  struct mlx5_mp_id *mp_id);
 
 /**
  * Query LKey from a packet buffer.
  *
- * @param cdev
- *   Pointer to the mlx5 device structure.
- * @param mp_id
- *   Multi-process identifier, may be NULL for the primary process.
  * @param mr_ctrl
  *   Pointer to per-queue MR control structure.
  * @param mbuf
  *   Pointer to mbuf.
+ * @param mp_id
+ *   Multi-process identifier, may be NULL for the primary process.
  *
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 static __rte_always_inline uint32_t
-mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
-	      struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf)
+mlx5_mr_mb2mr(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
+	      struct mlx5_mp_id *mp_id)
 {
 	uint32_t lkey;
 

@@ -216,14 +212,14 @@ mlx5_mr_mb2mr(struct mlx5_common_device *cdev, struct mlx5_mp_id *mp_id,
 	if (likely(lkey != UINT32_MAX))
 		return lkey;
 	/* Take slower bottom-half on miss. */
-	return mlx5_mr_mb2mr_bh(cdev, mp_id, mr_ctrl, mbuf);
+	return mlx5_mr_mb2mr_bh(mr_ctrl, mbuf, mp_id);
 }
 
 /* mlx5_common_mr.c */
 
 __rte_internal
-int mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl, uint32_t *dev_gen_ptr,
-		      int socket);
+int mlx5_mr_ctrl_init(struct mlx5_mr_ctrl *mr_ctrl,
+		      struct mlx5_common_device *cdev, int socket);
 __rte_internal
 void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
 void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused);

index c4081c5f7d..5cf6d647af 100644
@@ -205,7 +205,7 @@ mlx5_compress_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 		return -rte_errno;
 	}
 	dev->data->queue_pairs[qp_id] = qp;
-	if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
+	if (mlx5_mr_ctrl_init(&qp->mr_ctrl, priv->cdev,
 			      priv->dev_config.socket_id)) {
 		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 			(uint32_t)qp_id);

@@ -471,7 +471,7 @@ mlx5_compress_dseg_set(struct mlx5_compress_qp *qp,
 	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 
 	dseg->bcount = rte_cpu_to_be_32(len);
-	dseg->lkey = mlx5_mr_mb2mr(qp->priv->cdev, 0, &qp->mr_ctrl, mbuf);
+	dseg->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf, 0);
 	dseg->pbuf = rte_cpu_to_be_64(addr);
 	return dseg->lkey;
 }

index f430d8cde0..1740dba003 100644
@@ -305,9 +305,9 @@ mlx5_crypto_get_block_size(struct rte_crypto_op *op)
 }
 
 static __rte_always_inline uint32_t
-mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
-		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
-		      uint32_t offset, uint32_t *remain)
+mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf,
+		    struct mlx5_wqe_dseg *klm, uint32_t offset,
+		    uint32_t *remain)
 {
 	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
 	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);

@@ -317,22 +317,21 @@ mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 	*remain -= data_len;
 	klm->bcount = rte_cpu_to_be_32(data_len);
 	klm->pbuf = rte_cpu_to_be_64(addr);
-	klm->lkey = mlx5_mr_mb2mr(priv->cdev, 0, &qp->mr_ctrl, mbuf);
+	klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf, 0);
 	return klm->lkey;
 
 }
 
 static __rte_always_inline uint32_t
-mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
-		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
-		     struct mlx5_wqe_dseg *klm)
+mlx5_crypto_klms_set(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op,
+		     struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm)
 {
 	uint32_t remain_len = op->sym->cipher.data.length;
 	uint32_t nb_segs = mbuf->nb_segs;
 	uint32_t klm_n = 1u;
 
 	/* First mbuf needs to take the cipher offset. */
-	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+	if (unlikely(mlx5_crypto_klm_set(qp, mbuf, klm,
 		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
 		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 		return 0;

@@ -344,7 +343,7 @@ mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
 			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 			return 0;
 		}
-		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
+		if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0,
 						 &remain_len) == UINT32_MAX)) {
 			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 			return 0;

@@ -370,7 +369,7 @@ mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 	uint32_t ds;
 	bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
 	/* Set UMR WQE. */
-	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+	uint32_t klm_n = mlx5_crypto_klms_set(qp, op,
 				   ipl ? op->sym->m_src : op->sym->m_dst, klms);
 
 	if (unlikely(klm_n == 0))

@@ -396,8 +395,7 @@ mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
 	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
 	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
 	if (!ipl) {
-		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
-					     klms);
+		klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms);
 		if (unlikely(klm_n == 0))
 			return 0;
 	} else {

@@ -643,7 +641,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to create QP.");
 		goto error;
 	}
-	if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
+	if (mlx5_mr_ctrl_init(&qp->mr_ctrl, priv->cdev,
 			      priv->dev_config.socket_id) != 0) {
 		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
 			(uint32_t)qp_id);

index 4952fe1455..322f234628 100644
@@ -282,7 +282,6 @@ static __rte_always_inline uint32_t
 mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
 {
 	struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
-	struct mlx5_rxq_ctrl *rxq_ctrl;
 	struct rte_mempool *mp;
 	uint32_t lkey;
 

@@ -291,14 +290,9 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
 				   MLX5_MR_CACHE_N, addr);
 	if (likely(lkey != UINT32_MAX))
 		return lkey;
-	/*
-	 * Slower search in the mempool database on miss.
-	 * During queue creation rxq->sh is not yet set, so we use rxq_ctrl.
-	 */
-	rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
 	mp = mlx5_rxq_mprq_enabled(rxq) ? rxq->mprq_mp : rxq->mp;
-	return mlx5_mr_mempool2mr_bh(&rxq_ctrl->priv->sh->cdev->mr_scache,
-				     mr_ctrl, mp, addr);
+	return mlx5_mr_mempool2mr_bh(&mr_ctrl->cdev->mr_scache, mr_ctrl,
+				     mp, addr);
 }
 
 #define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))

index 4f02fe02b9..1fc2f0e0c1 100644
@@ -1455,8 +1455,7 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		goto error;
 	}
 	tmpl->type = MLX5_RXQ_TYPE_STANDARD;
-	if (mlx5_mr_ctrl_init(&tmpl->rxq.mr_ctrl,
-			      &priv->sh->cdev->mr_scache.dev_gen, socket)) {
+	if (mlx5_mr_ctrl_init(&tmpl->rxq.mr_ctrl, priv->sh->cdev, socket)) {
 		/* rte_errno is already set. */
 		goto error;
 	}

index ea20213a40..7fed0e7cb9 100644
@@ -368,10 +368,9 @@ mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
 	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
 	struct mlx5_txq_ctrl *txq_ctrl =
 			container_of(txq, struct mlx5_txq_ctrl, txq);
-	struct mlx5_priv *priv = txq_ctrl->priv;
 
 	/* Take slower bottom-half on miss. */
-	return mlx5_mr_mb2mr(priv->sh->cdev, &priv->mp_id, mr_ctrl, mb);
+	return mlx5_mr_mb2mr(mr_ctrl, mb, &txq_ctrl->priv->mp_id);
 }
 
 /**

index e2a38d980a..e9ab7fa266 100644
@@ -1134,8 +1134,7 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		rte_errno = ENOMEM;
 		return NULL;
 	}
-	if (mlx5_mr_ctrl_init(&tmpl->txq.mr_ctrl,
-			      &priv->sh->cdev->mr_scache.dev_gen, socket)) {
+	if (mlx5_mr_ctrl_init(&tmpl->txq.mr_ctrl, priv->sh->cdev, socket)) {
 		/* rte_errno is already set. */
 		goto error;
 	}

index 50c966a022..e40b1f20d9 100644
@@ -242,8 +242,7 @@ mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind,
 		nb_sq_config++;
 	}
 
-	ret = mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
-				rte_socket_id());
+	ret = mlx5_mr_ctrl_init(&qp->mr_ctrl, priv->cdev, rte_socket_id());
 	if (ret) {
 		DRV_LOG(ERR, "Error setting up mr btree");
 		goto err_btree;

index adb5343a46..943cb9c19e 100644
@@ -109,26 +109,6 @@ set_wqe_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi, uint8_t opcode,
 	seg->imm = imm;
 }
 
-/**
- * Query LKey from a packet buffer for QP. If not found, add the mempool.
- *
- * @param priv
- *   Pointer to the priv object.
- * @param mr_ctrl
- *   Pointer to per-queue MR control structure.
- * @param mbuf
- *   Pointer to source mbuf, to search in.
- *
- * @return
- *   Searched LKey on success, UINT32_MAX on no match.
- */
-static inline uint32_t
-mlx5_regex_mb2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
-		 struct rte_mbuf *mbuf)
-{
-	return mlx5_mr_mb2mr(priv->cdev, 0, mr_ctrl, mbuf);
-}
-
 static inline void
 __prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp_obj,
 	   struct rte_regex_ops *op, struct mlx5_regex_job *job,

@@ -180,7 +160,7 @@ prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 	struct mlx5_klm klm;
 
 	klm.byte_count = rte_pktmbuf_data_len(op->mbuf);
-	klm.mkey = mlx5_regex_mb2mr(priv, &qp->mr_ctrl, op->mbuf);
+	klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, op->mbuf, 0);
 	klm.address = rte_pktmbuf_mtod(op->mbuf, uintptr_t);
 	__prep_one(priv, qp_obj, op, job, qp_obj->pi, &klm);
 	qp_obj->db_pi = qp_obj->pi;

@@ -349,9 +329,8 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 			while (mbuf) {
 				addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
 				/* Build indirect mkey seg's KLM. */
-				mkey_klm->mkey = mlx5_regex_mb2mr(priv,
-								  &qp->mr_ctrl,
-								  mbuf);
+				mkey_klm->mkey = mlx5_mr_mb2mr(&qp->mr_ctrl,
+							       mbuf, 0);
 				mkey_klm->address = rte_cpu_to_be_64(addr);
 				mkey_klm->byte_count = rte_cpu_to_be_32
 						(rte_pktmbuf_data_len(mbuf));

@@ -368,7 +347,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 			klm.byte_count = scatter_size;
 		} else {
 			/* The single mubf case. Build the KLM directly. */
-			klm.mkey = mlx5_regex_mb2mr(priv, &qp->mr_ctrl, mbuf);
+			klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf, 0);
 			klm.address = rte_pktmbuf_mtod(mbuf, uintptr_t);
 			klm.byte_count = rte_pktmbuf_data_len(mbuf);
 		}

index e6ff045c95..1c36212a04 100644
@@ -318,10 +318,7 @@ static int
 mlx5_dev_mempool_register(struct mlx5_common_device *cdev,
 			  struct rte_mempool *mp)
 {
-	struct mlx5_mp_id mp_id;
-
-	mlx5_mp_id_init(&mp_id, 0);
-	return mlx5_mr_mempool_register(&cdev->mr_scache, cdev->pd, mp, &mp_id);
+	return mlx5_mr_mempool_register(cdev, mp);
 }
 
 /**

@@ -336,10 +333,7 @@ void
 mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
 			    struct rte_mempool *mp)
 {
-	struct mlx5_mp_id mp_id;
-
-	mlx5_mp_id_init(&mp_id, 0);
-	if (mlx5_mr_mempool_unregister(&cdev->mr_scache, mp, &mp_id) < 0)
+	if (mlx5_mr_mempool_unregister(cdev, mp) < 0)
 		DRV_LOG(WARNING, "Failed to unregister mempool %s for PD %p: %s",
 			mp->name, cdev->pd, rte_strerror(rte_errno));
 }

index 6dfc5535e0..536d61f66c 100644
@@ -16,8 +16,8 @@
 /**
  * Request Memory Region creation to the primary process.
  *
- * @param[in] mp_id
- *   ID of the MP process.
+ * @param cdev
+ *   Pointer to the mlx5 common device.
  * @param addr
  *   Target virtual address to register.
  *

@@ -25,23 +25,24 @@
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
 int
-mlx5_mp_req_mr_create(struct mlx5_mp_id *mp_id, uintptr_t addr)
+mlx5_mp_req_mr_create(struct mlx5_common_device *cdev, uintptr_t addr)
 {
 	struct rte_mp_msg mp_req;
 	struct rte_mp_msg *mp_res;
 	struct rte_mp_reply mp_rep;
 	struct mlx5_mp_param *req = (struct mlx5_mp_param *)mp_req.param;
+	struct mlx5_mp_arg_mr_manage *arg = &req->args.mr_manage;
 	struct mlx5_mp_param *res;
 	struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0};
 	int ret;
 
 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
-	mp_init_msg(mp_id, &mp_req, MLX5_MP_REQ_CREATE_MR);
-	req->args.addr = addr;
+	mp_init_port_agnostic_msg(&mp_req, MLX5_MP_REQ_CREATE_MR);
+	arg->addr = addr;
+	arg->cdev = cdev;
 	ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts);
 	if (ret) {
-		DRV_LOG(ERR, "port %u request to primary process failed",
-			mp_id->port_id);
+		DRV_LOG(ERR, "Create MR request to primary process failed.");
 		return -rte_errno;
 	}
 	MLX5_ASSERT(mp_rep.nb_received == 1);

@@ -55,27 +56,22 @@ mlx5_mp_req_mr_create(struct mlx5_mp_id *mp_id, uintptr_t addr)
 }
 
 /**
- * @param mp_id
- *   ID of the MP process.
- * @param share_cache
- *   Shared MR cache.
- * @param pd
- *   Protection domain.
+ * @param cdev
+ *   Pointer to the mlx5 common device.
  * @param mempool
  *   Mempool to register or unregister.
  * @param reg
  *   True to register the mempool, False to unregister.
  */
 int
-mlx5_mp_req_mempool_reg(struct mlx5_mp_id *mp_id,
-			struct mlx5_mr_share_cache *share_cache, void *pd,
+mlx5_mp_req_mempool_reg(struct mlx5_common_device *cdev,
 			struct rte_mempool *mempool, bool reg)
 {
 	struct rte_mp_msg mp_req;
 	struct rte_mp_msg *mp_res;
 	struct rte_mp_reply mp_rep;
 	struct mlx5_mp_param *req = (struct mlx5_mp_param *)mp_req.param;
-	struct mlx5_mp_arg_mempool_reg *arg = &req->args.mempool_reg;
+	struct mlx5_mp_arg_mr_manage *arg = &req->args.mr_manage;
 	struct mlx5_mp_param *res;
 	struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0};
 	enum mlx5_mp_req_type type;

@@ -84,14 +80,14 @@ mlx5_mp_req_mempool_reg(struct mlx5_mp_id *mp_id,
 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
 	type = reg ? MLX5_MP_REQ_MEMPOOL_REGISTER :
 		     MLX5_MP_REQ_MEMPOOL_UNREGISTER;
-	mp_init_msg(mp_id, &mp_req, type);
-	arg->share_cache = share_cache;
-	arg->pd = pd;
+	mp_init_port_agnostic_msg(&mp_req, type);
 	arg->mempool = mempool;
+	arg->cdev = cdev;
 	ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts);
 	if (ret) {
-		DRV_LOG(ERR, "port %u request to primary process failed",
-			mp_id->port_id);
+		DRV_LOG(ERR,
+			"Mempool %sregister request to primary process failed.",
+			reg ? "" : "un");
 		return -rte_errno;
 	}
 	MLX5_ASSERT(mp_rep.nb_received == 1);

71 lines not on this page · raw message

313 lines not on this page · raw message

index f65974b8a9..93903d8397 100644
@@ -174,15 +174,12 @@ void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
  *   Pointer to per-queue MR control structure.
  * @param mb
  *   Pointer to mbuf.
- * @param mp_id
- *   Multi-process identifier, may be NULL for the primary process.
  *
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 __rte_internal
-uint32_t mlx5_mr_mb2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
-			  struct mlx5_mp_id *mp_id);
+uint32_t mlx5_mr_mb2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf);
 
 /**
  * Query LKey from a packet buffer.

@@ -191,15 +188,12 @@ uint32_t mlx5_mr_mb2mr_bh(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
  *   Pointer to per-queue MR control structure.
  * @param mbuf
  *   Pointer to mbuf.
- * @param mp_id
- *   Multi-process identifier, may be NULL for the primary process.
  *
  * @return
  *   Searched LKey on success, UINT32_MAX on no match.
  */
 static __rte_always_inline uint32_t
-mlx5_mr_mb2mr(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
-	      struct mlx5_mp_id *mp_id)
+mlx5_mr_mb2mr(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf)
 {
 	uint32_t lkey;
 

@@ -212,7 +206,7 @@ mlx5_mr_mb2mr(struct mlx5_mr_ctrl *mr_ctrl, struct rte_mbuf *mbuf,
 	if (likely(lkey != UINT32_MAX))
 		return lkey;
 	/* Take slower bottom-half on miss. */
-	return mlx5_mr_mb2mr_bh(mr_ctrl, mbuf, mp_id);
+	return mlx5_mr_mb2mr_bh(mr_ctrl, mbuf);
 }
 
 /* mlx5_common_mr.c */

@@ -244,10 +238,9 @@ mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
 void mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
 __rte_internal
 uint32_t
-mlx5_mr_create_primary(void *pd,
-		       struct mlx5_mr_share_cache *share_cache,
-		       struct mr_cache_entry *entry, uintptr_t addr,
-		       unsigned int mr_ext_memseg_en);
+mlx5_mr_create(struct mlx5_common_device *cdev,
+	       struct mlx5_mr_share_cache *share_cache,
+	       struct mr_cache_entry *entry, uintptr_t addr);
 
 /* mlx5_common_verbs.c */
 

@@ -264,11 +257,11 @@ mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, mlx5_dereg_mr_t *dereg_mr_cb);
 
 __rte_internal
 int
-mlx5_mr_mempool_register(struct mlx5_mr_share_cache *share_cache, void *pd,
-			 struct rte_mempool *mp, struct mlx5_mp_id *mp_id);
+mlx5_mr_mempool_register(struct mlx5_common_device *cdev,
+			 struct rte_mempool *mp);
 __rte_internal
 int
-mlx5_mr_mempool_unregister(struct mlx5_mr_share_cache *share_cache,
-			   struct rte_mempool *mp, struct mlx5_mp_id *mp_id);
+mlx5_mr_mempool_unregister(struct mlx5_common_device *cdev,
+			   struct rte_mempool *mp);
 
 #endif /* RTE_PMD_MLX5_COMMON_MR_H_ */

index 5cf6d647af..0e87d3b819 100644
@@ -471,7 +471,7 @@ mlx5_compress_dseg_set(struct mlx5_compress_qp *qp,
 	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
 
 	dseg->bcount = rte_cpu_to_be_32(len);
-	dseg->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf, 0);
+	dseg->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
 	dseg->pbuf = rte_cpu_to_be_64(addr);
 	return dseg->lkey;
 }

index 1740dba003..6b0ea87438 100644
@@ -317,7 +317,7 @@ mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf,
 	*remain -= data_len;
 	klm->bcount = rte_cpu_to_be_32(data_len);
 	klm->pbuf = rte_cpu_to_be_64(addr);
-	klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf, 0);
+	klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
 	return klm->lkey;
 
 }

index 017a731b3f..edc5203dd6 100644
@@ -34,24 +34,26 @@ mlx5_mp_os_handle_port_agnostic(const struct rte_mp_msg *mp_msg,
 	struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param;
 	const struct mlx5_mp_param *param =
 		(const struct mlx5_mp_param *)mp_msg->param;
-	const struct mlx5_mp_arg_mempool_reg *mpr;
-	struct mlx5_mp_id mp_id;
+	const struct mlx5_mp_arg_mr_manage *mng = &param->args.mr_manage;
+	struct mr_cache_entry entry;
+	uint32_t lkey;
 
 	switch (param->type) {
+	case MLX5_MP_REQ_CREATE_MR:
+		mp_init_port_agnostic_msg(&mp_res, param->type);
+		lkey = mlx5_mr_create(mng->cdev, &mng->cdev->mr_scache, &entry,
+				      mng->addr);
+		if (lkey == UINT32_MAX)
+			res->result = -rte_errno;
+		return rte_mp_reply(&mp_res, peer);
 	case MLX5_MP_REQ_MEMPOOL_REGISTER:
-		mlx5_mp_id_init(&mp_id, param->port_id);
-		mp_init_msg(&mp_id, &mp_res, param->type);
-		mpr = &param->args.mempool_reg;
-		res->result = mlx5_mr_mempool_register(mpr->share_cache,
-						       mpr->pd, mpr->mempool,
-						       NULL);
+		mp_init_port_agnostic_msg(&mp_res, param->type);
+		res->result = mlx5_mr_mempool_register(mng->cdev, mng->mempool);
 		return rte_mp_reply(&mp_res, peer);
 	case MLX5_MP_REQ_MEMPOOL_UNREGISTER:
-		mlx5_mp_id_init(&mp_id, param->port_id);
-		mp_init_msg(&mp_id, &mp_res, param->type);
-		mpr = &param->args.mempool_reg;
-		res->result = mlx5_mr_mempool_unregister(mpr->share_cache,
-							 mpr->mempool, NULL);
+		mp_init_port_agnostic_msg(&mp_res, param->type);
+		res->result = mlx5_mr_mempool_unregister(mng->cdev,
+							 mng->mempool);
 		return rte_mp_reply(&mp_res, peer);
 	default:
 		return 1;

@@ -69,8 +71,6 @@ mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
 	struct rte_eth_dev *dev;
 	struct mlx5_priv *priv;
 	struct mlx5_common_device *cdev;
-	struct mr_cache_entry entry;
-	uint32_t lkey;
 	int ret;
 
 	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);

@@ -88,15 +88,6 @@ mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
 	priv = dev->data->dev_private;
 	cdev = priv->sh->cdev;
 	switch (param->type) {
-	case MLX5_MP_REQ_CREATE_MR:
-		mp_init_msg(&priv->mp_id, &mp_res, param->type);
-		lkey = mlx5_mr_create_primary(cdev->pd, &cdev->mr_scache,
-					      &entry, param->args.addr,
-					      cdev->config.mr_ext_memseg_en);
-		if (lkey == UINT32_MAX)
-			res->result = -rte_errno;
-		ret = rte_mp_reply(&mp_res, peer);
-		break;
 	case MLX5_MP_REQ_VERBS_CMD_FD:
 		mp_init_msg(&priv->mp_id, &mp_res, param->type);
 		mp_res.num_fds = 1;

index 1fc2f0e0c1..5f6dae953b 100644
@@ -1247,8 +1247,7 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev)
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
-	ret = mlx5_mr_mempool_register(&priv->sh->cdev->mr_scache,
-				       priv->sh->cdev->pd, mp, &priv->mp_id);
+	ret = mlx5_mr_mempool_register(priv->sh->cdev, mp);
 	if (ret < 0 && rte_errno != EEXIST) {
 		ret = rte_errno;
 		DRV_LOG(ERR, "port %u failed to register a mempool for Multi-Packet RQ",

index d916c8addc..172419ca56 100644
@@ -147,9 +147,7 @@ mlx5_rxq_mempool_register(struct mlx5_rxq_ctrl *rxq_ctrl)
 	}
 	for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++) {
 		mp = rxq_ctrl->rxq.rxseg[s].mp;
-		ret = mlx5_mr_mempool_register(&priv->sh->cdev->mr_scache,
-					       priv->sh->cdev->pd, mp,
-					       &priv->mp_id);
+		ret = mlx5_mr_mempool_register(priv->sh->cdev, mp);
 		if (ret < 0 && rte_errno != EEXIST)
 			return ret;
 		rte_mempool_mem_iter(mp, mlx5_rxq_mempool_register_cb,

index 7fed0e7cb9..0b9109a115 100644
@@ -351,28 +351,6 @@ __mlx5_uar_write64(uint64_t val, void *addr, rte_spinlock_t *lock)
 #define mlx5_uar_write64(val, dst, lock) __mlx5_uar_write64(val, dst, lock)
 #endif
 
-/**
- * Query LKey from a packet buffer for Tx.
- *
- * @param txq
- *   Pointer to Tx queue structure.
- * @param mb
- *   Pointer to mbuf.
- *
- * @return
- *   Searched LKey on success, UINT32_MAX on no match.
- */
-static __rte_always_inline uint32_t
-mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
-{
-	struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
-	struct mlx5_txq_ctrl *txq_ctrl =
-			container_of(txq, struct mlx5_txq_ctrl, txq);
-
-	/* Take slower bottom-half on miss. */
-	return mlx5_mr_mb2mr(mr_ctrl, mb, &txq_ctrl->priv->mp_id);
-}
-
 /**
  * Ring TX queue doorbell and flush the update if requested.
  *

@@ -1370,7 +1348,7 @@ mlx5_tx_dseg_ptr(struct mlx5_txq_data *__rte_restrict txq,
 {
 	MLX5_ASSERT(len);
 	dseg->bcount = rte_cpu_to_be_32(len);
-	dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);
+	dseg->lkey = mlx5_mr_mb2mr(&txq->mr_ctrl, loc->mbuf);
 	dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);
 }
 

@@ -1406,7 +1384,7 @@ mlx5_tx_dseg_iptr(struct mlx5_txq_data *__rte_restrict txq,
 	MLX5_ASSERT(len);
 	if (len > MLX5_DSEG_MIN_INLINE_SIZE) {
 		dseg->bcount = rte_cpu_to_be_32(len);
-		dseg->lkey = mlx5_tx_mb2mr(txq, loc->mbuf);
+		dseg->lkey = mlx5_mr_mb2mr(&txq->mr_ctrl, loc->mbuf);
 		dseg->pbuf = rte_cpu_to_be_64((uintptr_t)buf);
 
 		return;

index 943cb9c19e..03875632a8 100644
@@ -160,7 +160,7 @@ prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 	struct mlx5_klm klm;
 
 	klm.byte_count = rte_pktmbuf_data_len(op->mbuf);
-	klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, op->mbuf, 0);
+	klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, op->mbuf);
 	klm.address = rte_pktmbuf_mtod(op->mbuf, uintptr_t);
 	__prep_one(priv, qp_obj, op, job, qp_obj->pi, &klm);
 	qp_obj->db_pi = qp_obj->pi;

@@ -330,7 +330,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 				addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
 				/* Build indirect mkey seg's KLM. */
 				mkey_klm->mkey = mlx5_mr_mb2mr(&qp->mr_ctrl,
-							       mbuf, 0);
+							       mbuf);
 				mkey_klm->address = rte_cpu_to_be_64(addr);
 				mkey_klm->byte_count = rte_cpu_to_be_32
 						(rte_pktmbuf_data_len(mbuf));

@@ -347,7 +347,7 @@ prep_regex_umr_wqe_set(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp,
 			klm.byte_count = scatter_size;
 		} else {
 			/* The single mubf case. Build the KLM directly. */
-			klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf, 0);
+			klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
 			klm.address = rte_pktmbuf_mtod(mbuf, uintptr_t);
 			klm.byte_count = rte_pktmbuf_data_len(mbuf);
 		}