net/mlx5: reduce Tx datapath compile time

-----Original Message-----
From: Michael Baum <[email protected]>
Sent: Monday, April 5, 2021 17:01
To: [email protected]
Cc: Matan Azrad <[email protected]>; Raslan Darawsheh <[email protected]>; Slava Ovsiienko <[email protected]>
Subject: [PATCH 1/6] net/mlx5: separate Rx function declarations to another file

The mlx5_rxtx.c file contains a lot of Tx burst functions, each of those is performance-optimized for the specific set of requested offloads. These ones are generated on the basis of the template function and it takes significant time to compile, just due to a large number of giant functions generated in the same file and this compilation is not being done in parallel with using multithreading.

Therefore we can split the mlx5_rxtx.c file into several separate files to allow different functions to be compiled simultaneously. In this patch, we separate Rx function declarations to different header file in preparation for removing them from the source file and as an optional preparation step for further consolidation of Rx burst functions.

Signed-off-by Michael Baum <[email protected]>

---
 drivers/net/mlx5/linux/mlx5_mp_os.c |   1 +
 drivers/net/mlx5/linux/mlx5_os.c    |   1 +
 drivers/net/mlx5/linux/mlx5_verbs.c |   1 +
 drivers/net/mlx5/mlx5.c             |   1 +
 drivers/net/mlx5/mlx5_devx.c        |   1 +
 drivers/net/mlx5/mlx5_ethdev.c      |   1 +
 drivers/net/mlx5/mlx5_flow.c        |   1 +
 drivers/net/mlx5/mlx5_flow_dv.c     |   1 +
 drivers/net/mlx5/mlx5_flow_verbs.c  |   1 +
 drivers/net/mlx5/mlx5_mr.c          |   1 +
 drivers/net/mlx5/mlx5_rss.c         |   1 +
 drivers/net/mlx5/mlx5_rx.h          | 598
++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxq.c         |   1 +
 drivers/net/mlx5/mlx5_rxtx.c        |   1 +
 drivers/net/mlx5/mlx5_rxtx.h        | 569 ----------------------------------
 drivers/net/mlx5/mlx5_rxtx_vec.c    |   1 +
 drivers/net/mlx5/mlx5_stats.c       |   1 +
 drivers/net/mlx5/mlx5_trigger.c     |   1 +
 drivers/net/mlx5/mlx5_txpp.c        |   1 +
 drivers/net/mlx5/mlx5_vlan.c        |   1 +
 drivers/net/mlx5/windows/mlx5_os.c  |   1 +
 21 files changed, 617 insertions(+), 569 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rx.h

drivers/net/mlx5/linux/mlx5_mp_os.c +1 -0

index 8011ca8..63fa278 100644
@@ -16,6 +16,7 @@
 
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_utils.h"
 
 int

drivers/net/mlx5/linux/mlx5_os.c +1 -0

index 2d5bcab..97a28ec 100644
@@ -40,6 +40,7 @@
 #include "mlx5_common_os.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 #include "mlx5_flow.h"

drivers/net/mlx5/linux/mlx5_verbs.c +1 -0

index c7d4b17..73096af 100644
@@ -22,6 +22,7 @@
 #include <mlx5_common_mr.h>
 #include <mlx5_rxtx.h>
 #include <mlx5_verbs.h>
+#include <mlx5_rx.h>
 #include <mlx5_utils.h>
 #include <mlx5_malloc.h>
 

drivers/net/mlx5/mlx5.c +1 -0

index 9557d06..6f77bc2 100644
@@ -35,6 +35,7 @@
 #include "mlx5.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 #include "mlx5_flow.h"

drivers/net/mlx5/mlx5_devx.c +1 -0

index 5c940ed..76935f6 100644
@@ -21,6 +21,7 @@
 #include "mlx5.h"
 #include "mlx5_common_os.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_utils.h"
 #include "mlx5_devx.h"
 #include "mlx5_flow.h"

drivers/net/mlx5/mlx5_ethdev.c +1 -0

index 564d713..708e3a3 100644
@@ -23,6 +23,7 @@
 #include <mlx5_malloc.h>
 
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_autoconf.h"
 
 /**

drivers/net/mlx5/mlx5_flow.c +1 -0

index c347f81..b3877a1 100644
@@ -30,6 +30,7 @@
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_common_os.h"
 #include "rte_pmd_mlx5.h"
 

drivers/net/mlx5/mlx5_flow_dv.c +1 -0

index 533dadf..cac05fb 100644
@@ -33,6 +33,7 @@
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "rte_pmd_mlx5.h"
 
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) ||

!defined(HAVE_INFINIBAND_VERBS_H)

drivers/net/mlx5/mlx5_flow_verbs.c +1 -0

index b442b9b..c331350 100644
@@ -24,6 +24,7 @@
 #include "mlx5.h"
 #include "mlx5_flow.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 
 #define VERBS_SPEC_INNER(item_flags) \
 	(!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ?

IBV_FLOW_SPEC_INNER : 0)

drivers/net/mlx5/mlx5_mr.c +1 -0

index 3255393..2014936 100644
@@ -15,6 +15,7 @@
 #include "mlx5.h"
 #include "mlx5_mr.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 
 struct mr_find_contig_memsegs_data {
 	uintptr_t addr;

drivers/net/mlx5/mlx5_rss.c +1 -0

index dc0131a..c32129c 100644
@@ -16,6 +16,7 @@
 #include "mlx5_defs.h"
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 
 /**
  * DPDK callback to update the RSS hash configuration.

drivers/net/mlx5/mlx5_rx.h new file +31 -0

new file mode 100644
index 0000000..83b1f38
@@ -0,0 +1,598 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 6WIND S.A.
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_RX_H_
+#define RTE_PMD_MLX5_RX_H_
+
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_common.h>
+#include <rte_spinlock.h>
+
+#include <mlx5_common_mr.h>
+
+#include "mlx5.h"
+#include "mlx5_autoconf.h"
+#include "mlx5_mr.h"
+
+/* Support tunnel matching. */
+#define MLX5_FLOW_TUNNEL 10
+
+struct mlx5_rxq_stats {
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	uint64_t ipackets; /**< Total of successfully received packets. */
+	uint64_t ibytes; /**< Total of successfully received bytes. */
+#endif
+	uint64_t idropped; /**< Total of packets dropped when RX ring full.

*/

drivers/net/mlx5/mlx5_rx.h new file +17 -0

+	uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */
+};
+
+/* Compressed CQE context. */
+struct rxq_zip {
+	uint16_t ai; /* Array index. */
+	uint16_t ca; /* Current array index. */
+	uint16_t na; /* Next array index. */
+	uint16_t cq_ci; /* The next CQE. */
+	uint32_t cqe_cnt; /* Number of CQEs. */
+};
+
+/* Multi-Packet RQ buffer header. */
+struct mlx5_mprq_buf {
+	struct rte_mempool *mp;
+	uint16_t refcnt; /* Atomically accessed refcnt. */
+	uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first

packet. */

drivers/net/mlx5/mlx5_rx.h new file +4 -0

+	struct rte_mbuf_ext_shared_info shinfos[];
+	/*
+	 * Shared information per stride.
+	 * More memory will be allocated for the first stride head-room and

for

drivers/net/mlx5/mlx5_rx.h new file +28 -0

+	 * the strides data.
+	 */
+} __rte_cache_aligned;
+
+/* Get pointer to the first stride. */
+#define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \
+				sizeof(struct mlx5_mprq_buf) + \
+				(strd_n) * \
+				sizeof(struct rte_mbuf_ext_shared_info) + \
+				RTE_PKTMBUF_HEADROOM))
+
+#define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
+#define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
+
+enum mlx5_rxq_err_state {
+	MLX5_RXQ_ERR_STATE_NO_ERROR = 0,
+	MLX5_RXQ_ERR_STATE_NEED_RESET,
+	MLX5_RXQ_ERR_STATE_NEED_READY,
+};
+
+enum mlx5_rqx_code {
+	MLX5_RXQ_CODE_EXIT = 0,
+	MLX5_RXQ_CODE_NOMBUF,
+	MLX5_RXQ_CODE_DROPPED,
+};
+
+struct mlx5_eth_rxseg {
+	struct rte_mempool *mp; /**< Memory pool to allocate segment

from. */

drivers/net/mlx5/mlx5_rx.h new file +2 -0

+	uint16_t length; /**< Segment data length, configures split point. */
+	uint16_t offset; /**< Data offset from beginning of mbuf data

buffer. */

drivers/net/mlx5/mlx5_rx.h new file +40 -0

+	uint32_t reserved; /**< Reserved field. */
+};
+
+/* RX queue descriptor. */
+struct mlx5_rxq_data {
+	unsigned int csum:1; /* Enable checksum offloading. */
+	unsigned int hw_timestamp:1; /* Enable HW timestamp. */
+	unsigned int rt_timestamp:1; /* Realtime timestamp format. */
+	unsigned int vlan_strip:1; /* Enable VLAN stripping. */
+	unsigned int crc_present:1; /* CRC must be subtracted. */
+	unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */
+	unsigned int cqe_n:4; /* Log 2 of CQ elements. */
+	unsigned int elts_n:4; /* Log 2 of Mbufs. */
+	unsigned int rss_hash:1; /* RSS hash result is enabled. */
+	unsigned int mark:1; /* Marked flow available on the queue. */
+	unsigned int strd_num_n:5; /* Log 2 of the number of stride. */
+	unsigned int strd_sz_n:4; /* Log 2 of stride size. */
+	unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */
+	unsigned int err_state:2; /* enum mlx5_rxq_err_state. */
+	unsigned int strd_scatter_en:1; /* Scattered packets from a stride. */
+	unsigned int lro:1; /* Enable LRO. */
+	unsigned int dynf_meta:1; /* Dynamic metadata is configured. */
+	unsigned int mcqe_format:3; /* CQE compression format. */
+	volatile uint32_t *rq_db;
+	volatile uint32_t *cq_db;
+	uint16_t port_id;
+	uint32_t elts_ci;
+	uint32_t rq_ci;
+	uint16_t consumed_strd; /* Number of consumed strides in WQE. */
+	uint32_t rq_pi;
+	uint32_t cq_ci;
+	uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */
+	uint32_t byte_mask;
+	union {
+		struct rxq_zip zip; /* Compressed context. */
+		uint16_t decompressed;
+		/* Number of ready mbufs decompressed from the CQ. */
+	};
+	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
+	uint16_t mprq_max_memcpy_len; /* Maximum size of packet to

memcpy. */

drivers/net/mlx5/mlx5_rx.h new file +6 -0

+	volatile void *wqes;
+	volatile struct mlx5_cqe(*cqes)[];
+	struct rte_mbuf *(*elts)[];
+	struct mlx5_mprq_buf *(*mprq_bufs)[];
+	struct rte_mempool *mp;
+	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ.

*/

drivers/net/mlx5/mlx5_rx.h new file +1 -0

+	struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish.

*/

drivers/net/mlx5/mlx5_rx.h new file +4 -0

+	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
+	uint16_t idx; /* Queue index. */
+	struct mlx5_rxq_stats stats;
+	rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx.

*/

drivers/net/mlx5/mlx5_rx.h new file +36 -0

+	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
+	void *cq_uar; /* Verbs CQ user access region. */
+	uint32_t cqn; /* CQ number. */
+	uint8_t cq_arm_sn; /* CQ arm seq number. */
+#ifndef RTE_ARCH_64
+	rte_spinlock_t *uar_lock_cq;
+	/* CQ (UAR) access lock required for 32bit implementations */
+#endif
+	uint32_t tunnel; /* Tunnel information. */
+	int timestamp_offset; /* Dynamic mbuf field for timestamp. */
+	uint64_t timestamp_rx_flag; /* Dynamic mbuf flag for timestamp. */
+	uint64_t flow_meta_mask;
+	int32_t flow_meta_offset;
+	uint32_t flow_meta_port_mask;
+	uint32_t rxseg_n; /* Number of split segment descriptions. */
+	struct mlx5_eth_rxseg rxseg[MLX5_MAX_RXQ_NSEG];
+	/* Buffer split segment descriptions - sizes, offsets, pools. */
+} __rte_cache_aligned;
+
+enum mlx5_rxq_type {
+	MLX5_RXQ_TYPE_STANDARD, /* Standard Rx queue. */
+	MLX5_RXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */
+	MLX5_RXQ_TYPE_UNDEFINED,
+};
+
+/* RX queue control descriptor. */
+struct mlx5_rxq_ctrl {
+	struct mlx5_rxq_data rxq; /* Data path structure. */
+	LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
+	uint32_t refcnt; /* Reference counter. */
+	struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */
+	struct mlx5_priv *priv; /* Back pointer to private data. */
+	enum mlx5_rxq_type type; /* Rxq type. */
+	unsigned int socket; /* CPU socket ID for allocations. */
+	unsigned int irq:1; /* Whether IRQ is enabled. */
+	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this

Queue. */

drivers/net/mlx5/mlx5_rx.h new file +1 -0

+	uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels

counters. */

drivers/net/mlx5/mlx5_rx.h new file +16 -0

+	uint32_t wqn; /* WQ number. */
+	uint16_t dump_file_n; /* Number of dump files. */
+	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
+	uint32_t hairpin_status; /* Hairpin binding status. */
+};
+
+/* mlx5_rxq.c */
+
+extern uint8_t rss_hash_default_key[];
+
+unsigned int mlx5_rxq_cqe_num(struct mlx5_rxq_data *rxq_data);
+int mlx5_mprq_free_mp(struct rte_eth_dev *dev);
+int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev);
+int mlx5_rx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_rx_queue_start_primary(struct rte_eth_dev *dev, uint16_t

queue_id);
+int mlx5_rx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t queue_id);
+int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,

drivers/net/mlx5/mlx5_rx.h new file +1 -0

+			unsigned int socket, const struct rte_eth_rxconf

*conf,

drivers/net/mlx5/mlx5_rx.h new file +11 -0

+			struct rte_mempool *mp);
+int mlx5_rx_hairpin_queue_setup
+	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+	 const struct rte_eth_hairpin_conf *hairpin_conf);
+void mlx5_rx_queue_release(void *dpdk_rxq);
+int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev);
+void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
+int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mlx5_rxq_obj_verify(struct rte_eth_dev *dev);
+struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t

idx,

drivers/net/mlx5/mlx5_rx.h new file +13 -0

+				   uint16_t desc, unsigned int socket,
+				   const struct rte_eth_rxconf *conf,
+				   const struct rte_eth_rxseg_split *rx_seg,
+				   uint16_t n_seg);
+struct mlx5_rxq_ctrl *mlx5_rxq_hairpin_new
+	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
+	 const struct rte_eth_hairpin_conf *hairpin_conf);
+struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
+int mlx5_rxq_verify(struct rte_eth_dev *dev);
+int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
+int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev);
+struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev

*dev,

drivers/net/mlx5/mlx5_rx.h new file +12 -0

+						  const uint16_t *queues,
+						  uint32_t queues_n);
+int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
+			       struct mlx5_ind_table_obj *ind_tbl,
+			       bool standalone);
+int mlx5_ind_table_obj_setup(struct rte_eth_dev *dev,
+			     struct mlx5_ind_table_obj *ind_tbl);
+int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
+			      struct mlx5_ind_table_obj *ind_tbl,
+			      uint16_t *queues, const uint32_t queues_n,
+			      bool standalone);
+struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list

*list,

drivers/net/mlx5/mlx5_rx.h new file +1 -0

+		struct mlx5_cache_entry *entry __rte_unused, void

*cb_ctx);

drivers/net/mlx5/mlx5_rx.h new file +10 -0

+int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
+		       struct mlx5_cache_entry *entry,
+		       void *cb_ctx);
+void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
+			 struct mlx5_cache_entry *entry);
+uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
+		       struct mlx5_flow_rss_desc *rss_desc);
+int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
+uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev);
+enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev,

uint16_t idx);

drivers/net/mlx5/mlx5_rx.h new file +15 -0

+const struct rte_eth_hairpin_conf *mlx5_rxq_get_hairpin_conf
+	(struct rte_eth_dev *dev, uint16_t idx);
+struct mlx5_hrxq *mlx5_drop_action_create(struct rte_eth_dev *dev);
+void mlx5_drop_action_destroy(struct rte_eth_dev *dev);
+uint64_t mlx5_get_rx_port_offloads(void);
+uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
+void mlx5_rxq_timestamp_set(struct rte_eth_dev *dev);
+int mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hxrq_idx,
+		     const uint8_t *rss_key, uint32_t rss_key_len,
+		     uint64_t hash_fields,
+		     const uint16_t *queues, uint32_t queues_n);
+
+/* mlx5_rxtx.c */
+
+uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t

pkts_n);

drivers/net/mlx5/mlx5_rx.h new file +2 -0

+void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);
+__rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t

vec);

drivers/net/mlx5/mlx5_rx.h new file +8 -0

+void mlx5_mprq_buf_free_cb(void *addr, void *opaque);
+void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf);
+uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts,
+			    uint16_t pkts_n);
+uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts,
+			  uint16_t pkts_n);
+int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
+uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t

rx_queue_id);

drivers/net/mlx5/mlx5_rx.h new file +3 -0

+void mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+		       struct rte_eth_rxq_info *qinfo);
+int mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t

rx_queue_id,

drivers/net/mlx5/mlx5_rx.h new file +8 -0

+			   struct rte_eth_burst_mode *mode);
+
+/* Vectorized version of mlx5_rxtx.c */
+int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);
+int mlx5_check_vec_rx_support(struct rte_eth_dev *dev);
+uint16_t mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
+			   uint16_t pkts_n);
+uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf

**pkts,

drivers/net/mlx5/mlx5_rx.h new file +8 -0

+				uint16_t pkts_n);
+
+/* mlx5_mr.c */
+
+uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
+
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches for

Rx
+ * as mempool is pre-configured and static.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */

drivers/net/mlx5/mlx5_rx.h new file +16 -0

+static __rte_always_inline uint32_t
+mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
+{
+	struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+	uint32_t lkey;
+
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half (Binary Search) on miss. */
+	return mlx5_rx_addr2mr_bh(rxq, addr);
+}
+
+#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)-

Quoted 1 line

buf_addr))

drivers/net/mlx5/mlx5_rx.h new file +35 -0

+
+/**
+ * Convert timestamp from HW format to linear counter
+ * from Packet Pacing Clock Queue CQE timestamp format.
+ *
+ * @param sh
+ *   Pointer to the device shared context. Might be needed
+ *   to convert according current device configuration.
+ * @param ts
+ *   Timestamp from CQE to convert.
+ * @return
+ *   UTC in nanoseconds
+ */
+static __rte_always_inline uint64_t
+mlx5_txpp_convert_rx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t ts)
+{
+	RTE_SET_USED(sh);
+	return (ts & UINT32_MAX) + (ts >> 32) * NS_PER_S;
+}
+
+/**
+ * Set timestamp in mbuf dynamic field.
+ *
+ * @param mbuf
+ *   Structure to write into.
+ * @param offset
+ *   Dynamic field offset in mbuf structure.
+ * @param timestamp
+ *   Value to write.
+ */
+static __rte_always_inline void
+mlx5_timestamp_set(struct rte_mbuf *mbuf, int offset,
+		rte_mbuf_timestamp_t timestamp)
+{
+	*RTE_MBUF_DYNFIELD(mbuf, offset, rte_mbuf_timestamp_t *) =

timestamp;

drivers/net/mlx5/mlx5_rx.h new file +17 -0

+}
+
+/**
+ * Replace MPRQ buffer.
+ *
+ * @param rxq
+ *   Pointer to Rx queue structure.
+ * @param rq_idx
+ *   RQ index to replace.
+ */
+static __rte_always_inline void
+mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx)
+{
+	const uint32_t strd_n = 1 << rxq->strd_num_n;
+	struct mlx5_mprq_buf *rep = rxq->mprq_repl;
+	volatile struct mlx5_wqe_data_seg *wqe =
+		&((volatile struct mlx5_wqe_mprq *)rxq-

Quoted 1 line

wqes)[rq_idx].dseg;

drivers/net/mlx5/mlx5_rx.h new file +12 -0

+	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_idx];
+	void *addr;
+
+	if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) > 1) {
+		MLX5_ASSERT(rep != NULL);
+		/* Replace MPRQ buf. */
+		(*rxq->mprq_bufs)[rq_idx] = rep;
+		/* Replace WQE. */
+		addr = mlx5_mprq_buf_addr(rep, strd_n);
+		wqe->addr = rte_cpu_to_be_64((uintptr_t)addr);
+		/* If there's only one MR, no need to replace LKey in WQE. */
+		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) >

1))

drivers/net/mlx5/mlx5_rx.h new file +3 -0

+			wqe->lkey = mlx5_rx_addr2mr(rxq, (uintptr_t)addr);
+		/* Stash a mbuf for next replacement. */
+		if (likely(!rte_mempool_get(rxq->mprq_mp, (void

**)&rep)))

drivers/net/mlx5/mlx5_rx.h new file +37 -0

+			rxq->mprq_repl = rep;
+		else
+			rxq->mprq_repl = NULL;
+		/* Release the old buffer. */
+		mlx5_mprq_buf_free(buf);
+	} else if (unlikely(rxq->mprq_repl == NULL)) {
+		struct mlx5_mprq_buf *rep;
+
+		/*
+		 * Currently, the MPRQ mempool is out of buffer
+		 * and doing memcpy regardless of the size of Rx
+		 * packet. Retry allocation to get back to
+		 * normal.
+		 */
+		if (!rte_mempool_get(rxq->mprq_mp, (void **)&rep))
+			rxq->mprq_repl = rep;
+	}
+}
+
+/**
+ * Attach or copy MPRQ buffer content to a packet.
+ *
+ * @param rxq
+ *   Pointer to Rx queue structure.
+ * @param pkt
+ *   Pointer to a packet to fill.
+ * @param len
+ *   Packet length.
+ * @param buf
+ *   Pointer to a MPRQ buffer to take the data from.
+ * @param strd_idx
+ *   Stride index to start from.
+ * @param strd_cnt
+ *   Number of strides to consume.
+ */
+static __rte_always_inline enum mlx5_rqx_code
+mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,

uint32_t len,

drivers/net/mlx5/mlx5_rx.h new file +1 -0

+		struct mlx5_mprq_buf *buf, uint16_t strd_idx, uint16_t

strd_cnt)

drivers/net/mlx5/mlx5_rx.h new file +9 -0

+{
+	const uint32_t strd_n = 1 << rxq->strd_num_n;
+	const uint16_t strd_sz = 1 << rxq->strd_sz_n;
+	const uint16_t strd_shift =
+		MLX5_MPRQ_STRIDE_SHIFT_BYTE * rxq->strd_shift_en;
+	const int32_t hdrm_overlap =
+		len + RTE_PKTMBUF_HEADROOM - strd_cnt * strd_sz;
+	const uint32_t offset = strd_idx * strd_sz + strd_shift;
+	void *addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf, strd_n),

offset);

drivers/net/mlx5/mlx5_rx.h new file +12 -0

+
+	/*
+	 * Memcpy packets to the target mbuf if:
+	 * - The size of packet is smaller than mprq_max_memcpy_len.
+	 * - Out of buffer in the Mempool for Multi-Packet RQ.
+	 * - The packet's stride overlaps a headroom and scatter is off.
+	 */
+	if (len <= rxq->mprq_max_memcpy_len ||
+	    rxq->mprq_repl == NULL ||
+	    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
+		if (likely(len <=
+			   (uint32_t)(pkt->buf_len -

RTE_PKTMBUF_HEADROOM))) {

drivers/net/mlx5/mlx5_rx.h new file +7 -0

+			rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
+				   addr, len);
+			DATA_LEN(pkt) = len;
+		} else if (rxq->strd_scatter_en) {
+			struct rte_mbuf *prev = pkt;
+			uint32_t seg_len = RTE_MIN(len, (uint32_t)
+				(pkt->buf_len -

RTE_PKTMBUF_HEADROOM));

drivers/net/mlx5/mlx5_rx.h new file +16 -0

+			uint32_t rem_len = len - seg_len;
+
+			rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
+				   addr, seg_len);
+			DATA_LEN(pkt) = seg_len;
+			while (rem_len) {
+				struct rte_mbuf *next =
+					rte_pktmbuf_alloc(rxq->mp);
+
+				if (unlikely(next == NULL))
+					return MLX5_RXQ_CODE_NOMBUF;
+				NEXT(prev) = next;
+				SET_DATA_OFF(next, 0);
+				addr = RTE_PTR_ADD(addr, seg_len);
+				seg_len = RTE_MIN(rem_len, (uint32_t)
+					(next->buf_len -

RTE_PKTMBUF_HEADROOM));

drivers/net/mlx5/mlx5_rx.h new file +22 -0

+				rte_memcpy
+					(rte_pktmbuf_mtod(next, void *),
+					 addr, seg_len);
+				DATA_LEN(next) = seg_len;
+				rem_len -= seg_len;
+				prev = next;
+				++NB_SEGS(pkt);
+			}
+		} else {
+			return MLX5_RXQ_CODE_DROPPED;
+		}
+	} else {
+		rte_iova_t buf_iova;
+		struct rte_mbuf_ext_shared_info *shinfo;
+		uint16_t buf_len = strd_cnt * strd_sz;
+		void *buf_addr;
+
+		/* Increment the refcnt of the whole chunk. */
+		__atomic_add_fetch(&buf->refcnt, 1, __ATOMIC_RELAXED);
+		MLX5_ASSERT(__atomic_load_n(&buf->refcnt,
+			    __ATOMIC_RELAXED) <= strd_n + 1);
+		buf_addr = RTE_PTR_SUB(addr,

RTE_PKTMBUF_HEADROOM);

drivers/net/mlx5/mlx5_rx.h new file +110 -0

+		/*
+		 * MLX5 device doesn't use iova but it is necessary in a
+		 * case where the Rx packet is transmitted via a
+		 * different PMD.
+		 */
+		buf_iova = rte_mempool_virt2iova(buf) +
+			   RTE_PTR_DIFF(buf_addr, buf);
+		shinfo = &buf->shinfos[strd_idx];
+		rte_mbuf_ext_refcnt_set(shinfo, 1);
+		/*
+		 * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when
+		 * attaching the stride to mbuf and more offload flags
+		 * will be added below by calling rxq_cq_to_mbuf().
+		 * Other fields will be overwritten.
+		 */
+		rte_pktmbuf_attach_extbuf(pkt, buf_addr, buf_iova,
+					  buf_len, shinfo);
+		/* Set mbuf head-room. */
+		SET_DATA_OFF(pkt, RTE_PKTMBUF_HEADROOM);
+		MLX5_ASSERT(pkt->ol_flags == EXT_ATTACHED_MBUF);
+		MLX5_ASSERT(rte_pktmbuf_tailroom(pkt) >=
+			len - (hdrm_overlap > 0 ? hdrm_overlap : 0));
+		DATA_LEN(pkt) = len;
+		/*
+		 * Copy the last fragment of a packet (up to headroom
+		 * size bytes) in case there is a stride overlap with
+		 * a next packet's headroom. Allocate a separate mbuf
+		 * to store this fragment and link it. Scatter is on.
+		 */
+		if (hdrm_overlap > 0) {
+			MLX5_ASSERT(rxq->strd_scatter_en);
+			struct rte_mbuf *seg =
+				rte_pktmbuf_alloc(rxq->mp);
+
+			if (unlikely(seg == NULL))
+				return MLX5_RXQ_CODE_NOMBUF;
+			SET_DATA_OFF(seg, 0);
+			rte_memcpy(rte_pktmbuf_mtod(seg, void *),
+				RTE_PTR_ADD(addr, len - hdrm_overlap),
+				hdrm_overlap);
+			DATA_LEN(seg) = hdrm_overlap;
+			DATA_LEN(pkt) = len - hdrm_overlap;
+			NEXT(pkt) = seg;
+			NB_SEGS(pkt) = 2;
+		}
+	}
+	return MLX5_RXQ_CODE_EXIT;
+}
+
+/**
+ * Check whether Multi-Packet RQ can be enabled for the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   1 if supported, negative errno value if not.
+ */
+static __rte_always_inline int
+mlx5_check_mprq_support(struct rte_eth_dev *dev)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+
+	if (priv->config.mprq.enabled &&
+	    priv->rxqs_n >= priv->config.mprq.min_rxqs_num)
+		return 1;
+	return -ENOTSUP;
+}
+
+/**
+ * Check whether Multi-Packet RQ is enabled for the Rx queue.
+ *
+ *  @param rxq
+ *     Pointer to receive queue structure.
+ *
+ * @return
+ *   0 if disabled, otherwise enabled.
+ */
+static __rte_always_inline int
+mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq)
+{
+	return rxq->strd_num_n > 0;
+}
+
+/**
+ * Check whether Multi-Packet RQ is enabled for the device.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   0 if disabled, otherwise enabled.
+ */
+static __rte_always_inline int
+mlx5_mprq_enabled(struct rte_eth_dev *dev)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t i;
+	uint16_t n = 0;
+	uint16_t n_ibv = 0;
+
+	if (mlx5_check_mprq_support(dev) < 0)
+		return 0;
+	/* All the configured queues should be enabled. */
+	for (i = 0; i < priv->rxqs_n; ++i) {
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
+		struct mlx5_rxq_ctrl *rxq_ctrl = container_of
+			(rxq, struct mlx5_rxq_ctrl, rxq);
+
+		if (rxq == NULL || rxq_ctrl->type !=

MLX5_RXQ_TYPE_STANDARD)

drivers/net/mlx5/mlx5_rx.h new file +11 -0

+			continue;
+		n_ibv++;
+		if (mlx5_rxq_mprq_enabled(rxq))
+			++n;
+	}
+	/* Multi-Packet RQ can't be partially configured. */
+	MLX5_ASSERT(n == 0 || n == n_ibv);
+	return n == n_ibv;
+}
+
+#endif /* RTE_PMD_MLX5_RX_H_ */

drivers/net/mlx5/mlx5_rxq.c +1 -0

index 9009eb8..19df0fa 100644
@@ -25,6 +25,7 @@
 #include "mlx5_defs.h"
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_utils.h"
 #include "mlx5_autoconf.h"
 

drivers/net/mlx5/mlx5_rxtx.c +1 -0

index c76b995..d004e1e 100644
@@ -25,6 +25,7 @@
 #include "mlx5_mr.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 
 /* TX burst subroutines return codes. */
 enum mlx5_txcmp_code {

drivers/net/mlx5/mlx5_rxtx.h +0 -11

index 4f0fda0..d443db4 100644
@@ -31,21 +31,10 @@
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 
-/* Support tunnel matching. */
-#define MLX5_FLOW_TUNNEL 10
 
 /* Mbuf dynamic flag offset for inline. */
 extern uint64_t rte_net_mlx5_dynf_inline_mask;
 
-struct mlx5_rxq_stats {
-#ifdef MLX5_PMD_SOFT_COUNTERS
-	uint64_t ipackets; /**< Total of successfully received packets. */
-	uint64_t ibytes; /**< Total of successfully received bytes. */
-#endif
-	uint64_t idropped; /**< Total of packets dropped when RX ring full.
 */
-	uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */
-};
-
 struct mlx5_txq_stats {
 #ifdef MLX5_PMD_SOFT_COUNTERS

drivers/net/mlx5/mlx5_rxtx.h +0 -14

 	uint64_t opackets; /**< Total of successfully sent packets. */

@@ -56,148 +45,6 @@ struct mlx5_txq_stats {
 
 struct mlx5_priv;
 
-/* Compressed CQE context. */
-struct rxq_zip {
-	uint16_t ai; /* Array index. */
-	uint16_t ca; /* Current array index. */
-	uint16_t na; /* Next array index. */
-	uint16_t cq_ci; /* The next CQE. */
-	uint32_t cqe_cnt; /* Number of CQEs. */
-};
-
-/* Multi-Packet RQ buffer header. */
-struct mlx5_mprq_buf {
-	struct rte_mempool *mp;
-	uint16_t refcnt; /* Atomically accessed refcnt. */
-	uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first

packet. */

drivers/net/mlx5/mlx5_rxtx.h +0 -4

-	struct rte_mbuf_ext_shared_info shinfos[];
-	/*
-	 * Shared information per stride.
-	 * More memory will be allocated for the first stride head-room and

for

drivers/net/mlx5/mlx5_rxtx.h +0 -28

-	 * the strides data.
-	 */
-} __rte_cache_aligned;
-
-/* Get pointer to the first stride. */
-#define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \
-				sizeof(struct mlx5_mprq_buf) + \
-				(strd_n) * \
-				sizeof(struct rte_mbuf_ext_shared_info) + \
-				RTE_PKTMBUF_HEADROOM))
-
-#define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
-#define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
-
-enum mlx5_rxq_err_state {
-	MLX5_RXQ_ERR_STATE_NO_ERROR = 0,
-	MLX5_RXQ_ERR_STATE_NEED_RESET,
-	MLX5_RXQ_ERR_STATE_NEED_READY,
-};
-
-enum mlx5_rqx_code {
-	MLX5_RXQ_CODE_EXIT = 0,
-	MLX5_RXQ_CODE_NOMBUF,
-	MLX5_RXQ_CODE_DROPPED,
-};
-
-struct mlx5_eth_rxseg {
-	struct rte_mempool *mp; /**< Memory pool to allocate segment

from. */

drivers/net/mlx5/mlx5_rxtx.h +0 -2

-	uint16_t length; /**< Segment data length, configures split point. */
-	uint16_t offset; /**< Data offset from beginning of mbuf data

buffer. */

drivers/net/mlx5/mlx5_rxtx.h +0 -40

-	uint32_t reserved; /**< Reserved field. */
-};
-
-/* RX queue descriptor. */
-struct mlx5_rxq_data {
-	unsigned int csum:1; /* Enable checksum offloading. */
-	unsigned int hw_timestamp:1; /* Enable HW timestamp. */
-	unsigned int rt_timestamp:1; /* Realtime timestamp format. */
-	unsigned int vlan_strip:1; /* Enable VLAN stripping. */
-	unsigned int crc_present:1; /* CRC must be subtracted. */
-	unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */
-	unsigned int cqe_n:4; /* Log 2 of CQ elements. */
-	unsigned int elts_n:4; /* Log 2 of Mbufs. */
-	unsigned int rss_hash:1; /* RSS hash result is enabled. */
-	unsigned int mark:1; /* Marked flow available on the queue. */
-	unsigned int strd_num_n:5; /* Log 2 of the number of stride. */
-	unsigned int strd_sz_n:4; /* Log 2 of stride size. */
-	unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */
-	unsigned int err_state:2; /* enum mlx5_rxq_err_state. */
-	unsigned int strd_scatter_en:1; /* Scattered packets from a stride. */
-	unsigned int lro:1; /* Enable LRO. */
-	unsigned int dynf_meta:1; /* Dynamic metadata is configured. */
-	unsigned int mcqe_format:3; /* CQE compression format. */
-	volatile uint32_t *rq_db;
-	volatile uint32_t *cq_db;
-	uint16_t port_id;
-	uint32_t elts_ci;
-	uint32_t rq_ci;
-	uint16_t consumed_strd; /* Number of consumed strides in WQE. */
-	uint32_t rq_pi;
-	uint32_t cq_ci;
-	uint16_t rq_repl_thresh; /* Threshold for buffer replenishment. */
-	uint32_t byte_mask;
-	union {
-		struct rxq_zip zip; /* Compressed context. */
-		uint16_t decompressed;
-		/* Number of ready mbufs decompressed from the CQ. */
-	};
-	struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
-	uint16_t mprq_max_memcpy_len; /* Maximum size of packet to

memcpy. */

drivers/net/mlx5/mlx5_rxtx.h +0 -6

-	volatile void *wqes;
-	volatile struct mlx5_cqe(*cqes)[];
-	struct rte_mbuf *(*elts)[];
-	struct mlx5_mprq_buf *(*mprq_bufs)[];
-	struct rte_mempool *mp;
-	struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ.

*/

drivers/net/mlx5/mlx5_rxtx.h +0 -1

-	struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish.

*/

drivers/net/mlx5/mlx5_rxtx.h +0 -4

-	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
-	uint16_t idx; /* Queue index. */
-	struct mlx5_rxq_stats stats;
-	rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx.

*/

drivers/net/mlx5/mlx5_rxtx.h +0 -36

-	struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
-	void *cq_uar; /* Verbs CQ user access region. */
-	uint32_t cqn; /* CQ number. */
-	uint8_t cq_arm_sn; /* CQ arm seq number. */
-#ifndef RTE_ARCH_64
-	rte_spinlock_t *uar_lock_cq;
-	/* CQ (UAR) access lock required for 32bit implementations */
-#endif
-	uint32_t tunnel; /* Tunnel information. */
-	int timestamp_offset; /* Dynamic mbuf field for timestamp. */
-	uint64_t timestamp_rx_flag; /* Dynamic mbuf flag for timestamp. */
-	uint64_t flow_meta_mask;
-	int32_t flow_meta_offset;
-	uint32_t flow_meta_port_mask;
-	uint32_t rxseg_n; /* Number of split segment descriptions. */
-	struct mlx5_eth_rxseg rxseg[MLX5_MAX_RXQ_NSEG];
-	/* Buffer split segment descriptions - sizes, offsets, pools. */
-} __rte_cache_aligned;
-
-enum mlx5_rxq_type {
-	MLX5_RXQ_TYPE_STANDARD, /* Standard Rx queue. */
-	MLX5_RXQ_TYPE_HAIRPIN, /* Hairpin Rx queue. */
-	MLX5_RXQ_TYPE_UNDEFINED,
-};
-
-/* RX queue control descriptor. */
-struct mlx5_rxq_ctrl {
-	struct mlx5_rxq_data rxq; /* Data path structure. */
-	LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
-	uint32_t refcnt; /* Reference counter. */
-	struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */
-	struct mlx5_priv *priv; /* Back pointer to private data. */
-	enum mlx5_rxq_type type; /* Rxq type. */
-	unsigned int socket; /* CPU socket ID for allocations. */
-	unsigned int irq:1; /* Whether IRQ is enabled. */
-	uint32_t flow_mark_n; /* Number of Mark/Flag flows using this

Queue. */

drivers/net/mlx5/mlx5_rxtx.h +0 -1

-	uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels

counters. */

drivers/net/mlx5/mlx5_rxtx.h +0 -19

-	uint32_t wqn; /* WQ number. */
-	uint16_t dump_file_n; /* Number of dump files. */
-	struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */
-	uint32_t hairpin_status; /* Hairpin binding status. */
-};
-
 /* TX queue send local data. */
 __extension__
 struct mlx5_txq_local {

@@ -302,80 +149,6 @@ struct mlx5_txq_ctrl {
 #define MLX5_TX_BFREG(txq) \
 		(MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx])
 
-/* mlx5_rxq.c */
-
-extern uint8_t rss_hash_default_key[];
-
-unsigned int mlx5_rxq_cqe_num(struct mlx5_rxq_data *rxq_data);
-int mlx5_mprq_free_mp(struct rte_eth_dev *dev);
-int mlx5_mprq_alloc_mp(struct rte_eth_dev *dev);
-int mlx5_rx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);
-int mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id);
-int mlx5_rx_queue_start_primary(struct rte_eth_dev *dev, uint16_t
 queue_id);
-int mlx5_rx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t
 queue_id);
-int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t
 desc,
-			unsigned int socket, const struct rte_eth_rxconf

*conf,

drivers/net/mlx5/mlx5_rxtx.h +0 -24

-			struct rte_mempool *mp);
-int mlx5_rx_hairpin_queue_setup
-	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
-	 const struct rte_eth_hairpin_conf *hairpin_conf);
-void mlx5_rx_queue_release(void *dpdk_rxq);
-int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev);
-void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
-int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
-int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
-int mlx5_rxq_obj_verify(struct rte_eth_dev *dev);
-struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
-				   uint16_t desc, unsigned int socket,
-				   const struct rte_eth_rxconf *conf,
-				   const struct rte_eth_rxseg_split *rx_seg,
-				   uint16_t n_seg);
-struct mlx5_rxq_ctrl *mlx5_rxq_hairpin_new
-	(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
-	 const struct rte_eth_hairpin_conf *hairpin_conf);
-struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
-int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
-int mlx5_rxq_verify(struct rte_eth_dev *dev);
-int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
-int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev);
-struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev

*dev,

drivers/net/mlx5/mlx5_rxtx.h +0 -12

-						  const uint16_t *queues,
-						  uint32_t queues_n);
-int mlx5_ind_table_obj_release(struct rte_eth_dev *dev,
-			       struct mlx5_ind_table_obj *ind_tbl,
-			       bool standalone);
-int mlx5_ind_table_obj_setup(struct rte_eth_dev *dev,
-			     struct mlx5_ind_table_obj *ind_tbl);
-int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
-			      struct mlx5_ind_table_obj *ind_tbl,
-			      uint16_t *queues, const uint32_t queues_n,
-			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list

*list,

drivers/net/mlx5/mlx5_rxtx.h +0 -1

-		struct mlx5_cache_entry *entry __rte_unused, void

*cb_ctx);

drivers/net/mlx5/mlx5_rxtx.h +0 -12

-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
-		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
-uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
-		       struct mlx5_flow_rss_desc *rss_desc);
-int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
-uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev);
-
-
-enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev,

uint16_t idx);

drivers/net/mlx5/mlx5_rxtx.h +0 -33

-const struct rte_eth_hairpin_conf *mlx5_rxq_get_hairpin_conf
-	(struct rte_eth_dev *dev, uint16_t idx);
-struct mlx5_hrxq *mlx5_drop_action_create(struct rte_eth_dev *dev);
-void mlx5_drop_action_destroy(struct rte_eth_dev *dev);
-uint64_t mlx5_get_rx_port_offloads(void);
-uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
-void mlx5_rxq_timestamp_set(struct rte_eth_dev *dev);
-int mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hxrq_idx,
-		     const uint8_t *rss_key, uint32_t rss_key_len,
-		     uint64_t hash_fields,
-		     const uint16_t *queues, uint32_t queues_n);
-
 /* mlx5_txq.c */
 
 int mlx5_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id);

@@ -416,45 +189,21 @@ struct mlx5_txq_ctrl *mlx5_txq_hairpin_new
 void mlx5_set_ptype_table(void);
 void mlx5_set_cksum_table(void);
 void mlx5_set_swp_types_table(void);
-uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t
 pkts_n);
-void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);
-__rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t
 vec);
-void mlx5_mprq_buf_free_cb(void *addr, void *opaque);
-void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf);
-uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts,
-			    uint16_t pkts_n);
 uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts,
 			  uint16_t pkts_n);
-uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts,
-			  uint16_t pkts_n);
-int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
 int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
-uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t
 rx_queue_id);
 void mlx5_dump_debug_information(const char *path, const char *title,
 				 const void *buf, unsigned int len);
 int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
 			const struct mlx5_mp_arg_queue_state_modify
 *sm);
-void mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
-		       struct rte_eth_rxq_info *qinfo);
 void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 		       struct rte_eth_txq_info *qinfo);
-int mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
 rx_queue_id,
-			   struct rte_eth_burst_mode *mode);
 int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
 tx_queue_id,
 			   struct rte_eth_burst_mode *mode);
 
-/* Vectorized version of mlx5_rxtx.c */
-int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);
-int mlx5_check_vec_rx_support(struct rte_eth_dev *dev);
-uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
-			   uint16_t pkts_n);
-uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_txq, struct rte_mbuf

**pkts,

drivers/net/mlx5/mlx5_rxtx.h +0 -3

-				uint16_t pkts_n);
-
 /* mlx5_mr.c */
 
 void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
-uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
 uint32_t mlx5_tx_mb2mr_bh(struct mlx5_txq_data *txq, struct rte_mbuf

*mb);
uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t
addr,

drivers/net/mlx5/mlx5_rxtx.h +0 -1

 			       struct rte_mempool *mp);

@@ -538,35 +287,6 @@ int mlx5_dma_unmap(struct rte_pci_device *pdev,
 void *addr, uint64_t iova,
 }
 
 /**
- * Query LKey from a packet buffer for Rx. No need to flush local caches for

Rx
- * as mempool is pre-configured and static.
- *
- * @param rxq
- * Pointer to Rx queue structure.
- * @param addr
- * Address to search.
- *
- * @return
- * Searched LKey on success, UINT32_MAX on no match.
- */

drivers/net/mlx5/mlx5_rxtx.h +0 -16

-static __rte_always_inline uint32_t
-mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
-{
-	struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
-	uint32_t lkey;
-
-	/* Linear search on MR cache array. */
-	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
-				   MLX5_MR_CACHE_N, addr);
-	if (likely(lkey != UINT32_MAX))
-		return lkey;
-	/* Take slower bottom-half (Binary Search) on miss. */
-	return mlx5_rx_addr2mr_bh(rxq, addr);
-}
-
-#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)-

Quoted 1 line

buf_addr))

drivers/net/mlx5/mlx5_rxtx.h +0 -21

-
-/**
  * Query LKey from a packet buffer for Tx. If not found, add the mempool.
  *
  * @param txq

@@ -637,25 +357,6 @@ int mlx5_dma_unmap(struct rte_pci_device *pdev,
 void *addr, uint64_t iova,
 }
 
 /**
- * Convert timestamp from HW format to linear counter
- * from Packet Pacing Clock Queue CQE timestamp format.
- *
- * @param sh
- *   Pointer to the device shared context. Might be needed
- *   to convert according current device configuration.
- * @param ts
- *   Timestamp from CQE to convert.
- * @return
- *   UTC in nanoseconds
- */
-static __rte_always_inline uint64_t
-mlx5_txpp_convert_rx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t ts)
-{
-	RTE_SET_USED(sh);
-	return (ts & UINT32_MAX) + (ts >> 32) * NS_PER_S;
-}
-
-/**
  * Convert timestamp from mbuf format to linear counter
  * of Clock Queue completions (24 bits)

*

drivers/net/mlx5/mlx5_rxtx.h +0 -15

@@ -712,274 +413,4 @@ int mlx5_dma_unmap(struct rte_pci_device *pdev,
 void *addr, uint64_t iova,
 	return ci;
 }
 
-/**
- * Set timestamp in mbuf dynamic field.
- *
- * @param mbuf
- *   Structure to write into.
- * @param offset
- *   Dynamic field offset in mbuf structure.
- * @param timestamp
- *   Value to write.
- */
-static __rte_always_inline void
-mlx5_timestamp_set(struct rte_mbuf *mbuf, int offset,
-		rte_mbuf_timestamp_t timestamp)
-{
-	*RTE_MBUF_DYNFIELD(mbuf, offset, rte_mbuf_timestamp_t *) =

timestamp;

drivers/net/mlx5/mlx5_rxtx.h +0 -17

-}
-
-/**
- * Replace MPRQ buffer.
- *
- * @param rxq
- *   Pointer to Rx queue structure.
- * @param rq_idx
- *   RQ index to replace.
- */
-static __rte_always_inline void
-mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx)
-{
-	const uint32_t strd_n = 1 << rxq->strd_num_n;
-	struct mlx5_mprq_buf *rep = rxq->mprq_repl;
-	volatile struct mlx5_wqe_data_seg *wqe =
-		&((volatile struct mlx5_wqe_mprq *)rxq-

Quoted 1 line

wqes)[rq_idx].dseg;

drivers/net/mlx5/mlx5_rxtx.h +0 -12

-	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_idx];
-	void *addr;
-
-	if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) > 1) {
-		MLX5_ASSERT(rep != NULL);
-		/* Replace MPRQ buf. */
-		(*rxq->mprq_bufs)[rq_idx] = rep;
-		/* Replace WQE. */
-		addr = mlx5_mprq_buf_addr(rep, strd_n);
-		wqe->addr = rte_cpu_to_be_64((uintptr_t)addr);
-		/* If there's only one MR, no need to replace LKey in WQE. */
-		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) >

1))

drivers/net/mlx5/mlx5_rxtx.h +0 -3

-			wqe->lkey = mlx5_rx_addr2mr(rxq, (uintptr_t)addr);
-		/* Stash a mbuf for next replacement. */
-		if (likely(!rte_mempool_get(rxq->mprq_mp, (void

**)&rep)))

drivers/net/mlx5/mlx5_rxtx.h +0 -37

-			rxq->mprq_repl = rep;
-		else
-			rxq->mprq_repl = NULL;
-		/* Release the old buffer. */
-		mlx5_mprq_buf_free(buf);
-	} else if (unlikely(rxq->mprq_repl == NULL)) {
-		struct mlx5_mprq_buf *rep;
-
-		/*
-		 * Currently, the MPRQ mempool is out of buffer
-		 * and doing memcpy regardless of the size of Rx
-		 * packet. Retry allocation to get back to
-		 * normal.
-		 */
-		if (!rte_mempool_get(rxq->mprq_mp, (void **)&rep))
-			rxq->mprq_repl = rep;
-	}
-}
-
-/**
- * Attach or copy MPRQ buffer content to a packet.
- *
- * @param rxq
- *   Pointer to Rx queue structure.
- * @param pkt
- *   Pointer to a packet to fill.
- * @param len
- *   Packet length.
- * @param buf
- *   Pointer to a MPRQ buffer to take the data from.
- * @param strd_idx
- *   Stride index to start from.
- * @param strd_cnt
- *   Number of strides to consume.
- */
-static __rte_always_inline enum mlx5_rqx_code
-mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,

uint32_t len,

drivers/net/mlx5/mlx5_rxtx.h +0 -1

-		struct mlx5_mprq_buf *buf, uint16_t strd_idx, uint16_t

strd_cnt)

drivers/net/mlx5/mlx5_rxtx.h +0 -9

-{
-	const uint32_t strd_n = 1 << rxq->strd_num_n;
-	const uint16_t strd_sz = 1 << rxq->strd_sz_n;
-	const uint16_t strd_shift =
-		MLX5_MPRQ_STRIDE_SHIFT_BYTE * rxq->strd_shift_en;
-	const int32_t hdrm_overlap =
-		len + RTE_PKTMBUF_HEADROOM - strd_cnt * strd_sz;
-	const uint32_t offset = strd_idx * strd_sz + strd_shift;
-	void *addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf, strd_n),

offset);

drivers/net/mlx5/mlx5_rxtx.h +0 -12

-
-	/*
-	 * Memcpy packets to the target mbuf if:
-	 * - The size of packet is smaller than mprq_max_memcpy_len.
-	 * - Out of buffer in the Mempool for Multi-Packet RQ.
-	 * - The packet's stride overlaps a headroom and scatter is off.
-	 */
-	if (len <= rxq->mprq_max_memcpy_len ||
-	    rxq->mprq_repl == NULL ||
-	    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
-		if (likely(len <=
-			   (uint32_t)(pkt->buf_len -

RTE_PKTMBUF_HEADROOM))) {

drivers/net/mlx5/mlx5_rxtx.h +0 -7

-			rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
-				   addr, len);
-			DATA_LEN(pkt) = len;
-		} else if (rxq->strd_scatter_en) {
-			struct rte_mbuf *prev = pkt;
-			uint32_t seg_len = RTE_MIN(len, (uint32_t)
-				(pkt->buf_len -

RTE_PKTMBUF_HEADROOM));

drivers/net/mlx5/mlx5_rxtx.h +0 -16

-			uint32_t rem_len = len - seg_len;
-
-			rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
-				   addr, seg_len);
-			DATA_LEN(pkt) = seg_len;
-			while (rem_len) {
-				struct rte_mbuf *next =
-					rte_pktmbuf_alloc(rxq->mp);
-
-				if (unlikely(next == NULL))
-					return MLX5_RXQ_CODE_NOMBUF;
-				NEXT(prev) = next;
-				SET_DATA_OFF(next, 0);
-				addr = RTE_PTR_ADD(addr, seg_len);
-				seg_len = RTE_MIN(rem_len, (uint32_t)
-					(next->buf_len -

RTE_PKTMBUF_HEADROOM));

drivers/net/mlx5/mlx5_rxtx.h +0 -22

-				rte_memcpy
-					(rte_pktmbuf_mtod(next, void *),
-					 addr, seg_len);
-				DATA_LEN(next) = seg_len;
-				rem_len -= seg_len;
-				prev = next;
-				++NB_SEGS(pkt);
-			}
-		} else {
-			return MLX5_RXQ_CODE_DROPPED;
-		}
-	} else {
-		rte_iova_t buf_iova;
-		struct rte_mbuf_ext_shared_info *shinfo;
-		uint16_t buf_len = strd_cnt * strd_sz;
-		void *buf_addr;
-
-		/* Increment the refcnt of the whole chunk. */
-		__atomic_add_fetch(&buf->refcnt, 1, __ATOMIC_RELAXED);
-		MLX5_ASSERT(__atomic_load_n(&buf->refcnt,
-			    __ATOMIC_RELAXED) <= strd_n + 1);
-		buf_addr = RTE_PTR_SUB(addr,

RTE_PKTMBUF_HEADROOM);

drivers/net/mlx5/mlx5_rxtx.h +0 -110

-		/*
-		 * MLX5 device doesn't use iova but it is necessary in a
-		 * case where the Rx packet is transmitted via a
-		 * different PMD.
-		 */
-		buf_iova = rte_mempool_virt2iova(buf) +
-			   RTE_PTR_DIFF(buf_addr, buf);
-		shinfo = &buf->shinfos[strd_idx];
-		rte_mbuf_ext_refcnt_set(shinfo, 1);
-		/*
-		 * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when
-		 * attaching the stride to mbuf and more offload flags
-		 * will be added below by calling rxq_cq_to_mbuf().
-		 * Other fields will be overwritten.
-		 */
-		rte_pktmbuf_attach_extbuf(pkt, buf_addr, buf_iova,
-					  buf_len, shinfo);
-		/* Set mbuf head-room. */
-		SET_DATA_OFF(pkt, RTE_PKTMBUF_HEADROOM);
-		MLX5_ASSERT(pkt->ol_flags == EXT_ATTACHED_MBUF);
-		MLX5_ASSERT(rte_pktmbuf_tailroom(pkt) >=
-			len - (hdrm_overlap > 0 ? hdrm_overlap : 0));
-		DATA_LEN(pkt) = len;
-		/*
-		 * Copy the last fragment of a packet (up to headroom
-		 * size bytes) in case there is a stride overlap with
-		 * a next packet's headroom. Allocate a separate mbuf
-		 * to store this fragment and link it. Scatter is on.
-		 */
-		if (hdrm_overlap > 0) {
-			MLX5_ASSERT(rxq->strd_scatter_en);
-			struct rte_mbuf *seg =
-				rte_pktmbuf_alloc(rxq->mp);
-
-			if (unlikely(seg == NULL))
-				return MLX5_RXQ_CODE_NOMBUF;
-			SET_DATA_OFF(seg, 0);
-			rte_memcpy(rte_pktmbuf_mtod(seg, void *),
-				RTE_PTR_ADD(addr, len - hdrm_overlap),
-				hdrm_overlap);
-			DATA_LEN(seg) = hdrm_overlap;
-			DATA_LEN(pkt) = len - hdrm_overlap;
-			NEXT(pkt) = seg;
-			NB_SEGS(pkt) = 2;
-		}
-	}
-	return MLX5_RXQ_CODE_EXIT;
-}
-
-/**
- * Check whether Multi-Packet RQ can be enabled for the device.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   1 if supported, negative errno value if not.
- */
-static __rte_always_inline int
-mlx5_check_mprq_support(struct rte_eth_dev *dev)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	if (priv->config.mprq.enabled &&
-	    priv->rxqs_n >= priv->config.mprq.min_rxqs_num)
-		return 1;
-	return -ENOTSUP;
-}
-
-/**
- * Check whether Multi-Packet RQ is enabled for the Rx queue.
- *
- *  @param rxq
- *     Pointer to receive queue structure.
- *
- * @return
- *   0 if disabled, otherwise enabled.
- */
-static __rte_always_inline int
-mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq)
-{
-	return rxq->strd_num_n > 0;
-}
-
-/**
- * Check whether Multi-Packet RQ is enabled for the device.
- *
- * @param dev
- *   Pointer to Ethernet device.
- *
- * @return
- *   0 if disabled, otherwise enabled.
- */
-static __rte_always_inline int
-mlx5_mprq_enabled(struct rte_eth_dev *dev)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t i;
-	uint16_t n = 0;
-	uint16_t n_ibv = 0;
-
-	if (mlx5_check_mprq_support(dev) < 0)
-		return 0;
-	/* All the configured queues should be enabled. */
-	for (i = 0; i < priv->rxqs_n; ++i) {
-		struct mlx5_rxq_data *rxq = (*priv->rxqs)[i];
-		struct mlx5_rxq_ctrl *rxq_ctrl = container_of
-			(rxq, struct mlx5_rxq_ctrl, rxq);
-
-		if (rxq == NULL || rxq_ctrl->type !=

MLX5_RXQ_TYPE_STANDARD)

drivers/net/mlx5/mlx5_rxtx.h +0 -9

-			continue;
-		n_ibv++;
-		if (mlx5_rxq_mprq_enabled(rxq))
-			++n;
-	}
-	/* Multi-Packet RQ can't be partially configured. */
-	MLX5_ASSERT(n == 0 || n == n_ibv);
-	return n == n_ibv;
-}
 #endif /* RTE_PMD_MLX5_RXTX_H_ */

drivers/net/mlx5/mlx5_rxtx_vec.c +1 -0

index 028e0f6..d5af2d9 100644
@@ -19,6 +19,7 @@
 #include "mlx5.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_rxtx_vec.h"
 #include "mlx5_autoconf.h"
 

drivers/net/mlx5/mlx5_stats.c +1 -0

index a6569b2..4dbd831 100644
@@ -17,6 +17,7 @@
 #include "mlx5_defs.h"
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_malloc.h"
 
 /**

drivers/net/mlx5/mlx5_trigger.c +1 -0

index 94dd567..c88cb22 100644
@@ -16,6 +16,7 @@
 #include "mlx5.h"
 #include "mlx5_mr.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_utils.h"
 #include "rte_pmd_mlx5.h"
 

drivers/net/mlx5/mlx5_txpp.c +1 -0

index e8d632a..89e1c5d 100644
@@ -17,6 +17,7 @@
 
 #include "mlx5.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_common_os.h"
 
 static_assert(sizeof(struct mlx5_cqe_ts) == sizeof(rte_int128_t),

drivers/net/mlx5/mlx5_vlan.c +1 -0

index 64678d3..60f97f2 100644
@@ -16,6 +16,7 @@
 #include "mlx5.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_utils.h"
 #include "mlx5_devx.h"
 

drivers/net/mlx5/windows/mlx5_os.c +1 -1

index 6f39276..79eac80 100644
@@ -23,6 +23,7 @@
 #include "mlx5_common_os.h"
 #include "mlx5_utils.h"
 #include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
 #include "mlx5_autoconf.h"
 #include "mlx5_mr.h"
 #include "mlx5_flow.h"
--

1.8.3.1

-----Original Message-----
From: Michael Baum <[email protected]>
Sent: Monday, April 5, 2021 17:01
To: [email protected]
Cc: Matan Azrad <[email protected]>; Raslan Darawsheh <[email protected]>; Slava Ovsiienko <[email protected]>
Subject: [PATCH 2/6] net/mlx5: separate Rx function implementations to new file

This patch separates Rx function implementations to different source file as an optional preparation step for further consolidation of Rx burst functions.

Signed-off-by Michael Baum <[email protected]>

---
 drivers/net/mlx5/meson.build |    1 +
 drivers/net/mlx5/mlx5_rx.c   | 1203
++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rx.h   |    4 +-
 drivers/net/mlx5/mlx5_rxtx.c | 1179 +----------------------------------------
 drivers/net/mlx5/mlx5_rxtx.h |    2 +
 5 files changed, 1209 insertions(+), 1180 deletions(-)
 create mode 100644 drivers/net/mlx5/mlx5_rx.c

drivers/net/mlx5/meson.build +1 -0

index f2fafbd..0a89a27 100644
@@ -19,6 +19,7 @@ sources = files(
 	'mlx5_mac.c',
 	'mlx5_mr.c',
 	'mlx5_rss.c',
+    'mlx5_rx.c',
 	'mlx5_rxmode.c',
 	'mlx5_rxq.c',
 	'mlx5_rxtx.c',

drivers/net/mlx5/mlx5_rx.c new file +32 -0

new file mode 100644
index 0000000..e9fcb52
@@ -0,0 +1,1203 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 6WIND S.A.
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <stdint.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include <rte_mbuf.h>
+#include <rte_mempool.h>
+#include <rte_prefetch.h>
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+#include <rte_ether.h>
+#include <rte_cycles.h>
+#include <rte_flow.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_common.h>
+
+#include "mlx5_autoconf.h"
+#include "mlx5_defs.h"
+#include "mlx5.h"
+#include "mlx5_mr.h"
+#include "mlx5_utils.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_rx.h"
+
+
+static __rte_always_inline uint32_t
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe

*cqe,

drivers/net/mlx5/mlx5_rx.c new file +28 -0

+		   volatile struct mlx5_mini_cqe8 *mcqe);
+
+static __rte_always_inline int
+mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
+		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe);
+
+static __rte_always_inline uint32_t
+rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe);
+
+static __rte_always_inline void
+rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
+	       volatile struct mlx5_cqe *cqe,
+	       volatile struct mlx5_mini_cqe8 *mcqe);
+
+static inline void
+mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
+			volatile struct mlx5_cqe *__rte_restrict cqe,
+			uint32_t phcsum, uint8_t l4_type);
+
+static inline void
+mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
+		    volatile struct mlx5_cqe *__rte_restrict cqe,
+		    volatile struct mlx5_mini_cqe8 *mcqe,
+		    struct mlx5_rxq_data *rxq, uint32_t len);
+
+
+/**
+ * Internal function to compute the number of used descriptors in an RX

queue.
+ *
+ * @param rxq
+ * The Rx queue.
+ *
+ * @return
+ * The number of used Rx descriptor.
+ */

drivers/net/mlx5/mlx5_rx.c new file +22 -0

+static uint32_t
+rx_queue_count(struct mlx5_rxq_data *rxq)
+{
+	struct rxq_zip *zip = &rxq->zip;
+	volatile struct mlx5_cqe *cqe;
+	const unsigned int cqe_n = (1 << rxq->cqe_n);
+	const unsigned int sges_n = (1 << rxq->sges_n);
+	const unsigned int elts_n = (1 << rxq->elts_n);
+	const unsigned int strd_n = (1 << rxq->strd_num_n);
+	const unsigned int cqe_cnt = cqe_n - 1;
+	unsigned int cq_ci, used;
+
+	/* if we are processing a compressed cqe */
+	if (zip->ai) {
+		used = zip->cqe_cnt - zip->ai;
+		cq_ci = zip->cq_ci;
+	} else {
+		used = 0;
+		cq_ci = rxq->cq_ci;
+	}
+	cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
+	while (check_cqe(cqe, cqe_n, cq_ci) !=

MLX5_CQE_STATUS_HW_OWN) {

drivers/net/mlx5/mlx5_rx.c new file +124 -0

+		int8_t op_own;
+		unsigned int n;
+
+		op_own = cqe->op_own;
+		if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED)
+			n = rte_be_to_cpu_32(cqe->byte_cnt);
+		else
+			n = 1;
+		cq_ci += n;
+		used += n;
+		cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
+	}
+	used = RTE_MIN(used * sges_n, elts_n * strd_n);
+	return used;
+}
+
+/**
+ * DPDK callback to check the status of a Rx descriptor.
+ *
+ * @param rx_queue
+ *   The Rx queue.
+ * @param[in] offset
+ *   The index of the descriptor in the ring.
+ *
+ * @return
+ *   The status of the Rx descriptor.
+ */
+int
+mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset)
+{
+	struct mlx5_rxq_data *rxq = rx_queue;
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv);
+
+	if (dev->rx_pkt_burst == NULL ||
+	    dev->rx_pkt_burst == removed_rx_burst) {
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	if (offset >= (1 << rxq->cqe_n)) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	if (offset < rx_queue_count(rxq))
+		return RTE_ETH_RX_DESC_DONE;
+	return RTE_ETH_RX_DESC_AVAIL;
+}
+
+/**
+ * DPDK callback to get the RX queue information.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ *
+ * @param rx_queue_id
+ *   Rx queue identificator.
+ *
+ * @param qinfo
+ *   Pointer to the RX queue information structure.
+ *
+ * @return
+ *   None.
+ */
+
+void
+mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		  struct rte_eth_rxq_info *qinfo)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id];
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+		container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+
+	if (!rxq)
+		return;
+	qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
+					rxq->mprq_mp : rxq->mp;
+	qinfo->conf.rx_thresh.pthresh = 0;
+	qinfo->conf.rx_thresh.hthresh = 0;
+	qinfo->conf.rx_thresh.wthresh = 0;
+	qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh;
+	qinfo->conf.rx_drop_en = 1;
+	qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1;
+	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
+	qinfo->scattered_rx = dev->data->scattered_rx;
+	qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
+		(1 << rxq->elts_n) * (1 << rxq->strd_num_n) :
+		(1 << rxq->elts_n);
+}
+
+/**
+ * DPDK callback to get the RX packet burst mode information.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ *
+ * @param rx_queue_id
+ *   Rx queue identificatior.
+ *
+ * @param mode
+ *   Pointer to the burts mode information.
+ *
+ * @return
+ *   0 as success, -EINVAL as failure.
+ */
+int
+mlx5_rx_burst_mode_get(struct rte_eth_dev *dev,
+		       uint16_t rx_queue_id __rte_unused,
+		       struct rte_eth_burst_mode *mode)
+{
+	eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq;
+
+	rxq = (*priv->rxqs)[rx_queue_id];
+	if (!rxq) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	if (pkt_burst == mlx5_rx_burst) {
+		snprintf(mode->info, sizeof(mode->info), "%s", "Scalar");
+	} else if (pkt_burst == mlx5_rx_burst_mprq) {
+		snprintf(mode->info, sizeof(mode->info), "%s", "Multi-

Packet RQ");

drivers/net/mlx5/mlx5_rx.c new file +3 -0

+	} else if (pkt_burst == mlx5_rx_burst_vec) {
+#if defined RTE_ARCH_X86_64
+		snprintf(mode->info, sizeof(mode->info), "%s", "Vector

SSE");

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+#elif defined RTE_ARCH_ARM64
+		snprintf(mode->info, sizeof(mode->info), "%s", "Vector

Neon");

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+#elif defined RTE_ARCH_PPC_64
+		snprintf(mode->info, sizeof(mode->info), "%s", "Vector

AltiVec");

drivers/net/mlx5/mlx5_rx.c new file +6 -0

+#else
+		return -EINVAL;
+#endif
+	} else if (pkt_burst == mlx5_rx_burst_mprq_vec) {
+#if defined RTE_ARCH_X86_64
+		snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ

Vector SSE");

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+#elif defined RTE_ARCH_ARM64
+		snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ

Vector Neon");

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+#elif defined RTE_ARCH_PPC_64
+		snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ

Vector AltiVec");

drivers/net/mlx5/mlx5_rx.c new file +56 -0

+#else
+		return -EINVAL;
+#endif
+	} else {
+		return -EINVAL;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to get the number of used descriptors in a RX queue.
+ *
+ * @param dev
+ *   Pointer to the device structure.
+ *
+ * @param rx_queue_id
+ *   The Rx queue.
+ *
+ * @return
+ *   The number of used rx descriptor.
+ *   -EINVAL if the queue is invalid
+ */
+uint32_t
+mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq;
+
+	if (dev->rx_pkt_burst == NULL ||
+	    dev->rx_pkt_burst == removed_rx_burst) {
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	rxq = (*priv->rxqs)[rx_queue_id];
+	if (!rxq) {
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	return rx_queue_count(rxq);
+}
+
+/**
+ * Translate RX completion flags to packet type.
+ *
+ * @param[in] rxq
+ *   Pointer to RX queue structure.
+ * @param[in] cqe
+ *   Pointer to CQE.
+ *
+ * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
+ *
+ * @return
+ *   Packet type for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe

*cqe,

drivers/net/mlx5/mlx5_rx.c new file +48 -0

+				   volatile struct mlx5_mini_cqe8 *mcqe)
+{
+	uint8_t idx;
+	uint8_t ptype;
+	uint8_t pinfo = (cqe->pkt_info & 0x3) << 6;
+
+	/* Get l3/l4 header from mini-CQE in case L3/L4 format*/
+	if (mcqe == NULL ||
+	    rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
+		ptype = (cqe->hdr_type_etc & 0xfc00) >> 10;
+	else
+		ptype = mcqe->hdr_type >> 2;
+	/*
+	 * The index to the array should have:
+	 * bit[1:0] = l3_hdr_type
+	 * bit[4:2] = l4_hdr_type
+	 * bit[5] = ip_frag
+	 * bit[6] = tunneled
+	 * bit[7] = outer_l3_type
+	 */
+	idx = pinfo | ptype;
+	return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
+}
+
+/**
+ * Initialize Rx WQ and indexes.
+ *
+ * @param[in] rxq
+ *   Pointer to RX queue structure.
+ */
+void
+mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
+{
+	const unsigned int wqe_n = 1 << rxq->elts_n;
+	unsigned int i;
+
+	for (i = 0; (i != wqe_n); ++i) {
+		volatile struct mlx5_wqe_data_seg *scat;
+		uintptr_t addr;
+		uint32_t byte_count;
+
+		if (mlx5_rxq_mprq_enabled(rxq)) {
+			struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
+
+			scat = &((volatile struct mlx5_wqe_mprq *)
+				rxq->wqes)[i].dseg;
+			addr = (uintptr_t)mlx5_mprq_buf_addr(buf,
+							 1 << rxq-

Quoted 1 line

strd_num_n);

drivers/net/mlx5/mlx5_rx.c new file +36 -0

+			byte_count = (1 << rxq->strd_sz_n) *
+					(1 << rxq->strd_num_n);
+		} else {
+			struct rte_mbuf *buf = (*rxq->elts)[i];
+
+			scat = &((volatile struct mlx5_wqe_data_seg *)
+					rxq->wqes)[i];
+			addr = rte_pktmbuf_mtod(buf, uintptr_t);
+			byte_count = DATA_LEN(buf);
+		}
+		/* scat->addr must be able to store a pointer. */
+		MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
+		*scat = (struct mlx5_wqe_data_seg){
+			.addr = rte_cpu_to_be_64(addr),
+			.byte_count = rte_cpu_to_be_32(byte_count),
+			.lkey = mlx5_rx_addr2mr(rxq, addr),
+		};
+	}
+	rxq->consumed_strd = 0;
+	rxq->decompressed = 0;
+	rxq->rq_pi = 0;
+	rxq->zip = (struct rxq_zip){
+		.ai = 0,
+	};
+	rxq->elts_ci = mlx5_rxq_mprq_enabled(rxq) ?
+		(wqe_n >> rxq->sges_n) * (1 << rxq->strd_num_n) : 0;
+	/* Update doorbell counter. */
+	rxq->rq_ci = wqe_n >> rxq->sges_n;
+	rte_io_wmb();
+	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+}
+
+/**
+ * Handle a Rx error.
+ * The function inserts the RQ state to reset when the first error CQE is
+ * shown, then drains the CQ by the caller function loop. When the CQ is

empty,
+ * it moves the RQ state to ready and initializes the RQ.
+ * Next CQE identification and error counting are in the caller responsibility.
+ *
+ * @param[in] rxq
+ * Pointer to RX queue structure.
+ * @param[in] vec
+ * 1 when called from vectorized Rx burst, need to prepare mbufs for the
RQ.
+ * 0 when called from non-vectorized Rx burst.
+ *
+ * @return
+ * -1 in case of recovery error, otherwise the CQE status.
+ */

drivers/net/mlx5/mlx5_rx.c new file +27 -0

+int
+mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec)
+{
+	const uint16_t cqe_n = 1 << rxq->cqe_n;
+	const uint16_t cqe_mask = cqe_n - 1;
+	const uint16_t wqe_n = 1 << rxq->elts_n;
+	const uint16_t strd_n = 1 << rxq->strd_num_n;
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	union {
+		volatile struct mlx5_cqe *cqe;
+		volatile struct mlx5_err_cqe *err_cqe;
+	} u = {
+		.cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask],
+	};
+	struct mlx5_mp_arg_queue_state_modify sm;
+	int ret;
+
+	switch (rxq->err_state) {
+	case MLX5_RXQ_ERR_STATE_NO_ERROR:
+		rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET;
+		/* Fall-through */
+	case MLX5_RXQ_ERR_STATE_NEED_RESET:
+		sm.is_wq = 1;
+		sm.queue_id = rxq->idx;
+		sm.state = IBV_WQS_RESET;
+		if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv),

&sm))

drivers/net/mlx5/mlx5_rx.c new file +12 -0

+			return -1;
+		if (rxq_ctrl->dump_file_n <
+		    rxq_ctrl->priv->config.max_dump_files_num) {
+			MKSTR(err_str, "Unexpected CQE error syndrome "
+			      "0x%02x CQN = %u RQN = %u wqe_counter = %u"
+			      " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome,
+			      rxq->cqn, rxq_ctrl->wqn,
+			      rte_be_to_cpu_16(u.err_cqe->wqe_counter),
+			      rxq->rq_ci << rxq->sges_n, rxq->cq_ci);
+			MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u",
+			      rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc());
+			mlx5_dump_debug_information(name, NULL,

err_str, 0);

drivers/net/mlx5/mlx5_rx.c new file +1 -0

+			mlx5_dump_debug_information(name, "MLX5 Error

CQ:",

drivers/net/mlx5/mlx5_rx.c new file +4 -0

+						    (const void *)((uintptr_t)
+								    rxq->cqes),
+						    sizeof(*u.cqe) * cqe_n);
+			mlx5_dump_debug_information(name, "MLX5 Error

RQ:",

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+						    (const void *)((uintptr_t)
+								    rxq-

Quoted 1 line

wqes),

drivers/net/mlx5/mlx5_rx.c new file +13 -0

+						    16 * wqe_n);
+			rxq_ctrl->dump_file_n++;
+		}
+		rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY;
+		/* Fall-through */
+	case MLX5_RXQ_ERR_STATE_NEED_READY:
+		ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci);
+		if (ret == MLX5_CQE_STATUS_HW_OWN) {
+			rte_io_wmb();
+			*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+			rte_io_wmb();
+			/*
+			 * The RQ consumer index must be zeroed while

moving

drivers/net/mlx5/mlx5_rx.c new file +8 -0

+			 * from RESET state to RDY state.
+			 */
+			*rxq->rq_db = rte_cpu_to_be_32(0);
+			rte_io_wmb();
+			sm.is_wq = 1;
+			sm.queue_id = rxq->idx;
+			sm.state = IBV_WQS_RDY;
+			if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl-

Quoted 1 line

priv),

drivers/net/mlx5/mlx5_rx.c new file +27 -0

+						    &sm))
+				return -1;
+			if (vec) {
+				const uint32_t elts_n =
+					mlx5_rxq_mprq_enabled(rxq) ?
+					wqe_n * strd_n : wqe_n;
+				const uint32_t e_mask = elts_n - 1;
+				uint32_t elts_ci =
+					mlx5_rxq_mprq_enabled(rxq) ?
+					rxq->elts_ci : rxq->rq_ci;
+				uint32_t elt_idx;
+				struct rte_mbuf **elt;
+				int i;
+				unsigned int n = elts_n - (elts_ci -
+							  rxq->rq_pi);
+
+				for (i = 0; i < (int)n; ++i) {
+					elt_idx = (elts_ci + i) & e_mask;
+					elt = &(*rxq->elts)[elt_idx];
+					*elt = rte_mbuf_raw_alloc(rxq->mp);
+					if (!*elt) {
+						for (i--; i >= 0; --i) {
+							elt_idx = (elts_ci +
+								   i) & elts_n;
+							elt = &(*rxq->elts)
+								[elt_idx];
+

rte_pktmbuf_free_seg

drivers/net/mlx5/mlx5_rx.c new file +10 -0

+								(*elt);
+						}
+						return -1;
+					}
+				}
+				for (i = 0; i < (int)elts_n; ++i) {
+					elt = &(*rxq->elts)[i];
+					DATA_LEN(*elt) =
+						(uint16_t)((*elt)->buf_len -
+

	rte_pktmbuf_headroom(*elt));

drivers/net/mlx5/mlx5_rx.c new file +6 -0

+				}
+				/* Padding with a fake mbuf for vec Rx. */
+				for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP;
++i)
+					(*rxq->elts)[elts_n + i] =
+								&rxq-

Quoted 1 line

fake_mbuf;

drivers/net/mlx5/mlx5_rx.c new file +3 -0

+			}
+			mlx5_rxq_initialize(rxq);
+			rxq->err_state =

MLX5_RXQ_ERR_STATE_NO_ERROR;

drivers/net/mlx5/mlx5_rx.c new file +41 -0

+		}
+		return ret;
+	default:
+		return -1;
+	}
+}
+
+/**
+ * Get size of the next packet for a given CQE. For compressed CQEs, the
+ * consumer index is updated only once all packets of the current one have
+ * been processed.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ * @param cqe
+ *   CQE to process.
+ * @param[out] mcqe
+ *   Store pointer to mini-CQE if compressed. Otherwise, the pointer is not
+ *   written.
+ *
+ * @return
+ *   0 in case of empty CQE, otherwise the packet size in bytes.
+ */
+static inline int
+mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
+		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe)
+{
+	struct rxq_zip *zip = &rxq->zip;
+	uint16_t cqe_n = cqe_cnt + 1;
+	int len;
+	uint16_t idx, end;
+
+	do {
+		len = 0;
+		/* Process compressed data in the CQE and mini arrays. */
+		if (zip->ai) {
+			volatile struct mlx5_mini_cqe8 (*mc)[8] =
+				(volatile struct mlx5_mini_cqe8 (*)[8])
+				(uintptr_t)(&(*rxq->cqes)[zip->ca &
+							  cqe_cnt].pkt_info);
+			len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt

&

drivers/net/mlx5/mlx5_rx.c new file +8 -0

+					       rxq->byte_mask);
+			*mcqe = &(*mc)[zip->ai & 7];
+			if ((++zip->ai & 7) == 0) {
+				/* Invalidate consumed CQEs */
+				idx = zip->ca;
+				end = zip->na;
+				while (idx != end) {
+					(*rxq->cqes)[idx & cqe_cnt].op_own

=

drivers/net/mlx5/mlx5_rx.c new file +5 -0

+						MLX5_CQE_INVALIDATE;
+					++idx;
+				}
+				/*
+				 * Increment consumer index to skip the

number

drivers/net/mlx5/mlx5_rx.c new file +1 -0

+				 * of CQEs consumed. Hardware leaves holes

in

drivers/net/mlx5/mlx5_rx.c new file +12 -0

+				 * the CQ ring for software use.
+				 */
+				zip->ca = zip->na;
+				zip->na += 8;
+			}
+			if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) {
+				/* Invalidate the rest */
+				idx = zip->ca;
+				end = zip->cq_ci;
+
+				while (idx != end) {
+					(*rxq->cqes)[idx & cqe_cnt].op_own

=

drivers/net/mlx5/mlx5_rx.c new file +21 -0

+						MLX5_CQE_INVALIDATE;
+					++idx;
+				}
+				rxq->cq_ci = zip->cq_ci;
+				zip->ai = 0;
+			}
+		/*
+		 * No compressed data, get next CQE and verify if it is
+		 * compressed.
+		 */
+		} else {
+			int ret;
+			int8_t op_own;
+			uint32_t cq_ci;
+
+			ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
+			if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+				if (unlikely(ret == MLX5_CQE_STATUS_ERR ||
+					     rxq->err_state)) {
+					ret = mlx5_rx_err_handle(rxq, 0);
+					if (ret ==

MLX5_CQE_STATUS_HW_OWN ||

drivers/net/mlx5/mlx5_rx.c new file +15 -0

+					    ret == -1)
+						return 0;
+				} else {
+					return 0;
+				}
+			}
+			/*
+			 * Introduce the local variable to have queue cq_ci
+			 * index in queue structure always consistent with
+			 * actual CQE boundary (not pointing to the middle
+			 * of compressed CQE session).
+			 */
+			cq_ci = rxq->cq_ci + 1;
+			op_own = cqe->op_own;
+			if (MLX5_CQE_FORMAT(op_own) ==

MLX5_COMPRESSED) {

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+				volatile struct mlx5_mini_cqe8 (*mc)[8] =
+					(volatile struct mlx5_mini_cqe8

(*)[8])

drivers/net/mlx5/mlx5_rx.c new file +5 -0

+					(uintptr_t)(&(*rxq->cqes)
+						[cq_ci & cqe_cnt].pkt_info);
+
+				/* Fix endianness. */
+				zip->cqe_cnt = rte_be_to_cpu_32(cqe-

Quoted 1 line

byte_cnt);

drivers/net/mlx5/mlx5_rx.c new file +5 -0

+				/*
+				 * Current mini array position is the one
+				 * returned by check_cqe64().
+				 *
+				 * If completion comprises several mini

arrays,

drivers/net/mlx5/mlx5_rx.c new file +7 -0

+				 * as a special case the second one is located
+				 * 7 CQEs after the initial CQE instead of 8
+				 * for subsequent ones.
+				 */
+				zip->ca = cq_ci;
+				zip->na = zip->ca + 7;
+				/* Compute the next non compressed CQE.

*/

drivers/net/mlx5/mlx5_rx.c new file +81 -0

+				zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
+				/* Get packet size to return. */
+				len = rte_be_to_cpu_32((*mc)[0].byte_cnt &
+						       rxq->byte_mask);
+				*mcqe = &(*mc)[0];
+				zip->ai = 1;
+				/* Prefetch all to be invalidated */
+				idx = zip->ca;
+				end = zip->cq_ci;
+				while (idx != end) {
+					rte_prefetch0(&(*rxq->cqes)[(idx) &
+								    cqe_cnt]);
+					++idx;
+				}
+			} else {
+				rxq->cq_ci = cq_ci;
+				len = rte_be_to_cpu_32(cqe->byte_cnt);
+			}
+		}
+		if (unlikely(rxq->err_state)) {
+			cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
+			++rxq->stats.idropped;
+		} else {
+			return len;
+		}
+	} while (1);
+}
+
+/**
+ * Translate RX completion flags to offload flags.
+ *
+ * @param[in] cqe
+ *   Pointer to CQE.
+ *
+ * @return
+ *   Offload flags (ol_flags) for struct rte_mbuf.
+ */
+static inline uint32_t
+rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe)
+{
+	uint32_t ol_flags = 0;
+	uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc);
+
+	ol_flags =
+		TRANSPOSE(flags,
+			  MLX5_CQE_RX_L3_HDR_VALID,
+			  PKT_RX_IP_CKSUM_GOOD) |
+		TRANSPOSE(flags,
+			  MLX5_CQE_RX_L4_HDR_VALID,
+			  PKT_RX_L4_CKSUM_GOOD);
+	return ol_flags;
+}
+
+/**
+ * Fill in mbuf fields from RX completion flags.
+ * Note that pkt->ol_flags should be initialized outside of this function.
+ *
+ * @param rxq
+ *   Pointer to RX queue.
+ * @param pkt
+ *   mbuf to fill.
+ * @param cqe
+ *   CQE to process.
+ * @param rss_hash_res
+ *   Packet RSS Hash result.
+ */
+static inline void
+rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
+	       volatile struct mlx5_cqe *cqe,
+	       volatile struct mlx5_mini_cqe8 *mcqe)
+{
+	/* Update packet information. */
+	pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe, mcqe);
+
+	if (rxq->rss_hash) {
+		uint32_t rss_hash_res = 0;
+
+		/* If compressed, take hash result from mini-CQE. */
+		if (mcqe == NULL ||
+		    rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_HASH)
+			rss_hash_res = rte_be_to_cpu_32(cqe-

Quoted 1 line

rx_hash_res);

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+		else
+			rss_hash_res = rte_be_to_cpu_32(mcqe-

Quoted 1 line

rx_hash_result);

drivers/net/mlx5/mlx5_rx.c new file +11 -0

+		if (rss_hash_res) {
+			pkt->hash.rss = rss_hash_res;
+			pkt->ol_flags |= PKT_RX_RSS_HASH;
+		}
+	}
+	if (rxq->mark) {
+		uint32_t mark = 0;
+
+		/* If compressed, take flow tag from mini-CQE. */
+		if (mcqe == NULL ||
+		    rxq->mcqe_format !=

MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)

drivers/net/mlx5/mlx5_rx.c new file +7 -0

+			mark = cqe->sop_drop_qpn;
+		else
+			mark = ((mcqe->byte_cnt_flow & 0xff) << 8) |
+				(mcqe->flow_tag_high << 16);
+		if (MLX5_FLOW_MARK_IS_VALID(mark)) {
+			pkt->ol_flags |= PKT_RX_FDIR;
+			if (mark !=

RTE_BE32(MLX5_FLOW_MARK_DEFAULT)) {

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+				pkt->ol_flags |= PKT_RX_FDIR_ID;
+				pkt->hash.fdir.hi =

mlx5_flow_mark_get(mark);

drivers/net/mlx5/mlx5_rx.c new file +20 -0

+			}
+		}
+	}
+	if (rxq->dynf_meta) {
+		uint32_t meta = cqe->flow_table_metadata &
+				rxq->flow_meta_port_mask;
+
+		if (meta) {
+			pkt->ol_flags |= rxq->flow_meta_mask;
+			*RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset,
+						uint32_t *) = meta;
+		}
+	}
+	if (rxq->csum)
+		pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);
+	if (rxq->vlan_strip) {
+		bool vlan_strip;
+
+		if (mcqe == NULL ||
+		    rxq->mcqe_format !=

MLX5_CQE_RESP_FORMAT_L34H_STRIDX)

drivers/net/mlx5/mlx5_rx.c new file +7 -0

+			vlan_strip = cqe->hdr_type_etc &
+				     RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
+		else
+			vlan_strip = mcqe->hdr_type &
+				     RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
+		if (vlan_strip) {
+			pkt->ol_flags |= PKT_RX_VLAN |

PKT_RX_VLAN_STRIPPED;

drivers/net/mlx5/mlx5_rx.c new file +45 -0

+			pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
+		}
+	}
+	if (rxq->hw_timestamp) {
+		uint64_t ts = rte_be_to_cpu_64(cqe->timestamp);
+
+		if (rxq->rt_timestamp)
+			ts = mlx5_txpp_convert_rx_ts(rxq->sh, ts);
+		mlx5_timestamp_set(pkt, rxq->timestamp_offset, ts);
+		pkt->ol_flags |= rxq->timestamp_rx_flag;
+	}
+}
+
+/**
+ * DPDK callback for RX.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+{
+	struct mlx5_rxq_data *rxq = dpdk_rxq;
+	const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1;
+	const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1;
+	const unsigned int sges_n = rxq->sges_n;
+	struct rte_mbuf *pkt = NULL;
+	struct rte_mbuf *seg = NULL;
+	volatile struct mlx5_cqe *cqe =
+		&(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
+	unsigned int i = 0;
+	unsigned int rq_ci = rxq->rq_ci << sges_n;
+	int len = 0; /* keep its value across iterations. */
+
+	while (pkts_n) {
+		unsigned int idx = rq_ci & wqe_cnt;
+		volatile struct mlx5_wqe_data_seg *wqe =
+			&((volatile struct mlx5_wqe_data_seg *)rxq-

Quoted 1 line

wqes)[idx];

drivers/net/mlx5/mlx5_rx.c new file +50 -0

+		struct rte_mbuf *rep = (*rxq->elts)[idx];
+		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
+
+		if (pkt)
+			NEXT(seg) = rep;
+		seg = rep;
+		rte_prefetch0(seg);
+		rte_prefetch0(cqe);
+		rte_prefetch0(wqe);
+		/* Allocate the buf from the same pool. */
+		rep = rte_mbuf_raw_alloc(seg->pool);
+		if (unlikely(rep == NULL)) {
+			++rxq->stats.rx_nombuf;
+			if (!pkt) {
+				/*
+				 * no buffers before we even started,
+				 * bail out silently.
+				 */
+				break;
+			}
+			while (pkt != seg) {
+				MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
+				rep = NEXT(pkt);
+				NEXT(pkt) = NULL;
+				NB_SEGS(pkt) = 1;
+				rte_mbuf_raw_free(pkt);
+				pkt = rep;
+			}
+			rq_ci >>= sges_n;
+			++rq_ci;
+			rq_ci <<= sges_n;
+			break;
+		}
+		if (!pkt) {
+			cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
+			len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe);
+			if (!len) {
+				rte_mbuf_raw_free(rep);
+				break;
+			}
+			pkt = seg;
+			MLX5_ASSERT(len >= (rxq->crc_present << 2));
+			pkt->ol_flags &= EXT_ATTACHED_MBUF;
+			rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
+			if (rxq->crc_present)
+				len -= RTE_ETHER_CRC_LEN;
+			PKT_LEN(pkt) = len;
+			if (cqe->lro_num_seg > 1) {
+				mlx5_lro_update_hdr
+					(rte_pktmbuf_mtod(pkt, uint8_t *),

cqe,

drivers/net/mlx5/mlx5_rx.c new file +16 -0

+					 mcqe, rxq, len);
+				pkt->ol_flags |= PKT_RX_LRO;
+				pkt->tso_segsz = len / cqe->lro_num_seg;
+			}
+		}
+		DATA_LEN(rep) = DATA_LEN(seg);
+		PKT_LEN(rep) = PKT_LEN(seg);
+		SET_DATA_OFF(rep, DATA_OFF(seg));
+		PORT(rep) = PORT(seg);
+		(*rxq->elts)[idx] = rep;
+		/*
+		 * Fill NIC descriptor with the new buffer. The lkey and size
+		 * of the buffers are already known, only the buffer address
+		 * changes.
+		 */
+		wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep,

uintptr_t));

drivers/net/mlx5/mlx5_rx.c new file +2 -0

+		/* If there's only one MR, no need to replace LKey in WQE. */
+		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) >

1))

drivers/net/mlx5/mlx5_rx.c new file +56 -0

+			wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
+		if (len > DATA_LEN(seg)) {
+			len -= DATA_LEN(seg);
+			++NB_SEGS(pkt);
+			++rq_ci;
+			continue;
+		}
+		DATA_LEN(seg) = len;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment bytes counter. */
+		rxq->stats.ibytes += PKT_LEN(pkt);
+#endif
+		/* Return packet. */
+		*(pkts++) = pkt;
+		pkt = NULL;
+		--pkts_n;
+		++i;
+		/* Align consumer index to the next stride. */
+		rq_ci >>= sges_n;
+		++rq_ci;
+		rq_ci <<= sges_n;
+	}
+	if (unlikely(i == 0 && ((rq_ci >> sges_n) == rxq->rq_ci)))
+		return 0;
+	/* Update the consumer index. */
+	rxq->rq_ci = rq_ci >> sges_n;
+	rte_io_wmb();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	rte_io_wmb();
+	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment packets counter. */
+	rxq->stats.ipackets += i;
+#endif
+	return i;
+}
+
+/**
+ * Update LRO packet TCP header.
+ * The HW LRO feature doesn't update the TCP header after coalescing the
+ * TCP segments but supplies information in CQE to fill it by SW.
+ *
+ * @param tcp
+ *   Pointer to the TCP header.
+ * @param cqe
+ *   Pointer to the completion entry.
+ * @param phcsum
+ *   The L3 pseudo-header checksum.
+ */
+static inline void
+mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
+			volatile struct mlx5_cqe *__rte_restrict cqe,
+			uint32_t phcsum, uint8_t l4_type)
+{
+	/*
+	 * The HW calculates only the TCP payload checksum, need to

complete

drivers/net/mlx5/mlx5_rx.c new file +24 -0

+	 * the TCP header checksum and the L3 pseudo-header checksum.
+	 */
+	uint32_t csum = phcsum + cqe->csum;
+
+	if (l4_type == MLX5_L4_HDR_TYPE_TCP_EMPTY_ACK ||
+	    l4_type == MLX5_L4_HDR_TYPE_TCP_WITH_ACL) {
+		tcp->tcp_flags |= RTE_TCP_ACK_FLAG;
+		tcp->recv_ack = cqe->lro_ack_seq_num;
+		tcp->rx_win = cqe->lro_tcp_win;
+	}
+	if (cqe->lro_tcppsh_abort_dupack & MLX5_CQE_LRO_PUSH_MASK)
+		tcp->tcp_flags |= RTE_TCP_PSH_FLAG;
+	tcp->cksum = 0;
+	csum += rte_raw_cksum(tcp, (tcp->data_off >> 4) * 4);
+	csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff);
+	csum = (~csum) & 0xffff;
+	if (csum == 0)
+		csum = 0xffff;
+	tcp->cksum = csum;
+}
+
+/**
+ * Update LRO packet headers.
+ * The HW LRO feature doesn't update the L3/TCP headers after coalescing

the
+ * TCP segments but supply information in CQE to fill it by SW.
+ *
+ * @param padd
+ * The packet address.
+ * @param cqe
+ * Pointer to the completion entry.
+ * @param len
+ * The packet length.
+ */

drivers/net/mlx5/mlx5_rx.c new file +29 -0

+static inline void
+mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
+		    volatile struct mlx5_cqe *__rte_restrict cqe,
+		    volatile struct mlx5_mini_cqe8 *mcqe,
+		    struct mlx5_rxq_data *rxq, uint32_t len)
+{
+	union {
+		struct rte_ether_hdr *eth;
+		struct rte_vlan_hdr *vlan;
+		struct rte_ipv4_hdr *ipv4;
+		struct rte_ipv6_hdr *ipv6;
+		struct rte_tcp_hdr *tcp;
+		uint8_t *hdr;
+	} h = {
+		.hdr = padd,
+	};
+	uint16_t proto = h.eth->ether_type;
+	uint32_t phcsum;
+	uint8_t l4_type;
+
+	h.eth++;
+	while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
+	       proto == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
+		proto = h.vlan->eth_proto;
+		h.vlan++;
+	}
+	if (proto == RTE_BE16(RTE_ETHER_TYPE_IPV4)) {
+		h.ipv4->time_to_live = cqe->lro_min_ttl;
+		h.ipv4->total_length = rte_cpu_to_be_16(len - (h.hdr -

padd));

drivers/net/mlx5/mlx5_rx.c new file +15 -1

+		h.ipv4->hdr_checksum = 0;
+		h.ipv4->hdr_checksum = rte_ipv4_cksum(h.ipv4);
+		phcsum = rte_ipv4_phdr_cksum(h.ipv4, 0);
+		h.ipv4++;
+	} else {
+		h.ipv6->hop_limits = cqe->lro_min_ttl;
+		h.ipv6->payload_len = rte_cpu_to_be_16(len - (h.hdr - padd)
-
+						       sizeof(*h.ipv6));
+		phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0);
+		h.ipv6++;
+	}
+	if (mcqe == NULL ||
+	    rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
+		l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) &
+			   MLX5_CQE_L4_TYPE_MASK) >>

MLX5_CQE_L4_TYPE_SHIFT;

drivers/net/mlx5/mlx5_rx.c new file +3 -0

+	else
+		l4_type = (rte_be_to_cpu_16(mcqe->hdr_type) &
+			   MLX5_CQE_L4_TYPE_MASK) >>

MLX5_CQE_L4_TYPE_SHIFT;

drivers/net/mlx5/mlx5_rx.c new file +38 -0

+	mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type);
+}
+
+void
+mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
+{
+	struct mlx5_mprq_buf *buf = opaque;
+
+	if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) {
+		rte_mempool_put(buf->mp, buf);
+	} else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1,
+					       __ATOMIC_RELAXED) == 0)) {
+		__atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED);
+		rte_mempool_put(buf->mp, buf);
+	}
+}
+
+void
+mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf)
+{
+	mlx5_mprq_buf_free_cb(NULL, buf);
+}
+
+/**
+ * DPDK callback for RX with Multi-Packet RQ support.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t

pkts_n)

drivers/net/mlx5/mlx5_rx.c new file +7 -0

+{
+	struct mlx5_rxq_data *rxq = dpdk_rxq;
+	const uint32_t strd_n = 1 << rxq->strd_num_n;
+	const uint32_t strd_sz = 1 << rxq->strd_sz_n;
+	const uint32_t cq_mask = (1 << rxq->cqe_n) - 1;
+	const uint32_t wq_mask = (1 << rxq->elts_n) - 1;
+	volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci &

cq_mask];

drivers/net/mlx5/mlx5_rx.c new file +29 -0

+	unsigned int i = 0;
+	uint32_t rq_ci = rxq->rq_ci;
+	uint16_t consumed_strd = rxq->consumed_strd;
+	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
+
+	while (i < pkts_n) {
+		struct rte_mbuf *pkt;
+		int ret;
+		uint32_t len;
+		uint16_t strd_cnt;
+		uint16_t strd_idx;
+		uint32_t byte_cnt;
+		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
+		enum mlx5_rqx_code rxq_code;
+
+		if (consumed_strd == strd_n) {
+			/* Replace WQE if the buffer is still in use. */
+			mprq_buf_replace(rxq, rq_ci & wq_mask);
+			/* Advance to the next WQE. */
+			consumed_strd = 0;
+			++rq_ci;
+			buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
+		}
+		cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
+		ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe);
+		if (!ret)
+			break;
+		byte_cnt = ret;
+		len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >>

MLX5_MPRQ_LEN_SHIFT;

drivers/net/mlx5/mlx5_rx.c new file +5 -0

+		MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
+		if (rxq->crc_present)
+			len -= RTE_ETHER_CRC_LEN;
+		if (mcqe &&
+		    rxq->mcqe_format ==

MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)

drivers/net/mlx5/mlx5_rx.c new file +3 -0

+			strd_cnt = (len / strd_sz) + !!(len % strd_sz);
+		else
+			strd_cnt = (byte_cnt &

MLX5_MPRQ_STRIDE_NUM_MASK) >>

drivers/net/mlx5/mlx5_rx.c new file +10 -0

+				   MLX5_MPRQ_STRIDE_NUM_SHIFT;
+		MLX5_ASSERT(strd_cnt);
+		consumed_strd += strd_cnt;
+		if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
+			continue;
+		strd_idx = rte_be_to_cpu_16(mcqe == NULL ?
+					cqe->wqe_counter :
+					mcqe->stride_idx);
+		MLX5_ASSERT(strd_idx < strd_n);
+		MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci)

&

drivers/net/mlx5/mlx5_rx.c new file +7 -0

+			    wq_mask));
+		pkt = rte_pktmbuf_alloc(rxq->mp);
+		if (unlikely(pkt == NULL)) {
+			++rxq->stats.rx_nombuf;
+			break;
+		}
+		len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >>

MLX5_MPRQ_LEN_SHIFT;

drivers/net/mlx5/mlx5_rx.c new file +19 -0

+		MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
+		if (rxq->crc_present)
+			len -= RTE_ETHER_CRC_LEN;
+		rxq_code = mprq_buf_to_pkt(rxq, pkt, len, buf,
+					   strd_idx, strd_cnt);
+		if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) {
+			rte_pktmbuf_free_seg(pkt);
+			if (rxq_code == MLX5_RXQ_CODE_DROPPED) {
+				++rxq->stats.idropped;
+				continue;
+			}
+			if (rxq_code == MLX5_RXQ_CODE_NOMBUF) {
+				++rxq->stats.rx_nombuf;
+				break;
+			}
+		}
+		rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
+		if (cqe->lro_num_seg > 1) {
+			mlx5_lro_update_hdr(rte_pktmbuf_mtod(pkt,

uint8_t *),

drivers/net/mlx5/mlx5_rx.c new file +57 -0

+					    cqe, mcqe, rxq, len);
+			pkt->ol_flags |= PKT_RX_LRO;
+			pkt->tso_segsz = len / cqe->lro_num_seg;
+		}
+		PKT_LEN(pkt) = len;
+		PORT(pkt) = rxq->port_id;
+#ifdef MLX5_PMD_SOFT_COUNTERS
+		/* Increment bytes counter. */
+		rxq->stats.ibytes += PKT_LEN(pkt);
+#endif
+		/* Return packet. */
+		*(pkts++) = pkt;
+		++i;
+	}
+	/* Update the consumer indexes. */
+	rxq->consumed_strd = consumed_strd;
+	rte_io_wmb();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	if (rq_ci != rxq->rq_ci) {
+		rxq->rq_ci = rq_ci;
+		rte_io_wmb();
+		*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
+	}
+#ifdef MLX5_PMD_SOFT_COUNTERS
+	/* Increment packets counter. */
+	rxq->stats.ipackets += i;
+#endif
+	return i;
+}
+
+/**
+ * Dummy DPDK callback for RX.
+ *
+ * This function is used to temporarily replace the real callback during
+ * unsafe control operations on the queue, or in case of error.
+ *
+ * @param dpdk_rxq
+ *   Generic pointer to RX queue structure.
+ * @param[out] pkts
+ *   Array to store received packets.
+ * @param pkts_n
+ *   Maximum number of packets in array.
+ *
+ * @return
+ *   Number of packets successfully received (<= pkts_n).
+ */
+uint16_t
+removed_rx_burst(void *dpdk_rxq __rte_unused,
+		 struct rte_mbuf **pkts __rte_unused,
+		 uint16_t pkts_n __rte_unused)
+{
+	rte_mb();
+	return 0;
+}
+
+/*
+ * Vectorized Rx routines are not compiled in when required vector

instructions
+ * are not supported on a target architecture.
+ * The following null stubs are needed for linkage when those are not included
+ * outside of this file (e.g. mlx5_rxtx_vec_sse.c for x86).
+ */

drivers/net/mlx5/mlx5_rx.c new file +29 -0

+
+__rte_weak uint16_t
+mlx5_rx_burst_vec(void *dpdk_rxq __rte_unused,
+		  struct rte_mbuf **pkts __rte_unused,
+		  uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+__rte_weak uint16_t
+mlx5_rx_burst_mprq_vec(void *dpdk_rxq __rte_unused,
+		       struct rte_mbuf **pkts __rte_unused,
+		       uint16_t pkts_n __rte_unused)
+{
+	return 0;
+}
+
+__rte_weak int
+mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
+{
+	return -ENOTSUP;
+}
+
+__rte_weak int
+mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
+{
+	return -ENOTSUP;
+}
+

drivers/net/mlx5/mlx5_rx.h +1 -1

index 83b1f38..d5a2de8 100644
@@ -246,7 +246,7 @@ int mlx5_hrxq_modify(struct rte_eth_dev *dev,
 uint32_t hxrq_idx,
 		     uint64_t hash_fields,
 		     const uint16_t *queues, uint32_t queues_n);
 
-/* mlx5_rxtx.c */
+/* mlx5_rx.c */
 
 uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t

pkts_n);
void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);

drivers/net/mlx5/mlx5_rx.h +1 -1

@@ -264,7 +264,7 @@ void mlx5_rxq_info_get(struct rte_eth_dev *dev,
 uint16_t queue_id,
 int mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
 rx_queue_id,
 			   struct rte_eth_burst_mode *mode);
 
-/* Vectorized version of mlx5_rxtx.c */
+/* Vectorized version of mlx5_rx.c */
 int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);

int mlx5_check_vec_rx_support(struct rte_eth_dev *dev); uint16_t mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts,

drivers/net/mlx5/mlx5_rxtx.c +0 -31

index d004e1e..c7f2605 100644
@@ -130,37 +130,6 @@ enum mlx5_txcmp_code {
 static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE,
 		"invalid WQE size");
 
-static __rte_always_inline uint32_t
-rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe
 *cqe,
-				   volatile struct mlx5_mini_cqe8 *mcqe);
-
-static __rte_always_inline int
-mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
-		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe);
-
-static __rte_always_inline uint32_t
-rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe);
-
-static __rte_always_inline void
-rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
-	       volatile struct mlx5_cqe *cqe,
-	       volatile struct mlx5_mini_cqe8 *mcqe);
-
-static int
-mlx5_queue_state_modify(struct rte_eth_dev *dev,
-			struct mlx5_mp_arg_queue_state_modify *sm);
-
-static inline void
-mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
-			volatile struct mlx5_cqe *__rte_restrict cqe,
-			uint32_t phcsum, uint8_t l4_type);
-
-static inline void
-mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
-		    volatile struct mlx5_cqe *__rte_restrict cqe,
-			volatile struct mlx5_mini_cqe8 *mcqe,
-		    struct mlx5_rxq_data *rxq, uint32_t len);
-
 uint32_t mlx5_ptype_table[] __rte_cache_aligned = {
 	[0xff] = RTE_PTYPE_ALL_MASK, /* Last entry for errored packet. */

};

drivers/net/mlx5/mlx5_rxtx.c +0 -2

@@ -499,220 +468,6 @@ enum mlx5_txcmp_code {
 	return mlx5_cksum_table[idx];
 }
 
-/**
- * Internal function to compute the number of used descriptors in an RX

queue
- *
- * @param rxq
- * The Rx queue.
- *
- * @return
- * The number of used rx descriptor.
- */

drivers/net/mlx5/mlx5_rxtx.c +0 -22

-static uint32_t
-rx_queue_count(struct mlx5_rxq_data *rxq)
-{
-	struct rxq_zip *zip = &rxq->zip;
-	volatile struct mlx5_cqe *cqe;
-	const unsigned int cqe_n = (1 << rxq->cqe_n);
-	const unsigned int sges_n = (1 << rxq->sges_n);
-	const unsigned int elts_n = (1 << rxq->elts_n);
-	const unsigned int strd_n = (1 << rxq->strd_num_n);
-	const unsigned int cqe_cnt = cqe_n - 1;
-	unsigned int cq_ci, used;
-
-	/* if we are processing a compressed cqe */
-	if (zip->ai) {
-		used = zip->cqe_cnt - zip->ai;
-		cq_ci = zip->cq_ci;
-	} else {
-		used = 0;
-		cq_ci = rxq->cq_ci;
-	}
-	cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
-	while (check_cqe(cqe, cqe_n, cq_ci) !=

MLX5_CQE_STATUS_HW_OWN) {

drivers/net/mlx5/mlx5_rxtx.c +0 -125

-		int8_t op_own;
-		unsigned int n;
-
-		op_own = cqe->op_own;
-		if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED)
-			n = rte_be_to_cpu_32(cqe->byte_cnt);
-		else
-			n = 1;
-		cq_ci += n;
-		used += n;
-		cqe = &(*rxq->cqes)[cq_ci & cqe_cnt];
-	}
-	used = RTE_MIN(used * sges_n, elts_n * strd_n);
-	return used;
-}
-
-/**
- * DPDK callback to check the status of a rx descriptor.
- *
- * @param rx_queue
- *   The Rx queue.
- * @param[in] offset
- *   The index of the descriptor in the ring.
- *
- * @return
- *   The status of the tx descriptor.
- */
-int
-mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset)
-{
-	struct mlx5_rxq_data *rxq = rx_queue;
-	struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
-	struct rte_eth_dev *dev = ETH_DEV(rxq_ctrl->priv);
-
-	if (dev->rx_pkt_burst == NULL ||
-	    dev->rx_pkt_burst == removed_rx_burst) {
-		rte_errno = ENOTSUP;
-		return -rte_errno;
-	}
-	if (offset >= (1 << rxq->cqe_n)) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	if (offset < rx_queue_count(rxq))
-		return RTE_ETH_RX_DESC_DONE;
-	return RTE_ETH_RX_DESC_AVAIL;
-}
-
-/**
- * DPDK callback to get the RX queue information
- *
- * @param dev
- *   Pointer to the device structure.
- *
- * @param rx_queue_id
- *   Rx queue identificator.
- *
- * @param qinfo
- *   Pointer to the RX queue information structure.
- *
- * @return
- *   None.
- */
-
-void
-mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
-		  struct rte_eth_rxq_info *qinfo)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id];
-	struct mlx5_rxq_ctrl *rxq_ctrl =
-		container_of(rxq, struct mlx5_rxq_ctrl, rxq);
-
-	if (!rxq)
-		return;
-	qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
-					rxq->mprq_mp : rxq->mp;
-	qinfo->conf.rx_thresh.pthresh = 0;
-	qinfo->conf.rx_thresh.hthresh = 0;
-	qinfo->conf.rx_thresh.wthresh = 0;
-	qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh;
-	qinfo->conf.rx_drop_en = 1;
-	qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1;
-	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
-	qinfo->scattered_rx = dev->data->scattered_rx;
-	qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
-		(1 << rxq->elts_n) * (1 << rxq->strd_num_n) :
-		(1 << rxq->elts_n);
-}
-
-/**
- * DPDK callback to get the RX packet burst mode information
- *
- * @param dev
- *   Pointer to the device structure.
- *
- * @param rx_queue_id
- *   Rx queue identificatior.
- *
- * @param mode
- *   Pointer to the burts mode information.
- *
- * @return
- *   0 as success, -EINVAL as failure.
- */
-
-int
-mlx5_rx_burst_mode_get(struct rte_eth_dev *dev,
-		       uint16_t rx_queue_id __rte_unused,
-		       struct rte_eth_burst_mode *mode)
-{
-	eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_rxq_data *rxq;
-
-	rxq = (*priv->rxqs)[rx_queue_id];
-	if (!rxq) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	if (pkt_burst == mlx5_rx_burst) {
-		snprintf(mode->info, sizeof(mode->info), "%s", "Scalar");
-	} else if (pkt_burst == mlx5_rx_burst_mprq) {
-		snprintf(mode->info, sizeof(mode->info), "%s", "Multi-

Packet RQ");

drivers/net/mlx5/mlx5_rxtx.c +0 -3

-	} else if (pkt_burst == mlx5_rx_burst_vec) {
-#if defined RTE_ARCH_X86_64
-		snprintf(mode->info, sizeof(mode->info), "%s", "Vector

SSE");

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-#elif defined RTE_ARCH_ARM64
-		snprintf(mode->info, sizeof(mode->info), "%s", "Vector

Neon");

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-#elif defined RTE_ARCH_PPC_64
-		snprintf(mode->info, sizeof(mode->info), "%s", "Vector

AltiVec");

drivers/net/mlx5/mlx5_rxtx.c +0 -6

-#else
-		return -EINVAL;
-#endif
-	} else if (pkt_burst == mlx5_rx_burst_mprq_vec) {
-#if defined RTE_ARCH_X86_64
-		snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ

Vector SSE");

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-#elif defined RTE_ARCH_ARM64
-		snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ

Vector Neon");

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-#elif defined RTE_ARCH_PPC_64
-		snprintf(mode->info, sizeof(mode->info), "%s", "MPRQ

Vector AltiVec");

drivers/net/mlx5/mlx5_rxtx.c +0 -136

-#else
-		return -EINVAL;
-#endif
-	} else {
-		return -EINVAL;
-	}
-	return 0;
-}
-
-/**
- * DPDK callback to get the number of used descriptors in a RX queue
- *
- * @param dev
- *   Pointer to the device structure.
- *
- * @param rx_queue_id
- *   The Rx queue.
- *
- * @return
- *   The number of used rx descriptor.
- *   -EINVAL if the queue is invalid
- */
-uint32_t
-mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
-{
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_rxq_data *rxq;
-
-	if (dev->rx_pkt_burst == NULL ||
-	    dev->rx_pkt_burst == removed_rx_burst) {
-		rte_errno = ENOTSUP;
-		return -rte_errno;
-	}
-	rxq = (*priv->rxqs)[rx_queue_id];
-	if (!rxq) {
-		rte_errno = EINVAL;
-		return -rte_errno;
-	}
-	return rx_queue_count(rxq);
-}
-
 #define MLX5_SYSTEM_LOG_DIR "/var/log"
 /**
  * Dump debug information to log file.

@@ -863,101 +618,6 @@ enum mlx5_txcmp_code {
 }
 
 /**
- * Translate RX completion flags to packet type.
- *
- * @param[in] rxq
- *   Pointer to RX queue structure.
- * @param[in] cqe
- *   Pointer to CQE.
- *
- * @note: fix mlx5_dev_supported_ptypes_get() if any change here.
- *
- * @return
- *   Packet type for struct rte_mbuf.
- */
-static inline uint32_t
-rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe
 *cqe,
-				   volatile struct mlx5_mini_cqe8 *mcqe)
-{
-	uint8_t idx;
-	uint8_t ptype;
-	uint8_t pinfo = (cqe->pkt_info & 0x3) << 6;
-
-	/* Get l3/l4 header from mini-CQE in case L3/L4 format*/
-	if (mcqe == NULL ||
-	    rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
-		ptype = (cqe->hdr_type_etc & 0xfc00) >> 10;
-	else
-		ptype = mcqe->hdr_type >> 2;
-	/*
-	 * The index to the array should have:
-	 * bit[1:0] = l3_hdr_type
-	 * bit[4:2] = l4_hdr_type
-	 * bit[5] = ip_frag
-	 * bit[6] = tunneled
-	 * bit[7] = outer_l3_type
-	 */
-	idx = pinfo | ptype;
-	return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
-}
-
-/**
- * Initialize Rx WQ and indexes.
- *
- * @param[in] rxq
- *   Pointer to RX queue structure.
- */
-void
-mlx5_rxq_initialize(struct mlx5_rxq_data *rxq)
-{
-	const unsigned int wqe_n = 1 << rxq->elts_n;
-	unsigned int i;
-
-	for (i = 0; (i != wqe_n); ++i) {
-		volatile struct mlx5_wqe_data_seg *scat;
-		uintptr_t addr;
-		uint32_t byte_count;
-
-		if (mlx5_rxq_mprq_enabled(rxq)) {
-			struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[i];
-
-			scat = &((volatile struct mlx5_wqe_mprq *)
-				rxq->wqes)[i].dseg;
-			addr = (uintptr_t)mlx5_mprq_buf_addr(buf,
-							 1 << rxq-
 >strd_num_n);
-			byte_count = (1 << rxq->strd_sz_n) *
-					(1 << rxq->strd_num_n);
-		} else {
-			struct rte_mbuf *buf = (*rxq->elts)[i];
-
-			scat = &((volatile struct mlx5_wqe_data_seg *)
-					rxq->wqes)[i];
-			addr = rte_pktmbuf_mtod(buf, uintptr_t);
-			byte_count = DATA_LEN(buf);
-		}
-		/* scat->addr must be able to store a pointer. */
-		MLX5_ASSERT(sizeof(scat->addr) >= sizeof(uintptr_t));
-		*scat = (struct mlx5_wqe_data_seg){
-			.addr = rte_cpu_to_be_64(addr),
-			.byte_count = rte_cpu_to_be_32(byte_count),
-			.lkey = mlx5_rx_addr2mr(rxq, addr),
-		};
-	}
-	rxq->consumed_strd = 0;
-	rxq->decompressed = 0;
-	rxq->rq_pi = 0;
-	rxq->zip = (struct rxq_zip){
-		.ai = 0,
-	};
-	rxq->elts_ci = mlx5_rxq_mprq_enabled(rxq) ?
-		(wqe_n >> rxq->sges_n) * (1 << rxq->strd_num_n) : 0;
-	/* Update doorbell counter. */
-	rxq->rq_ci = wqe_n >> rxq->sges_n;
-	rte_io_wmb();
-	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
-}
-
-/**
  * Modify a Verbs/DevX queue state.

* This must be called from the primary process.
*

drivers/net/mlx5/mlx5_rxtx.c +1 -11

@@ -1013,7 +673,7 @@ enum mlx5_txcmp_code {
  * @return
  *   0 in case of success else non-zero value.
  */
-static int
+int
 mlx5_queue_state_modify(struct rte_eth_dev *dev,
 			struct mlx5_mp_arg_queue_state_modify *sm)
 {

@@ -1034,783 +694,6 @@ enum mlx5_txcmp_code {
 }
 
 /**
- * Handle a Rx error.
- * The function inserts the RQ state to reset when the first error CQE is
- * shown, then drains the CQ by the caller function loop. When the CQ is
 empty,
- * it moves the RQ state to ready and initializes the RQ.
- * Next CQE identification and error counting are in the caller responsibility.
- *
- * @param[in] rxq
- *   Pointer to RX queue structure.
- * @param[in] vec
- *   1 when called from vectorized Rx burst, need to prepare mbufs for the

RQ.
- * 0 when called from non-vectorized Rx burst.
- *
- * @return
- * -1 in case of recovery error, otherwise the CQE status.
- */

drivers/net/mlx5/mlx5_rxtx.c +0 -27

-int
-mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec)
-{
-	const uint16_t cqe_n = 1 << rxq->cqe_n;
-	const uint16_t cqe_mask = cqe_n - 1;
-	const uint16_t wqe_n = 1 << rxq->elts_n;
-	const uint16_t strd_n = 1 << rxq->strd_num_n;
-	struct mlx5_rxq_ctrl *rxq_ctrl =
-			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
-	union {
-		volatile struct mlx5_cqe *cqe;
-		volatile struct mlx5_err_cqe *err_cqe;
-	} u = {
-		.cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask],
-	};
-	struct mlx5_mp_arg_queue_state_modify sm;
-	int ret;
-
-	switch (rxq->err_state) {
-	case MLX5_RXQ_ERR_STATE_NO_ERROR:
-		rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_RESET;
-		/* Fall-through */
-	case MLX5_RXQ_ERR_STATE_NEED_RESET:
-		sm.is_wq = 1;
-		sm.queue_id = rxq->idx;
-		sm.state = IBV_WQS_RESET;
-		if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl->priv),

&sm))

drivers/net/mlx5/mlx5_rxtx.c +0 -12

-			return -1;
-		if (rxq_ctrl->dump_file_n <
-		    rxq_ctrl->priv->config.max_dump_files_num) {
-			MKSTR(err_str, "Unexpected CQE error syndrome "
-			      "0x%02x CQN = %u RQN = %u wqe_counter = %u"
-			      " rq_ci = %u cq_ci = %u", u.err_cqe->syndrome,
-			      rxq->cqn, rxq_ctrl->wqn,
-			      rte_be_to_cpu_16(u.err_cqe->wqe_counter),
-			      rxq->rq_ci << rxq->sges_n, rxq->cq_ci);
-			MKSTR(name, "dpdk_mlx5_port_%u_rxq_%u_%u",
-			      rxq->port_id, rxq->idx, (uint32_t)rte_rdtsc());
-			mlx5_dump_debug_information(name, NULL,

err_str, 0);

drivers/net/mlx5/mlx5_rxtx.c +0 -1

-			mlx5_dump_debug_information(name, "MLX5 Error

CQ:",

drivers/net/mlx5/mlx5_rxtx.c +0 -4

-						    (const void *)((uintptr_t)
-								    rxq->cqes),
-						    sizeof(*u.cqe) * cqe_n);
-			mlx5_dump_debug_information(name, "MLX5 Error

RQ:",

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-						    (const void *)((uintptr_t)
-								    rxq-

Quoted 1 line

wqes),

drivers/net/mlx5/mlx5_rxtx.c +0 -13

-						    16 * wqe_n);
-			rxq_ctrl->dump_file_n++;
-		}
-		rxq->err_state = MLX5_RXQ_ERR_STATE_NEED_READY;
-		/* Fall-through */
-	case MLX5_RXQ_ERR_STATE_NEED_READY:
-		ret = check_cqe(u.cqe, cqe_n, rxq->cq_ci);
-		if (ret == MLX5_CQE_STATUS_HW_OWN) {
-			rte_io_wmb();
-			*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
-			rte_io_wmb();
-			/*
-			 * The RQ consumer index must be zeroed while

moving

drivers/net/mlx5/mlx5_rxtx.c +0 -8

-			 * from RESET state to RDY state.
-			 */
-			*rxq->rq_db = rte_cpu_to_be_32(0);
-			rte_io_wmb();
-			sm.is_wq = 1;
-			sm.queue_id = rxq->idx;
-			sm.state = IBV_WQS_RDY;
-			if (mlx5_queue_state_modify(ETH_DEV(rxq_ctrl-

Quoted 1 line

priv),

drivers/net/mlx5/mlx5_rxtx.c +0 -27

-						    &sm))
-				return -1;
-			if (vec) {
-				const uint32_t elts_n =
-					mlx5_rxq_mprq_enabled(rxq) ?
-					wqe_n * strd_n : wqe_n;
-				const uint32_t e_mask = elts_n - 1;
-				uint32_t elts_ci =
-					mlx5_rxq_mprq_enabled(rxq) ?
-					rxq->elts_ci : rxq->rq_ci;
-				uint32_t elt_idx;
-				struct rte_mbuf **elt;
-				int i;
-				unsigned int n = elts_n - (elts_ci -
-							  rxq->rq_pi);
-
-				for (i = 0; i < (int)n; ++i) {
-					elt_idx = (elts_ci + i) & e_mask;
-					elt = &(*rxq->elts)[elt_idx];
-					*elt = rte_mbuf_raw_alloc(rxq->mp);
-					if (!*elt) {
-						for (i--; i >= 0; --i) {
-							elt_idx = (elts_ci +
-								   i) & elts_n;
-							elt = &(*rxq->elts)
-								[elt_idx];
-

rte_pktmbuf_free_seg

drivers/net/mlx5/mlx5_rxtx.c +0 -10

-								(*elt);
-						}
-						return -1;
-					}
-				}
-				for (i = 0; i < (int)elts_n; ++i) {
-					elt = &(*rxq->elts)[i];
-					DATA_LEN(*elt) =
-						(uint16_t)((*elt)->buf_len -
-

	rte_pktmbuf_headroom(*elt));

drivers/net/mlx5/mlx5_rxtx.c +1 -5

-				}
-				/* Padding with a fake mbuf for vec Rx. */
-				for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP;
++i)
-					(*rxq->elts)[elts_n + i] =
-								&rxq-

Quoted 1 line

fake_mbuf;

drivers/net/mlx5/mlx5_rxtx.c +0 -3

-			}
-			mlx5_rxq_initialize(rxq);
-			rxq->err_state =

MLX5_RXQ_ERR_STATE_NO_ERROR;

drivers/net/mlx5/mlx5_rxtx.c +0 -41

-		}
-		return ret;
-	default:
-		return -1;
-	}
-}
-
-/**
- * Get size of the next packet for a given CQE. For compressed CQEs, the
- * consumer index is updated only once all packets of the current one have
- * been processed.
- *
- * @param rxq
- *   Pointer to RX queue.
- * @param cqe
- *   CQE to process.
- * @param[out] mcqe
- *   Store pointer to mini-CQE if compressed. Otherwise, the pointer is not
- *   written.
- *
- * @return
- *   0 in case of empty CQE, otherwise the packet size in bytes.
- */
-static inline int
-mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
-		 uint16_t cqe_cnt, volatile struct mlx5_mini_cqe8 **mcqe)
-{
-	struct rxq_zip *zip = &rxq->zip;
-	uint16_t cqe_n = cqe_cnt + 1;
-	int len;
-	uint16_t idx, end;
-
-	do {
-		len = 0;
-		/* Process compressed data in the CQE and mini arrays. */
-		if (zip->ai) {
-			volatile struct mlx5_mini_cqe8 (*mc)[8] =
-				(volatile struct mlx5_mini_cqe8 (*)[8])
-				(uintptr_t)(&(*rxq->cqes)[zip->ca &
-							  cqe_cnt].pkt_info);
-			len = rte_be_to_cpu_32((*mc)[zip->ai & 7].byte_cnt

&

drivers/net/mlx5/mlx5_rxtx.c +0 -8

-					       rxq->byte_mask);
-			*mcqe = &(*mc)[zip->ai & 7];
-			if ((++zip->ai & 7) == 0) {
-				/* Invalidate consumed CQEs */
-				idx = zip->ca;
-				end = zip->na;
-				while (idx != end) {
-					(*rxq->cqes)[idx & cqe_cnt].op_own

=

drivers/net/mlx5/mlx5_rxtx.c +0 -5

-						MLX5_CQE_INVALIDATE;
-					++idx;
-				}
-				/*
-				 * Increment consumer index to skip the

number

drivers/net/mlx5/mlx5_rxtx.c +0 -1

-				 * of CQEs consumed. Hardware leaves holes

in

drivers/net/mlx5/mlx5_rxtx.c +0 -12

-				 * the CQ ring for software use.
-				 */
-				zip->ca = zip->na;
-				zip->na += 8;
-			}
-			if (unlikely(rxq->zip.ai == rxq->zip.cqe_cnt)) {
-				/* Invalidate the rest */
-				idx = zip->ca;
-				end = zip->cq_ci;
-
-				while (idx != end) {
-					(*rxq->cqes)[idx & cqe_cnt].op_own

=

drivers/net/mlx5/mlx5_rxtx.c +0 -21

-						MLX5_CQE_INVALIDATE;
-					++idx;
-				}
-				rxq->cq_ci = zip->cq_ci;
-				zip->ai = 0;
-			}
-		/*
-		 * No compressed data, get next CQE and verify if it is
-		 * compressed.
-		 */
-		} else {
-			int ret;
-			int8_t op_own;
-			uint32_t cq_ci;
-
-			ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
-			if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
-				if (unlikely(ret == MLX5_CQE_STATUS_ERR ||
-					     rxq->err_state)) {
-					ret = mlx5_rx_err_handle(rxq, 0);
-					if (ret ==

MLX5_CQE_STATUS_HW_OWN ||

drivers/net/mlx5/mlx5_rxtx.c +0 -15

-					    ret == -1)
-						return 0;
-				} else {
-					return 0;
-				}
-			}
-			/*
-			 * Introduce the local variable to have queue cq_ci
-			 * index in queue structure always consistent with
-			 * actual CQE boundary (not pointing to the middle
-			 * of compressed CQE session).
-			 */
-			cq_ci = rxq->cq_ci + 1;
-			op_own = cqe->op_own;
-			if (MLX5_CQE_FORMAT(op_own) ==

MLX5_COMPRESSED) {

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-				volatile struct mlx5_mini_cqe8 (*mc)[8] =
-					(volatile struct mlx5_mini_cqe8

(*)[8])

drivers/net/mlx5/mlx5_rxtx.c +0 -5

-					(uintptr_t)(&(*rxq->cqes)
-						[cq_ci & cqe_cnt].pkt_info);
-
-				/* Fix endianness. */
-				zip->cqe_cnt = rte_be_to_cpu_32(cqe-

Quoted 1 line

byte_cnt);

drivers/net/mlx5/mlx5_rxtx.c +0 -5

-				/*
-				 * Current mini array position is the one
-				 * returned by check_cqe64().
-				 *
-				 * If completion comprises several mini

arrays,

drivers/net/mlx5/mlx5_rxtx.c +0 -7

-				 * as a special case the second one is located
-				 * 7 CQEs after the initial CQE instead of 8
-				 * for subsequent ones.
-				 */
-				zip->ca = cq_ci;
-				zip->na = zip->ca + 7;
-				/* Compute the next non compressed CQE.

*/

drivers/net/mlx5/mlx5_rxtx.c +0 -81

-				zip->cq_ci = rxq->cq_ci + zip->cqe_cnt;
-				/* Get packet size to return. */
-				len = rte_be_to_cpu_32((*mc)[0].byte_cnt &
-						       rxq->byte_mask);
-				*mcqe = &(*mc)[0];
-				zip->ai = 1;
-				/* Prefetch all to be invalidated */
-				idx = zip->ca;
-				end = zip->cq_ci;
-				while (idx != end) {
-					rte_prefetch0(&(*rxq->cqes)[(idx) &
-								    cqe_cnt]);
-					++idx;
-				}
-			} else {
-				rxq->cq_ci = cq_ci;
-				len = rte_be_to_cpu_32(cqe->byte_cnt);
-			}
-		}
-		if (unlikely(rxq->err_state)) {
-			cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
-			++rxq->stats.idropped;
-		} else {
-			return len;
-		}
-	} while (1);
-}
-
-/**
- * Translate RX completion flags to offload flags.
- *
- * @param[in] cqe
- *   Pointer to CQE.
- *
- * @return
- *   Offload flags (ol_flags) for struct rte_mbuf.
- */
-static inline uint32_t
-rxq_cq_to_ol_flags(volatile struct mlx5_cqe *cqe)
-{
-	uint32_t ol_flags = 0;
-	uint16_t flags = rte_be_to_cpu_16(cqe->hdr_type_etc);
-
-	ol_flags =
-		TRANSPOSE(flags,
-			  MLX5_CQE_RX_L3_HDR_VALID,
-			  PKT_RX_IP_CKSUM_GOOD) |
-		TRANSPOSE(flags,
-			  MLX5_CQE_RX_L4_HDR_VALID,
-			  PKT_RX_L4_CKSUM_GOOD);
-	return ol_flags;
-}
-
-/**
- * Fill in mbuf fields from RX completion flags.
- * Note that pkt->ol_flags should be initialized outside of this function.
- *
- * @param rxq
- *   Pointer to RX queue.
- * @param pkt
- *   mbuf to fill.
- * @param cqe
- *   CQE to process.
- * @param rss_hash_res
- *   Packet RSS Hash result.
- */
-static inline void
-rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt,
-	       volatile struct mlx5_cqe *cqe,
-	       volatile struct mlx5_mini_cqe8 *mcqe)
-{
-	/* Update packet information. */
-	pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe, mcqe);
-
-	if (rxq->rss_hash) {
-		uint32_t rss_hash_res = 0;
-
-		/* If compressed, take hash result from mini-CQE. */
-		if (mcqe == NULL ||
-		    rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_HASH)
-			rss_hash_res = rte_be_to_cpu_32(cqe-

Quoted 1 line

rx_hash_res);

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-		else
-			rss_hash_res = rte_be_to_cpu_32(mcqe-

Quoted 1 line

rx_hash_result);

drivers/net/mlx5/mlx5_rxtx.c +0 -11

-		if (rss_hash_res) {
-			pkt->hash.rss = rss_hash_res;
-			pkt->ol_flags |= PKT_RX_RSS_HASH;
-		}
-	}
-	if (rxq->mark) {
-		uint32_t mark = 0;
-
-		/* If compressed, take flow tag from mini-CQE. */
-		if (mcqe == NULL ||
-		    rxq->mcqe_format !=

MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)

drivers/net/mlx5/mlx5_rxtx.c +0 -7

-			mark = cqe->sop_drop_qpn;
-		else
-			mark = ((mcqe->byte_cnt_flow & 0xff) << 8) |
-				(mcqe->flow_tag_high << 16);
-		if (MLX5_FLOW_MARK_IS_VALID(mark)) {
-			pkt->ol_flags |= PKT_RX_FDIR;
-			if (mark !=

RTE_BE32(MLX5_FLOW_MARK_DEFAULT)) {

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-				pkt->ol_flags |= PKT_RX_FDIR_ID;
-				pkt->hash.fdir.hi =

mlx5_flow_mark_get(mark);

drivers/net/mlx5/mlx5_rxtx.c +0 -20

-			}
-		}
-	}
-	if (rxq->dynf_meta) {
-		uint32_t meta = cqe->flow_table_metadata &
-				rxq->flow_meta_port_mask;
-
-		if (meta) {
-			pkt->ol_flags |= rxq->flow_meta_mask;
-			*RTE_MBUF_DYNFIELD(pkt, rxq->flow_meta_offset,
-						uint32_t *) = meta;
-		}
-	}
-	if (rxq->csum)
-		pkt->ol_flags |= rxq_cq_to_ol_flags(cqe);
-	if (rxq->vlan_strip) {
-		bool vlan_strip;
-
-		if (mcqe == NULL ||
-		    rxq->mcqe_format !=

MLX5_CQE_RESP_FORMAT_L34H_STRIDX)

drivers/net/mlx5/mlx5_rxtx.c +0 -7

-			vlan_strip = cqe->hdr_type_etc &
-				     RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
-		else
-			vlan_strip = mcqe->hdr_type &
-				     RTE_BE16(MLX5_CQE_VLAN_STRIPPED);
-		if (vlan_strip) {
-			pkt->ol_flags |= PKT_RX_VLAN |

PKT_RX_VLAN_STRIPPED;

drivers/net/mlx5/mlx5_rxtx.c +0 -45

-			pkt->vlan_tci = rte_be_to_cpu_16(cqe->vlan_info);
-		}
-	}
-	if (rxq->hw_timestamp) {
-		uint64_t ts = rte_be_to_cpu_64(cqe->timestamp);
-
-		if (rxq->rt_timestamp)
-			ts = mlx5_txpp_convert_rx_ts(rxq->sh, ts);
-		mlx5_timestamp_set(pkt, rxq->timestamp_offset, ts);
-		pkt->ol_flags |= rxq->timestamp_rx_flag;
-	}
-}
-
-/**
- * DPDK callback for RX.
- *
- * @param dpdk_rxq
- *   Generic pointer to RX queue structure.
- * @param[out] pkts
- *   Array to store received packets.
- * @param pkts_n
- *   Maximum number of packets in array.
- *
- * @return
- *   Number of packets successfully received (<= pkts_n).
- */
-uint16_t
-mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
-{
-	struct mlx5_rxq_data *rxq = dpdk_rxq;
-	const unsigned int wqe_cnt = (1 << rxq->elts_n) - 1;
-	const unsigned int cqe_cnt = (1 << rxq->cqe_n) - 1;
-	const unsigned int sges_n = rxq->sges_n;
-	struct rte_mbuf *pkt = NULL;
-	struct rte_mbuf *seg = NULL;
-	volatile struct mlx5_cqe *cqe =
-		&(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
-	unsigned int i = 0;
-	unsigned int rq_ci = rxq->rq_ci << sges_n;
-	int len = 0; /* keep its value across iterations. */
-
-	while (pkts_n) {
-		unsigned int idx = rq_ci & wqe_cnt;
-		volatile struct mlx5_wqe_data_seg *wqe =
-			&((volatile struct mlx5_wqe_data_seg *)rxq-

Quoted 1 line

wqes)[idx];

drivers/net/mlx5/mlx5_rxtx.c +0 -50

-		struct rte_mbuf *rep = (*rxq->elts)[idx];
-		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
-
-		if (pkt)
-			NEXT(seg) = rep;
-		seg = rep;
-		rte_prefetch0(seg);
-		rte_prefetch0(cqe);
-		rte_prefetch0(wqe);
-		/* Allocate the buf from the same pool. */
-		rep = rte_mbuf_raw_alloc(seg->pool);
-		if (unlikely(rep == NULL)) {
-			++rxq->stats.rx_nombuf;
-			if (!pkt) {
-				/*
-				 * no buffers before we even started,
-				 * bail out silently.
-				 */
-				break;
-			}
-			while (pkt != seg) {
-				MLX5_ASSERT(pkt != (*rxq->elts)[idx]);
-				rep = NEXT(pkt);
-				NEXT(pkt) = NULL;
-				NB_SEGS(pkt) = 1;
-				rte_mbuf_raw_free(pkt);
-				pkt = rep;
-			}
-			rq_ci >>= sges_n;
-			++rq_ci;
-			rq_ci <<= sges_n;
-			break;
-		}
-		if (!pkt) {
-			cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_cnt];
-			len = mlx5_rx_poll_len(rxq, cqe, cqe_cnt, &mcqe);
-			if (!len) {
-				rte_mbuf_raw_free(rep);
-				break;
-			}
-			pkt = seg;
-			MLX5_ASSERT(len >= (rxq->crc_present << 2));
-			pkt->ol_flags &= EXT_ATTACHED_MBUF;
-			rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
-			if (rxq->crc_present)
-				len -= RTE_ETHER_CRC_LEN;
-			PKT_LEN(pkt) = len;
-			if (cqe->lro_num_seg > 1) {
-				mlx5_lro_update_hdr
-					(rte_pktmbuf_mtod(pkt, uint8_t *),

cqe,

drivers/net/mlx5/mlx5_rxtx.c +0 -16

-					 mcqe, rxq, len);
-				pkt->ol_flags |= PKT_RX_LRO;
-				pkt->tso_segsz = len / cqe->lro_num_seg;
-			}
-		}
-		DATA_LEN(rep) = DATA_LEN(seg);
-		PKT_LEN(rep) = PKT_LEN(seg);
-		SET_DATA_OFF(rep, DATA_OFF(seg));
-		PORT(rep) = PORT(seg);
-		(*rxq->elts)[idx] = rep;
-		/*
-		 * Fill NIC descriptor with the new buffer.  The lkey and size
-		 * of the buffers are already known, only the buffer address
-		 * changes.
-		 */
-		wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep,

uintptr_t));

drivers/net/mlx5/mlx5_rxtx.c +0 -2

-		/* If there's only one MR, no need to replace LKey in WQE. */
-		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) >

1))

drivers/net/mlx5/mlx5_rxtx.c +0 -56

-			wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
-		if (len > DATA_LEN(seg)) {
-			len -= DATA_LEN(seg);
-			++NB_SEGS(pkt);
-			++rq_ci;
-			continue;
-		}
-		DATA_LEN(seg) = len;
-#ifdef MLX5_PMD_SOFT_COUNTERS
-		/* Increment bytes counter. */
-		rxq->stats.ibytes += PKT_LEN(pkt);
-#endif
-		/* Return packet. */
-		*(pkts++) = pkt;
-		pkt = NULL;
-		--pkts_n;
-		++i;
-		/* Align consumer index to the next stride. */
-		rq_ci >>= sges_n;
-		++rq_ci;
-		rq_ci <<= sges_n;
-	}
-	if (unlikely((i == 0) && ((rq_ci >> sges_n) == rxq->rq_ci)))
-		return 0;
-	/* Update the consumer index. */
-	rxq->rq_ci = rq_ci >> sges_n;
-	rte_io_wmb();
-	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
-	rte_io_wmb();
-	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
-#ifdef MLX5_PMD_SOFT_COUNTERS
-	/* Increment packets counter. */
-	rxq->stats.ipackets += i;
-#endif
-	return i;
-}
-
-/**
- * Update LRO packet TCP header.
- * The HW LRO feature doesn't update the TCP header after coalescing the
- * TCP segments but supplies information in CQE to fill it by SW.
- *
- * @param tcp
- *   Pointer to the TCP header.
- * @param cqe
- *   Pointer to the completion entry..
- * @param phcsum
- *   The L3 pseudo-header checksum.
- */
-static inline void
-mlx5_lro_update_tcp_hdr(struct rte_tcp_hdr *__rte_restrict tcp,
-			volatile struct mlx5_cqe *__rte_restrict cqe,
-			uint32_t phcsum, uint8_t l4_type)
-{
-	/*
-	 * The HW calculates only the TCP payload checksum, need to

complete

drivers/net/mlx5/mlx5_rxtx.c +0 -24

-	 * the TCP header checksum and the L3 pseudo-header checksum.
-	 */
-	uint32_t csum = phcsum + cqe->csum;
-
-	if (l4_type == MLX5_L4_HDR_TYPE_TCP_EMPTY_ACK ||
-	    l4_type == MLX5_L4_HDR_TYPE_TCP_WITH_ACL) {
-		tcp->tcp_flags |= RTE_TCP_ACK_FLAG;
-		tcp->recv_ack = cqe->lro_ack_seq_num;
-		tcp->rx_win = cqe->lro_tcp_win;
-	}
-	if (cqe->lro_tcppsh_abort_dupack & MLX5_CQE_LRO_PUSH_MASK)
-		tcp->tcp_flags |= RTE_TCP_PSH_FLAG;
-	tcp->cksum = 0;
-	csum += rte_raw_cksum(tcp, (tcp->data_off >> 4) * 4);
-	csum = ((csum & 0xffff0000) >> 16) + (csum & 0xffff);
-	csum = (~csum) & 0xffff;
-	if (csum == 0)
-		csum = 0xffff;
-	tcp->cksum = csum;
-}
-
-/**
- * Update LRO packet headers.
- * The HW LRO feature doesn't update the L3/TCP headers after coalescing

the
- * TCP segments but supply information in CQE to fill it by SW.
- *
- * @param padd
- * The packet address.
- * @param cqe
- * Pointer to the completion entry..
- * @param len
- * The packet length.
- */

drivers/net/mlx5/mlx5_rxtx.c +0 -29

-static inline void
-mlx5_lro_update_hdr(uint8_t *__rte_restrict padd,
-		    volatile struct mlx5_cqe *__rte_restrict cqe,
-		    volatile struct mlx5_mini_cqe8 *mcqe,
-		    struct mlx5_rxq_data *rxq, uint32_t len)
-{
-	union {
-		struct rte_ether_hdr *eth;
-		struct rte_vlan_hdr *vlan;
-		struct rte_ipv4_hdr *ipv4;
-		struct rte_ipv6_hdr *ipv6;
-		struct rte_tcp_hdr *tcp;
-		uint8_t *hdr;
-	} h = {
-			.hdr = padd,
-	};
-	uint16_t proto = h.eth->ether_type;
-	uint32_t phcsum;
-	uint8_t l4_type;
-
-	h.eth++;
-	while (proto == RTE_BE16(RTE_ETHER_TYPE_VLAN) ||
-	       proto == RTE_BE16(RTE_ETHER_TYPE_QINQ)) {
-		proto = h.vlan->eth_proto;
-		h.vlan++;
-	}
-	if (proto == RTE_BE16(RTE_ETHER_TYPE_IPV4)) {
-		h.ipv4->time_to_live = cqe->lro_min_ttl;
-		h.ipv4->total_length = rte_cpu_to_be_16(len - (h.hdr -

padd));

drivers/net/mlx5/mlx5_rxtx.c +0 -16

-		h.ipv4->hdr_checksum = 0;
-		h.ipv4->hdr_checksum = rte_ipv4_cksum(h.ipv4);
-		phcsum = rte_ipv4_phdr_cksum(h.ipv4, 0);
-		h.ipv4++;
-	} else {
-		h.ipv6->hop_limits = cqe->lro_min_ttl;
-		h.ipv6->payload_len = rte_cpu_to_be_16(len - (h.hdr - padd)
-
-						       sizeof(*h.ipv6));
-		phcsum = rte_ipv6_phdr_cksum(h.ipv6, 0);
-		h.ipv6++;
-	}
-	if (mcqe == NULL ||
-	    rxq->mcqe_format != MLX5_CQE_RESP_FORMAT_L34H_STRIDX)
-		l4_type = (rte_be_to_cpu_16(cqe->hdr_type_etc) &
-			   MLX5_CQE_L4_TYPE_MASK) >>

MLX5_CQE_L4_TYPE_SHIFT;

drivers/net/mlx5/mlx5_rxtx.c +0 -3

-	else
-		l4_type = (rte_be_to_cpu_16(mcqe->hdr_type) &
-			   MLX5_CQE_L4_TYPE_MASK) >>

MLX5_CQE_L4_TYPE_SHIFT;

drivers/net/mlx5/mlx5_rxtx.c +0 -38

-	mlx5_lro_update_tcp_hdr(h.tcp, cqe, phcsum, l4_type);
-}
-
-void
-mlx5_mprq_buf_free_cb(void *addr __rte_unused, void *opaque)
-{
-	struct mlx5_mprq_buf *buf = opaque;
-
-	if (__atomic_load_n(&buf->refcnt, __ATOMIC_RELAXED) == 1) {
-		rte_mempool_put(buf->mp, buf);
-	} else if (unlikely(__atomic_sub_fetch(&buf->refcnt, 1,
-					       __ATOMIC_RELAXED) == 0)) {
-		__atomic_store_n(&buf->refcnt, 1, __ATOMIC_RELAXED);
-		rte_mempool_put(buf->mp, buf);
-	}
-}
-
-void
-mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf)
-{
-	mlx5_mprq_buf_free_cb(NULL, buf);
-}
-
-/**
- * DPDK callback for RX with Multi-Packet RQ support.
- *
- * @param dpdk_rxq
- *   Generic pointer to RX queue structure.
- * @param[out] pkts
- *   Array to store received packets.
- * @param pkts_n
- *   Maximum number of packets in array.
- *
- * @return
- *   Number of packets successfully received (<= pkts_n).
- */
-uint16_t
-mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t

pkts_n)

drivers/net/mlx5/mlx5_rxtx.c +0 -7

-{
-	struct mlx5_rxq_data *rxq = dpdk_rxq;
-	const uint32_t strd_n = 1 << rxq->strd_num_n;
-	const uint32_t strd_sz = 1 << rxq->strd_sz_n;
-	const uint32_t cq_mask = (1 << rxq->cqe_n) - 1;
-	const uint32_t wq_mask = (1 << rxq->elts_n) - 1;
-	volatile struct mlx5_cqe *cqe = &(*rxq->cqes)[rxq->cq_ci &

cq_mask];

drivers/net/mlx5/mlx5_rxtx.c +0 -29

-	unsigned int i = 0;
-	uint32_t rq_ci = rxq->rq_ci;
-	uint16_t consumed_strd = rxq->consumed_strd;
-	struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
-
-	while (i < pkts_n) {
-		struct rte_mbuf *pkt;
-		int ret;
-		uint32_t len;
-		uint16_t strd_cnt;
-		uint16_t strd_idx;
-		uint32_t byte_cnt;
-		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
-		enum mlx5_rqx_code rxq_code;
-
-		if (consumed_strd == strd_n) {
-			/* Replace WQE if the buffer is still in use. */
-			mprq_buf_replace(rxq, rq_ci & wq_mask);
-			/* Advance to the next WQE. */
-			consumed_strd = 0;
-			++rq_ci;
-			buf = (*rxq->mprq_bufs)[rq_ci & wq_mask];
-		}
-		cqe = &(*rxq->cqes)[rxq->cq_ci & cq_mask];
-		ret = mlx5_rx_poll_len(rxq, cqe, cq_mask, &mcqe);
-		if (!ret)
-			break;
-		byte_cnt = ret;
-		len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >>

MLX5_MPRQ_LEN_SHIFT;

drivers/net/mlx5/mlx5_rxtx.c +0 -5

-		MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
-		if (rxq->crc_present)
-			len -= RTE_ETHER_CRC_LEN;
-		if (mcqe &&
-		    rxq->mcqe_format ==

MLX5_CQE_RESP_FORMAT_FTAG_STRIDX)

drivers/net/mlx5/mlx5_rxtx.c +0 -3

-			strd_cnt = (len / strd_sz) + !!(len % strd_sz);
-		else
-			strd_cnt = (byte_cnt &

MLX5_MPRQ_STRIDE_NUM_MASK) >>

drivers/net/mlx5/mlx5_rxtx.c +0 -10

-				   MLX5_MPRQ_STRIDE_NUM_SHIFT;
-		MLX5_ASSERT(strd_cnt);
-		consumed_strd += strd_cnt;
-		if (byte_cnt & MLX5_MPRQ_FILLER_MASK)
-			continue;
-		strd_idx = rte_be_to_cpu_16(mcqe == NULL ?
-					cqe->wqe_counter :
-					mcqe->stride_idx);
-		MLX5_ASSERT(strd_idx < strd_n);
-		MLX5_ASSERT(!((rte_be_to_cpu_16(cqe->wqe_id) ^ rq_ci)

&

drivers/net/mlx5/mlx5_rxtx.c +0 -7

-			    wq_mask));
-		pkt = rte_pktmbuf_alloc(rxq->mp);
-		if (unlikely(pkt == NULL)) {
-			++rxq->stats.rx_nombuf;
-			break;
-		}
-		len = (byte_cnt & MLX5_MPRQ_LEN_MASK) >>

MLX5_MPRQ_LEN_SHIFT;

drivers/net/mlx5/mlx5_rxtx.c +0 -19

-		MLX5_ASSERT((int)len >= (rxq->crc_present << 2));
-		if (rxq->crc_present)
-			len -= RTE_ETHER_CRC_LEN;
-		rxq_code = mprq_buf_to_pkt(rxq, pkt, len, buf,
-					   strd_idx, strd_cnt);
-		if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) {
-			rte_pktmbuf_free_seg(pkt);
-			if (rxq_code == MLX5_RXQ_CODE_DROPPED) {
-				++rxq->stats.idropped;
-				continue;
-			}
-			if (rxq_code == MLX5_RXQ_CODE_NOMBUF) {
-				++rxq->stats.rx_nombuf;
-				break;
-			}
-		}
-		rxq_cq_to_mbuf(rxq, pkt, cqe, mcqe);
-		if (cqe->lro_num_seg > 1) {
-			mlx5_lro_update_hdr(rte_pktmbuf_mtod(pkt,

uint8_t *),

drivers/net/mlx5/mlx5_rxtx.c +0 -59

-					    cqe, mcqe, rxq, len);
-			pkt->ol_flags |= PKT_RX_LRO;
-			pkt->tso_segsz = len / cqe->lro_num_seg;
-		}
-		PKT_LEN(pkt) = len;
-		PORT(pkt) = rxq->port_id;
-#ifdef MLX5_PMD_SOFT_COUNTERS
-		/* Increment bytes counter. */
-		rxq->stats.ibytes += PKT_LEN(pkt);
-#endif
-		/* Return packet. */
-		*(pkts++) = pkt;
-		++i;
-	}
-	/* Update the consumer indexes. */
-	rxq->consumed_strd = consumed_strd;
-	rte_io_wmb();
-	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
-	if (rq_ci != rxq->rq_ci) {
-		rxq->rq_ci = rq_ci;
-		rte_io_wmb();
-		*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
-	}
-#ifdef MLX5_PMD_SOFT_COUNTERS
-	/* Increment packets counter. */
-	rxq->stats.ipackets += i;
-#endif
-	return i;
-}
-
-/**
  * Dummy DPDK callback for TX.
  *
  * This function is used to temporarily replace the real callback during

@@ -1836,66 +719,6 @@ enum mlx5_txcmp_code {
 }
 
 /**
- * Dummy DPDK callback for RX.
- *
- * This function is used to temporarily replace the real callback during
- * unsafe control operations on the queue, or in case of error.
- *
- * @param dpdk_rxq
- *   Generic pointer to RX queue structure.
- * @param[out] pkts
- *   Array to store received packets.
- * @param pkts_n
- *   Maximum number of packets in array.
- *
- * @return
- *   Number of packets successfully received (<= pkts_n).
- */
-uint16_t
-removed_rx_burst(void *dpdk_txq __rte_unused,
-		 struct rte_mbuf **pkts __rte_unused,
-		 uint16_t pkts_n __rte_unused)
-{
-	rte_mb();
-	return 0;
-}
-
-/*
- * Vectorized Rx/Tx routines are not compiled in when required vector
- * instructions are not supported on a target architecture. The following null
- * stubs are needed for linkage when those are not included outside of this

file
- * (e.g. mlx5_rxtx_vec_sse.c for x86).
- */

drivers/net/mlx5/mlx5_rxtx.c +0 -30

-
-__rte_weak uint16_t
-mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
-		  struct rte_mbuf **pkts __rte_unused,
-		  uint16_t pkts_n __rte_unused)
-{
-	return 0;
-}
-
-__rte_weak uint16_t
-mlx5_rx_burst_mprq_vec(void *dpdk_txq __rte_unused,
-		       struct rte_mbuf **pkts __rte_unused,
-		       uint16_t pkts_n __rte_unused)
-{
-	return 0;
-}
-
-__rte_weak int
-mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
-{
-	return -ENOTSUP;
-}
-
-__rte_weak int
-mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
-{
-	return -ENOTSUP;
-}
-
-/**
  * Free the mbufs from the linear array of pointers.
  *
  * @param txq

drivers/net/mlx5/mlx5_rxtx.h +2 -0

index d443db4..f1ebc99 100644
@@ -196,6 +196,8 @@ void mlx5_dump_debug_information(const char
 *path, const char *title,
 				 const void *buf, unsigned int len);
 int mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
 			const struct mlx5_mp_arg_queue_state_modify
 *sm);
+int mlx5_queue_state_modify(struct rte_eth_dev *dev,
+			    struct mlx5_mp_arg_queue_state_modify *sm);
 void mlx5_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,

struct rte_eth_txq_info *qinfo);
int mlx5_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t
tx_queue_id,
--
1.8.3.1