new mlx4 Tx datapath bypassing ibverbs

new file mode 100644
index 0000000..c5ce33b
@@ -0,0 +1,253 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 Mellanox.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RTE_PMD_MLX4_MLX4_PRM_H_
+#define RTE_PMD_MLX4_MLX4_PRM_H_
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* Basic TxQ building block */
+#define TXBB_SHIFT 6
+#define TXBB_SIZE (1 << TXBB_SHIFT)
+
+/* Typical TSO descriptor with 16 gather entries is 352 bytes... */
+#define MAX_WQE_SIZE		512
+#define MAX_WQE_TXBBS		(MAX_WQE_SIZE / TXBB_SIZE)
+
+/* Send Queue Stamping/Invalidating info */
+#define SQ_STAMP_STRIDE		64
+#define SQ_STAMP_DWORDS		(SQ_STAMP_STRIDE / 4)
+#define SQ_STAMP_SHIFT		31
+#define SQ_STAMP_VAL		0x7fffffff
+
+/* WQE flags */
+#define MLX4_OPCODE_SEND	0x0a
+#define MLX4_EN_BIT_WQE_OWN	0x80000000
+
+#define SIZE_TO_TXBBS(size)     (RTE_ALIGN((size), (TXBB_SIZE)) / (TXBB_SIZE))
+
+/**
+ * Update the HW with the new  CQ consumer value.
+ *
+ * @param cq
+ *   Pointer to the cq structure.
+ */
+static inline void
+mlx4_cq_set_ci(struct mlx4_cq *cq)
+{
+	*cq->set_ci_db = rte_cpu_to_be_32(cq->cons_index & 0xffffff);
+}
+
+/**
+ * Returns a pointer to the cqe in position n.
+ *
+ * @param cq
+ *   Pointer to the cq structure.
+ * @param n
+ *   The number of the entry its address we seek.
+ *
+ * @return
+ *   pointer to the cqe.
+ */
+static inline struct mlx4_cqe
+*mlx4_get_cqe(struct mlx4_cq *cq, int n)
+{
+	return (struct mlx4_cqe *)(cq->buf + n * cq->cqe_size);
+}
+
+/**
+ * Returns a pointer to the cqe in position n if it is owned by SW.
+ *
+ * @param cq
+ *   Pointer to the cq structure.
+ * @param n
+ *   The number of the entry its address we seek.
+ *
+ * @return
+ *   pointer to the cqe if owned by SW, otherwise returns NULL.
+ */
+static inline void
+*mlx4_get_sw_cqe(struct mlx4_cq *cq, int n)
+{
+	struct mlx4_cqe *cqe = mlx4_get_cqe(cq, n & (cq->cqe_cnt - 1));
+	struct mlx4_cqe *tcqe = cq->cqe_size == 64 ? cqe + 1 : cqe;
+
+	return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
+		!!(n & cq->cqe_cnt)) ? NULL : cqe;
+}
+
+/**
+ * returns pointer to the wqe at position n.
+ *
+ * @param sq
+ *   Pointer to the sq.
+ * @param n
+ *   The entry number the queue.
+ *
+ * @return
+ *   A pointer to the required entry.
+ */
+static inline void
+*mlx4_get_send_wqe(struct mlx4_sq *sq, unsigned int n)
+{
+	return sq->buf + n * TXBB_SIZE;
+}
+
+/**
+ * returns the size in bytes of this WQE.
+ *
+ * @param wqe
+ *   Pointer to the WQE we want to interrogate.
+ *
+ * @return
+ *   WQE size in bytes.
+ */
+static inline int
+mlx4_wqe_get_real_size(void *wqe)
+{
+	struct mlx4_wqe_ctrl_seg *ctrl = (struct mlx4_wqe_ctrl_seg *)wqe;
+	return ((ctrl->fence_size & 0x3f) << 4);
+}
+
+/**
+ * Fills the ctrl segment of a WQE with info needed for transmitting the packet.
+ *
+ * @param seg
+ *   Pointer to the control structure in the WQE.
+ * @param owner
+ *   The value for the owner field.
+ * @param fence_size
+ *   Fence bit and WQE size in octowords.
+ * @param srcrb_flags
+ *   High 24 bits are SRC remote buffer; low 8 bits are flags.
+ * @param imm
+ *   Immediate data/Invalidation key..
+ */
+static inline void
+mlx4_set_ctrl_seg(struct mlx4_wqe_ctrl_seg *seg, uint32_t owner,
+	     uint8_t fence_size, uint32_t srcrb_flags, uint32_t imm)
+{
+	seg->fence_size = fence_size;
+	seg->srcrb_flags = rte_cpu_to_be_32(srcrb_flags);
+	/*
+	 * The caller should prepare "imm" in advance based on WR opcode.
+	 * For IBV_WR_SEND_WITH_IMM and IBV_WR_RDMA_WRITE_WITH_IMM,
+	 * the "imm" should be assigned as is.
+	 * For the IBV_WR_SEND_WITH_INV, it should be htobe32(imm).
+	 */
+	seg->imm = imm;
+	/*
+	 * Make sure descriptor is fully written before
+	 * setting ownership bit (because HW can start
+	 * executing as soon as we do).
+	 */
+	rte_wmb();
+	seg->owner_opcode = rte_cpu_to_be_32(owner);
+}
+
+/**
+ * Fills a data segment of a WQE with info needed for transmitting
+ * a data fragment.
+ *
+ * @param dseg
+ *   Pointer to a data segment structure in the WQE.
+ *   (WQE may contain several data segments).
+ * @param sq
+ *   fragment info (addr, length, lkey).
+ */
+static inline void
+mlx4_set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ibv_sge *sg)
+{
+	dseg->lkey       = rte_cpu_to_be_32(sg->lkey);
+	dseg->addr       = rte_cpu_to_be_64(sg->addr);
+
+	/*
+	 * Need a barrier here before writing the byte_count field to
+	 * make sure that all the data is visible before the
+	 * byte_count field is set.  Otherwise, if the segment begins
+	 * a new cacheline, the HCA prefetcher could grab the 64-byte
+	 * chunk and get a valid (!= * 0xffffffff) byte count but
+	 * stale data, and end up sending the wrong data.
+	 */
+	rte_io_wmb();
+
+	if (likely(sg->length))
+		dseg->byte_count = rte_cpu_to_be_32(sg->length);
+	else
+		/* Zero len seg is treated as inline segment with zero data */
+		dseg->byte_count = rte_cpu_to_be_32(0x80000000);
+}
+
+/**
+ * Checks if a BBE sized WQE can be inserted to the sq without
+ * overflowing the Q and that theWQE is not over-sized.
+ *
+ * @param sq
+ *   Pointer to the sq we want to put the wqe in.
+ * @param ntxbb
+ *   Number of EBBs the WQE occupies.
+ */
+static inline int
+mlx4_wq_overflow(struct mlx4_sq *sq, int ntxbb)
+{
+	unsigned int cur;
+
+	cur = sq->head - sq->tail;
+	return ((cur + ntxbb + sq->headroom_txbbs >= sq->txbb_cnt) ||
+		(ntxbb > MAX_WQE_TXBBS));
+}
+
+/**
+ * Calc the  WQE size (in bytes) needed for posting this packet.
+ *
+ * @param count
+ *   The number of data-segments the WQE contains.
+ *
+ * @return
+ *   WQE size in bytes.
+ */
+static inline int
+mlx4_wqe_calc_real_size(unsigned int count)
+{
+	return sizeof(struct mlx4_wqe_ctrl_seg) +
+		(count * sizeof(struct mlx4_wqe_data_seg));
+}
+
+#endif /* RTE_PMD_MLX4_MLX4_PRM_H_ */

326 lines not on this page · raw message

index fec998a..e442730 100644
@@ -41,6 +41,7 @@
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
 #include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"
 #endif

@@ -90,7 +91,7 @@ struct txq_elt {
 	struct rte_mbuf *buf; /**< Buffer. */
 };
 
-/** Rx queue counters. */
+/** Tx queue counters. */
 struct mlx4_txq_stats {
 	unsigned int idx; /**< Mapping index. */
 	uint64_t opackets; /**< Total of successfully sent packets. */

@@ -98,6 +99,31 @@ struct mlx4_txq_stats {
 	uint64_t odropped; /**< Total of packets not sent when Tx ring full. */
 };
 
+/** TXQ Info */
+struct mlx4_sq {
+	char	*buf;  /**< SQ buffer. */
+	uint32_t size; /**< SQ size in bytes. */
+	uint32_t head; /**< SQ head counter in units of TXBBS. */
+	uint32_t tail; /**< SQ tail counter in units of TXBBS. */
+	uint32_t txbb_cnt;       /**< Num of WQEBB in the Q (should be ^2). */
+	uint32_t txbb_shift;     /**< The log2 size of the basic block. */
+	uint32_t txbb_cnt_mask;	 /**< txbbs_cnt mask (txbb_cnt is ^2). */
+	uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free .*/
+	uint32_t *db;            /**< Pointer to the doorbell. */
+	uint32_t doorbell_qpn;   /**< qp number to write to the doorbell. */
+};
+
+struct mlx4_cq {
+	char	 *buf;       /**< CQ buffer. */
+	uint32_t size;       /**< CQ size in bytes. */
+	uint32_t cqe_cnt;    /**< Num of entries the CQ has. */
+	int	 cqe_size;   /**< size (in bytes) of a CQE. */
+	uint32_t *set_ci_db; /**< Pionter of the consumer-index doorbell. */
+	uint32_t cons_index; /**< last CQE entry that was handled. */
+	uint32_t factor;     /**< CQ data location in a CQE. */
+	int	 cqn;        /**< CQ number */
+};
+
 /** Tx queue descriptor. */
 struct txq {
 	struct priv *priv; /**< Back pointer to private data. */

@@ -118,6 +144,8 @@ struct txq {
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
+	struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
+	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 };
 
 /* mlx4_rxq.c */

88 lines not on this page · raw message

index c5ce33b..8b0248a 100644
@@ -61,7 +61,7 @@
 #define MLX4_OPCODE_SEND	0x0a
 #define MLX4_EN_BIT_WQE_OWN	0x80000000
 
-#define SIZE_TO_TXBBS(size)     (RTE_ALIGN((size), (TXBB_SIZE)) / (TXBB_SIZE))
+#define SIZE_TO_TXBBS(size)	(RTE_ALIGN((size), (TXBB_SIZE)) / (TXBB_SIZE))
 
 /**
  * Update the HW with the new  CQ consumer value.

@@ -148,6 +148,7 @@
 
 /**
  * Fills the ctrl segment of a WQE with info needed for transmitting the packet.
+ * Owner field is filled later.
  *
  * @param seg
  *   Pointer to the control structure in the WQE.

@@ -161,8 +162,8 @@
  *   Immediate data/Invalidation key..
  */
 static inline void
-mlx4_set_ctrl_seg(struct mlx4_wqe_ctrl_seg *seg, uint32_t owner,
-	     uint8_t fence_size, uint32_t srcrb_flags, uint32_t imm)
+mlx4_set_ctrl_seg(struct mlx4_wqe_ctrl_seg *seg, uint8_t fence_size,
+		  uint32_t srcrb_flags, uint32_t imm)
 {
 	seg->fence_size = fence_size;
 	seg->srcrb_flags = rte_cpu_to_be_32(srcrb_flags);

@@ -173,13 +174,6 @@
 	 * For the IBV_WR_SEND_WITH_INV, it should be htobe32(imm).
 	 */
 	seg->imm = imm;
-	/*
-	 * Make sure descriptor is fully written before
-	 * setting ownership bit (because HW can start
-	 * executing as soon as we do).
-	 */
-	rte_wmb();
-	seg->owner_opcode = rte_cpu_to_be_32(owner);
 }
 
 /**

@@ -241,7 +235,7 @@
  *   The number of data-segments the WQE contains.
  *
  * @return
- *   WQE size in bytes.
+ *   The calculated WQE size in bytes.
  */
 static inline int
 mlx4_wqe_calc_real_size(unsigned int count)

264 lines not on this page · raw message

index e442730..7cae7e2 100644
@@ -139,13 +139,14 @@ struct txq {
 	struct txq_elt (*elts)[]; /**< Tx elements. */
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
-	unsigned int elts_comp; /**< Number of completion requests. */
+	unsigned int elts_comp; /**< Number of pkts waiting for completion. */
 	unsigned int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
 	struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
 	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
+	char *bounce_buf; /**< Side memory to be used when wqe wraps around */
 };
 
 /* mlx4_rxq.c */

index 1273738..6f6ea9c 100644
@@ -83,8 +83,14 @@
 		rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq->socket);
 	int ret = 0;
 
-	if (elts == NULL) {
-		ERROR("%p: can't allocate packets array", (void *)txq);
+	/* Allocate Bounce-buf memory */
+	txq->bounce_buf = (char *)rte_zmalloc_socket("TXQ",
+						     MAX_WQE_SIZE,
+						     RTE_CACHE_LINE_MIN_SIZE,
+						     txq->socket);
+
+	if ((elts == NULL) || (txq->bounce_buf == NULL)) {
+		ERROR("%p: can't allocate TXQ memory", (void *)txq);
 		ret = ENOMEM;
 		goto error;
 	}

@@ -110,6 +116,8 @@
 	assert(ret == 0);
 	return 0;
 error:
+	if (txq->bounce_buf != NULL)
+		rte_free(txq->bounce_buf);
 	if (elts != NULL)
 		rte_free(elts);
 	DEBUG("%p: failed, freed everything", (void *)txq);

index e41ea9e..dae0e47 100644
@@ -461,14 +461,16 @@
 	/* Fill-in date from last to first */
 	for (i = wr->num_sge  - 1; i >= 0; --i)
 		mlx4_set_data_seg(dseg + i,  wr->sg_list + i);
-	/* Handle control info
-	 *
-	 * For raw eth, the SOLICIT flag is used to indicate that
-	 * no icrc should be calculated
-	 */
-	srcrb_flags = MLX4_WQE_CTRL_SOLICIT |
-		      ((wr->send_flags & IBV_SEND_SIGNALED) ?
-				      MLX4_WQE_CTRL_CQ_UPDATE : 0);
+	/* Handle control info */
+	/* For raw eth always set the SOLICIT flag */
+	/*  Request Tx completion. */
+	txq->elts_comp_cd -= nr_txbbs;
+	if (unlikely(txq->elts_comp_cd <= 0)) {
+		srcrb_flags = MLX4_WQE_CTRL_SOLICIT | MLX4_WQE_CTRL_CQ_UPDATE;
+		txq->elts_comp_cd = txq->elts_comp_cd_init;
+	} else {
+		srcrb_flags = MLX4_WQE_CTRL_SOLICIT;
+	}
 	fence_size = (wr->send_flags & IBV_SEND_FENCE ?
 		MLX4_WQE_CTRL_FENCE : 0) | ((wqe_real_size / 16) & 0x3f);
 	owner_opcode = MLX4_OPCODE_SEND |

@@ -514,13 +516,12 @@
 	struct ibv_send_wr *wr_bad = NULL;
 	unsigned int elts_head = txq->elts_head;
 	const unsigned int elts_n = txq->elts_n;
-	unsigned int elts_comp_cd = txq->elts_comp_cd;
 	unsigned int elts_comp = 0;
 	unsigned int i;
 	unsigned int max;
 	int err;
 
-	assert(elts_comp_cd != 0);
+	assert(txq->elts_comp_cd != 0);
 	mlx4_txq_complete(txq);
 	max = (elts_n - (elts_head - txq->elts_tail));
 	if (max > elts_n)

@@ -560,11 +561,6 @@
 				tmp = next;
 			} while (tmp != NULL);
 		}
-		/* Request Tx completion. */
-		if (unlikely(--elts_comp_cd == 0)) {
-			elts_comp_cd = txq->elts_comp_cd_init;
-			send_flags |= IBV_SEND_SIGNALED;
-		}
 		if (buf->pkt_len <= txq->max_inline)
 			send_flags |= IBV_SEND_INLINE;
 		/* Update element. */

@@ -580,9 +576,6 @@
 		/* post the pkt for sending */
 		err = mlx4_post_send(txq, buf, wr, &wr_bad);
 		if (unlikely(err)) {
-			if (unlikely(wr_bad->send_flags &
-				     IBV_SEND_SIGNALED))
-				elts_comp_cd = 1;
 			elt->buf = NULL;
 			goto stop;
 		}

@@ -602,7 +595,6 @@
 	mlx4_send_flush(txq);
 	txq->elts_head = elts_head;
 	txq->elts_comp += elts_comp;
-	txq->elts_comp_cd = elts_comp_cd;
 	return i;
 }
 

index 7cae7e2..35e0de7 100644
@@ -140,7 +140,7 @@ struct txq {
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
 	unsigned int elts_comp; /**< Number of pkts waiting for completion. */
-	unsigned int elts_comp_cd; /**< Countdown for next completion. */
+	int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */

index b084903..3149be6 100644
@@ -397,6 +397,7 @@ struct mlx4_conf {
 		.ports.present = 0,
 	};
 	unsigned int vf;
+	unsigned int tunnel_en;
 	int i;
 
 	(void)pci_drv;

@@ -456,6 +457,9 @@ struct mlx4_conf {
 		rte_errno = ENODEV;
 		goto error;
 	}
+	/* Only cx3-pro supports L3 tunneling */
+	tunnel_en = (device_attr.vendor_part_id ==
+		     PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO);
 	INFO("%u port(s) detected", device_attr.phys_port_cnt);
 	conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1;
 	if (mlx4_args(pci_dev->device.devargs, &conf)) {

@@ -529,6 +533,9 @@ struct mlx4_conf {
 		priv->pd = pd;
 		priv->mtu = ETHER_MTU;
 		priv->vf = vf;
+		priv->tunnel_en = tunnel_en;
+		priv->hw_csum =
+		     !!(device_attr.device_cap_flags & IBV_DEVICE_RAW_IP_CSUM);
 		/* Configure the first MAC address by default. */
 		if (mlx4_get_mac(priv, &mac.addr_bytes)) {
 			ERROR("cannot get MAC address, is mlx4_en loaded?"

index 93e5502..439a828 100644
@@ -104,6 +104,8 @@ struct priv {
 	unsigned int vf:1; /* This is a VF device. */
 	unsigned int intr_alarm:1; /* An interrupt alarm is scheduled. */
 	unsigned int isolated:1; /* Toggle isolated mode. */
+	unsigned int hw_csum:1; /* Checksum offload is supported. */
+	unsigned int tunnel_en:1; /* Device tunneling is enabled */
 	struct rte_intr_handle intr_handle; /* Port interrupt handle. */
 	struct rte_flow_drop *flow_drop_queue; /* Flow drop queue. */
 	LIST_HEAD(mlx4_flows, rte_flow) flows;

index a9e8059..e4ecbfa 100644
@@ -553,6 +553,12 @@
 	info->max_mac_addrs = 1;
 	info->rx_offload_capa = 0;
 	info->tx_offload_capa = 0;
+	if (priv->hw_csum)
+		info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
+					  DEV_TX_OFFLOAD_UDP_CKSUM  |
+					  DEV_TX_OFFLOAD_TCP_CKSUM);
+	if (priv->tunnel_en)
+		info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 	if (mlx4_get_ifname(priv, &ifname) == 0)
 		info->if_index = if_nametoindex(ifname);
 	info->speed_capa =

index 8b0248a..38e9a45 100644
@@ -60,6 +60,8 @@
 /* WQE flags */
 #define MLX4_OPCODE_SEND	0x0a
 #define MLX4_EN_BIT_WQE_OWN	0x80000000
+#define MLX4_WQE_CTRL_IIP_HDR_CSUM (1 << 28)
+#define MLX4_WQE_CTRL_IL4_HDR_CSUM (1 << 27)
 
 #define SIZE_TO_TXBBS(size)	(RTE_ALIGN((size), (TXBB_SIZE)) / (TXBB_SIZE))
 

index dae0e47..3415f63 100644
@@ -475,9 +475,27 @@
 		MLX4_WQE_CTRL_FENCE : 0) | ((wqe_real_size / 16) & 0x3f);
 	owner_opcode = MLX4_OPCODE_SEND |
 		       ((sq->head & sq->txbb_cnt) ? MLX4_EN_BIT_WQE_OWN : 0);
+	/* Should we enable HW CKSUM offload ? */
+	if (txq->priv->hw_csum &&
+	    (pkt->ol_flags &
+	    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))) {
+		const uint64_t is_tunneled = pkt->ol_flags &
+					     (PKT_TX_TUNNEL_GRE |
+					      PKT_TX_TUNNEL_VXLAN);
+
+		if (is_tunneled && txq->tunnel_en) {
+			owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
+					MLX4_WQE_CTRL_IL4_HDR_CSUM;
+			if (pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM;
+		} else {
+			srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM |
+				      MLX4_WQE_CTRL_TCP_UDP_CSUM;
+		}
+	}
 	/* fill in ctrl info but ownership */
 	mlx4_set_ctrl_seg(ctrl, fence_size, srcrb_flags, 0);
-       /* If we used a bounce buffer then copy wqe back into sq */
+	/* If we used a bounce buffer then copy wqe back into sq */
 	if (unlikely(bounce))
 		ctrl = mlx4_bounce_to_desc(txq, head_idx, wqe_size);
 	/*

@@ -485,9 +503,8 @@
 	 * setting ownership bit (because HW can start
 	 * executing as soon as we do).
 	 */
-	 rte_wmb();
-	 ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode);
-
+	rte_wmb();
+	ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode);
 	sq->head += nr_txbbs;
 	rte_wmb();
 	return 0;

index 35e0de7..b4675b7 100644
@@ -146,6 +146,8 @@ struct txq {
 	unsigned int socket; /**< CPU socket ID for allocations. */
 	struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
 	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
+	uint16_t tunnel_en:1;
+	/* When set TX offload for tunneled packets are supported. */
 	char *bounce_buf; /**< Side memory to be used when wqe wraps around */
 };
 

index 6f6ea9c..cecd5e8 100644
@@ -306,7 +306,7 @@ struct txq_mp2mr_mbuf_check_data {
 	struct mlx4dv_obj mlxdv;
 	struct mlx4dv_qp dv_qp;
 	struct mlx4dv_cq dv_cq;
-		struct txq tmpl = {
+	struct txq tmpl = {
 		.priv = priv,
 		.socket = socket
 	};

@@ -334,6 +334,8 @@ struct txq_mp2mr_mbuf_check_data {
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	if (priv->tunnel_en)
+		tmpl.tunnel_en = 1;
 	DEBUG("priv->device_attr.max_qp_wr is %d",
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",

index 38e9a45..e328cff 100644
@@ -168,7 +168,7 @@
 		  uint32_t srcrb_flags, uint32_t imm)
 {
 	seg->fence_size = fence_size;
-	seg->srcrb_flags = rte_cpu_to_be_32(srcrb_flags);
+	seg->srcrb_flags = srcrb_flags;
 	/*
 	 * The caller should prepare "imm" in advance based on WR opcode.
 	 * For IBV_WR_SEND_WITH_IMM and IBV_WR_RDMA_WRITE_WITH_IMM,

index 3415f63..ed19c72 100644
@@ -426,7 +426,11 @@
 	struct mlx4_wqe_data_seg *dseg;
 	struct mlx4_sq *sq = &txq->msq;
 	struct ibv_sge sge[wr->num_sge];
-	uint32_t srcrb_flags;
+	union {
+		uint32_t flags;
+		uint16_t flags16[2];
+	} srcrb;
+	uint32_t imm = 0;
 	uint8_t fence_size;
 	uint32_t head_idx = sq->head & sq->txbb_cnt_mask;
 	uint32_t owner_opcode;

@@ -466,10 +470,10 @@
 	/*  Request Tx completion. */
 	txq->elts_comp_cd -= nr_txbbs;
 	if (unlikely(txq->elts_comp_cd <= 0)) {
-		srcrb_flags = MLX4_WQE_CTRL_SOLICIT | MLX4_WQE_CTRL_CQ_UPDATE;
+		srcrb.flags = MLX4_WQE_CTRL_SOLICIT | MLX4_WQE_CTRL_CQ_UPDATE;
 		txq->elts_comp_cd = txq->elts_comp_cd_init;
 	} else {
-		srcrb_flags = MLX4_WQE_CTRL_SOLICIT;
+		srcrb.flags = MLX4_WQE_CTRL_SOLICIT;
 	}
 	fence_size = (wr->send_flags & IBV_SEND_FENCE ?
 		MLX4_WQE_CTRL_FENCE : 0) | ((wqe_real_size / 16) & 0x3f);

@@ -487,14 +491,26 @@
 			owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
 					MLX4_WQE_CTRL_IL4_HDR_CSUM;
 			if (pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM)
-				srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM;
+				srcrb.flags |= MLX4_WQE_CTRL_IP_HDR_CSUM;
 		} else {
-			srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM |
+			srcrb.flags |= MLX4_WQE_CTRL_IP_HDR_CSUM |
 				      MLX4_WQE_CTRL_TCP_UDP_CSUM;
 		}
 	}
+	/* convert flags to BE before adding the mac address (if at all)
+	 * to it
+	 */
+	srcrb.flags = rte_cpu_to_be_32(srcrb.flags);
+	/* Copy dst mac address to wqe. This allows loopback in eSwitch,
+	 * so that VFs and PF can communicate with each other
+	 */
+	if (txq->lb) {
+		srcrb.flags16[0] = *(rte_pktmbuf_mtod(pkt, uint16_t *));
+		imm = *(rte_pktmbuf_mtod_offset(pkt, uint32_t *,
+						sizeof(uint16_t)));
+	}
 	/* fill in ctrl info but ownership */
-	mlx4_set_ctrl_seg(ctrl, fence_size, srcrb_flags, 0);
+	mlx4_set_ctrl_seg(ctrl, fence_size, srcrb.flags, imm);
 	/* If we used a bounce buffer then copy wqe back into sq */
 	if (unlikely(bounce))
 		ctrl = mlx4_bounce_to_desc(txq, head_idx, wqe_size);

index b4675b7..8e407f5 100644
@@ -148,6 +148,8 @@ struct txq {
 	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 	uint16_t tunnel_en:1;
 	/* When set TX offload for tunneled packets are supported. */
+	uint16_t lb:1;
+	/* Whether pkts should be looped-back by eswitch or not */
 	char *bounce_buf; /**< Side memory to be used when wqe wraps around */
 };
 

index cecd5e8..296d72d 100644
@@ -410,6 +410,8 @@ struct txq_mp2mr_mbuf_check_data {
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	/* If a VF device - need to loopback xmitted packets */
+	tmpl.lb = !!(priv->vf);
 	/* Clean up txq in case we're reinitializing it. */
 	DEBUG("%p: cleaning-up old txq just in case", (void *)txq);
 	mlx4_txq_cleanup(txq);

new file mode 100644
index 0000000..6d1800a
@@ -0,0 +1,108 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RTE_PMD_MLX4_PRM_H_
+#define RTE_PMD_MLX4_PRM_H_
+
+#include <rte_byteorder.h>
+#include <rte_branch_prediction.h>
+#include <rte_atomic.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx4dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* ConnectX-3 Tx queue basic block. */
+#define MLX4_TXBB_SHIFT 6
+#define MLX4_TXBB_SIZE (1 << MLX4_TXBB_SHIFT)
+
+/* Typical TSO descriptor with 16 gather entries is 352 bytes. */
+#define MLX4_MAX_WQE_SIZE 512
+#define MLX4_MAX_WQE_TXBBS (MLX4_MAX_WQE_SIZE / MLX4_TXBB_SIZE)
+
+/* Send queue stamping/invalidating information. */
+#define MLX4_SQ_STAMP_STRIDE 64
+#define MLX4_SQ_STAMP_DWORDS (MLX4_SQ_STAMP_STRIDE / 4)
+#define MLX4_SQ_STAMP_SHIFT 31
+#define MLX4_SQ_STAMP_VAL 0x7fffffff
+
+/* Work queue element (WQE) flags. */
+#define MLX4_BIT_WQE_OWN 0x80000000
+
+#define MLX4_SIZE_TO_TXBBS(size) \
+		(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))
+
+/* Send queue information. */
+struct mlx4_sq {
+	char *buf; /**< SQ buffer. */
+	char *eob; /**< End of SQ buffer */
+	uint32_t head; /**< SQ head counter in units of TXBBS. */
+	uint32_t tail; /**< SQ tail counter in units of TXBBS. */
+	uint32_t txbb_cnt; /**< Num of WQEBB in the Q (should be ^2). */
+	uint32_t txbb_cnt_mask; /**< txbbs_cnt mask (txbb_cnt is ^2). */
+	uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free. */
+	uint32_t *db; /**< Pointer to the doorbell. */
+	uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */
+};
+
+#define mlx4_get_send_wqe(sq, n) ((sq)->buf + ((n) * (MLX4_TXBB_SIZE)))
+
+/* Completion queue information. */
+struct mlx4_cq {
+	char *buf; /**< Pointer to the completion queue buffer. */
+	uint32_t cqe_cnt; /**< Number of entries in the queue. */
+	uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */
+	uint32_t cons_index; /**< Last queue entry that was handled. */
+	uint32_t *set_ci_db; /**< Pointer to the completion queue doorbell. */
+};
+
+/*
+ * cqe = cq->buf + cons_index * cqe_size + cqe_offset
+ * Where cqe_size is 32 or 64 bytes and
+ * cqe_offset is 0 or 32 (depending on cqe_size).
+ */
+#define mlx4_get_cqe(cq, n) (__extension__({ \
+				typeof(cq) q = (cq); \
+				(q)->buf + \
+				(((n) & ((q)->cqe_cnt - 1)) << \
+				 (5 + (q)->cqe_64)) + \
+				((q)->cqe_64 << 5); \
+			    }))
+
+#endif /* RTE_PMD_MLX4_PRM_H_ */

445 lines not on this page · raw message

index fec998a..b515472 100644
@@ -40,6 +40,7 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"

@@ -50,6 +51,7 @@
 #include <rte_mempool.h>
 
 #include "mlx4.h"
+#include "mlx4_prm.h"
 
 /** Rx queue counters. */
 struct mlx4_rxq_stats {

@@ -85,8 +87,6 @@ struct rxq {
 
 /** Tx element. */
 struct txq_elt {
-	struct ibv_send_wr wr; /* Work request. */
-	struct ibv_sge sge; /* Scatter/gather element. */
 	struct rte_mbuf *buf; /**< Buffer. */
 };
 

@@ -100,24 +100,28 @@ struct mlx4_txq_stats {
 
 /** Tx queue descriptor. */
 struct txq {
-	struct priv *priv; /**< Back pointer to private data. */
-	struct {
-		const struct rte_mempool *mp; /**< Cached memory pool. */
-		struct ibv_mr *mr; /**< Memory region (for mp). */
-		uint32_t lkey; /**< mr->lkey copy. */
-	} mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
-	struct ibv_cq *cq; /**< Completion queue. */
-	struct ibv_qp *qp; /**< Queue pair. */
-	uint32_t max_inline; /**< Max inline send size. */
-	unsigned int elts_n; /**< (*elts)[] length. */
-	struct txq_elt (*elts)[]; /**< Tx elements. */
+	struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
+	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
 	unsigned int elts_comp; /**< Number of completion requests. */
 	unsigned int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
+	unsigned int elts_n; /**< (*elts)[] length. */
+	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
-	unsigned int socket; /**< CPU socket ID for allocations. */
+	uint32_t max_inline; /**< Max inline send size. */
+	struct {
+		const struct rte_mempool *mp; /**< Cached memory pool. */
+		struct ibv_mr *mr; /**< Memory region (for mp). */
+		uint32_t lkey; /**< mr->lkey copy. */
+	} mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
+	struct {
+		struct priv *priv; /**< Back pointer to private data. */
+		unsigned int socket; /**< CPU socket ID for allocations. */
+		struct ibv_cq *cq; /**< Completion queue. */
+		struct ibv_qp *qp; /**< Queue pair. */
+	} ctrl;
 };
 
 /* mlx4_rxq.c */

170 lines not on this page · raw message

index 93e5502..b6e1ef2 100644
@@ -57,17 +57,6 @@
 /* Maximum size for inline data. */
 #define MLX4_PMD_MAX_INLINE 0
 
-/*
- * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP
- * from which buffers are to be transmitted will have to be mapped by this
- * driver to their own Memory Region (MR). This is a slow operation.
- *
- * This value is always 1 for RX queues.
- */
-#ifndef MLX4_PMD_TX_MP_CACHE
-#define MLX4_PMD_TX_MP_CACHE 8
-#endif
-
 /* Interrupt alarm timeout value in microseconds. */
 #define MLX4_INTR_ALARM_TIMEOUT 100000
 

291 lines not on this page · raw message

252 lines not on this page · raw message

index b515472..df83552 100644
@@ -62,13 +62,6 @@ struct mlx4_rxq_stats {
 	uint64_t rx_nombuf; /**< Total of Rx mbuf allocation failures. */
 };
 
-/** Rx element. */
-struct rxq_elt {
-	struct ibv_recv_wr wr; /**< Work request. */
-	struct ibv_sge sge; /**< Scatter/gather element. */
-	struct rte_mbuf *buf; /**< Buffer. */
-};
-
 /** Rx queue descriptor. */
 struct rxq {
 	struct priv *priv; /**< Back pointer to private data. */

@@ -78,9 +71,15 @@ struct rxq {
 	struct ibv_qp *qp; /**< Queue pair. */
 	struct ibv_comp_channel *channel; /**< Rx completion channel. */
 	unsigned int port_id; /**< Port ID for incoming packets. */
-	unsigned int elts_n; /**< (*elts)[] length. */
-	unsigned int elts_head; /**< Current index in (*elts)[]. */
-	struct rxq_elt (*elts)[]; /**< Rx elements. */
+	unsigned int elts_n; /**< Log 2 of Mbufs. */
+	struct rte_mbuf *(*elts)[]; /**< Rx elements. */
+	struct {
+		volatile struct mlx4_wqe_data_seg(*wqes)[];
+		volatile uint32_t *rq_db;
+		uint16_t rq_ci;
+	} hw;
+	struct mlx4_cq mcq;  /**< Info for directly manipulating the CQ. */
+	unsigned int sge_n; /**< Log 2 of SGEs number. */
 	struct mlx4_rxq_stats stats; /**< Rx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
 };

index 0fbdc71..d6f729f 100644
@@ -108,4 +108,24 @@
 
 int mlx4_fd_set_non_blocking(int fd);
 
+/**
+ * Return nearest power of two above input value.
+ *
+ * @param v
+ *   Input value.
+ *
+ * @return
+ *   Nearest power of two above input value.
+ */
+static inline unsigned int
+log2above(unsigned int v)
+{
+	unsigned int l;
+	unsigned int r;
+
+	for (l = 0, r = 0; (v >> 1); ++l, v >>= 1)
+		r |= (v & 1);
+	return l + r;
+}
+
 #endif /* MLX4_UTILS_H_ */

index e45bb3b..4200716 100644
@@ -63,6 +63,16 @@
 #include "mlx4_rxtx.h"
 #include "mlx4_utils.h"
 
+/*
+ * Pointer-value pair structure
+ * used in tx_post_send for saving the first DWORD (32 byte)
+ * of a TXBB0
+ */
+struct pv {
+	struct mlx4_wqe_data_seg *dseg;
+	uint32_t val;
+};
+
 /**
  * Stamp a WQE so it won't be reused by the HW.
  * Routine is used when freeing WQE used by the chip or when failing

@@ -296,34 +306,38 @@
  *
  * @param txq
  *   The Tx queue to post to.
- * @param wr
- *   The work request to handle.
- * @param bad_wr
- *   The wr in case that posting had failed.
+ * @param pkt
+ *   The packet to transmit.
  *
  * @return
  *   0 - success, negative errno value otherwise and rte_errno is set.
  */
 static inline int
 mlx4_post_send(struct txq *txq,
-	       struct rte_mbuf *pkt,
-	       uint32_t send_flags)
+	       struct rte_mbuf *pkt)
 {
 	struct mlx4_wqe_ctrl_seg *ctrl;
 	struct mlx4_wqe_data_seg *dseg;
 	struct mlx4_sq *sq = &txq->msq;
+	struct rte_mbuf *buf;
 	uint32_t head_idx = sq->head & sq->txbb_cnt_mask;
 	uint32_t lkey;
 	uintptr_t addr;
+	uint32_t srcrb_flags;
+	uint32_t owner_opcode = MLX4_OPCODE_SEND;
+	uint32_t byte_count;
 	int wqe_real_size;
 	int nr_txbbs;
 	int rc;
+	struct pv *pv = (struct pv *)txq->bounce_buf;
+	int pv_counter = 0;
 
 	/* Calculate the needed work queue entry size for this packet. */
 	wqe_real_size = sizeof(struct mlx4_wqe_ctrl_seg) +
 			pkt->nb_segs * sizeof(struct mlx4_wqe_data_seg);
 	nr_txbbs = MLX4_SIZE_TO_TXBBS(wqe_real_size);
-	/* Check that there is room for this WQE in the send queue and
+	/*
+	 * Check that there is room for this WQE in the send queue and
 	 * that the WQE size is legal.
 	 */
 	if (likely(((sq->head - sq->tail) + nr_txbbs +

@@ -332,76 +346,108 @@
 		rc = ENOSPC;
 		goto err;
 	}
-	/* Get the control and single-data entries of the WQE */
+	/* Get the control and data entries of the WQE. */
 	ctrl = (struct mlx4_wqe_ctrl_seg *)mlx4_get_send_wqe(sq, head_idx);
 	dseg = (struct mlx4_wqe_data_seg *)(((char *)ctrl) +
 		sizeof(struct mlx4_wqe_ctrl_seg));
-	/*
-	 * Fill the data segment with buffer information.
-	 */
-	addr = rte_pktmbuf_mtod(pkt, uintptr_t);
-	rte_prefetch0((volatile void *)addr);
-	dseg->addr = rte_cpu_to_be_64(addr);
-	/* Memory region key for this memory pool. */
-	lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(pkt));
-	if (unlikely(lkey == (uint32_t)-1)) {
-		/* MR does not exist. */
-		DEBUG("%p: unable to get MP <-> MR"
-		      " association", (void *)txq);
-		/*
-		 * Restamp entry in case of failure.
-		 * Make sure that size is written correctly.
-		 * Note that we give ownership to the SW, not the HW.
+	/* Fill the data segments with buffer information. */
+	for (buf = pkt; buf != NULL; buf = buf->next, dseg++) {
+		addr = rte_pktmbuf_mtod(buf, uintptr_t);
+		rte_prefetch0((volatile void *)addr);
+		/* Handle WQE wraparound. */
+		if (unlikely(dseg >= (struct mlx4_wqe_data_seg *)sq->eob))
+			dseg = (struct mlx4_wqe_data_seg *)sq->buf;
+		dseg->addr = rte_cpu_to_be_64(addr);
+		/* Memory region key for this memory pool. */
+		lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
+		if (unlikely(lkey == (uint32_t)-1)) {
+			/* MR does not exist. */
+			DEBUG("%p: unable to get MP <-> MR"
+			      " association", (void *)txq);
+			/*
+			 * Restamp entry in case of failure.
+			 * Make sure that size is written correctly
+			 * Note that we give ownership to the SW, not the HW.
+			 */
+			ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
+			mlx4_txq_stamp_freed_wqe(sq, head_idx,
+				     (sq->head & sq->txbb_cnt) ? 0 : 1);
+			rc = EFAULT;
+			goto err;
+		}
+		dseg->lkey = rte_cpu_to_be_32(lkey);
+		if (likely(buf->data_len))
+			byte_count = rte_cpu_to_be_32(buf->data_len);
+		else
+			/*
+			 * Zero length segment is treated as inline segment
+			 * with zero data.
+			 */
+			byte_count = RTE_BE32(0x80000000);
+		/* If the data segment is not at the beginning of a
+		 * Tx basic block(TXBB) then write the byte count,
+		 * else postpone the writing to just before updating the
+		 * control segment.
 		 */
-		ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
-		mlx4_txq_stamp_freed_wqe(sq, head_idx,
-					 (sq->head & sq->txbb_cnt) ? 0 : 1);
-		rc = EFAULT;
-		goto err;
+		if ((uintptr_t)dseg & (uintptr_t)(MLX4_TXBB_SIZE - 1)) {
+			/*
+			 * Need a barrier here before writing the byte_count
+			 * fields to make sure that all the data is visible
+			 * before the byte_count field is set.
+			 * Otherwise, if the segment begins a new cacheline,
+			 * the HCA prefetcher could grab the 64-byte chunk and
+			 * get a valid (!= * 0xffffffff) byte count but stale
+			 * data, and end up sending the wrong data.
+			 */
+			rte_io_wmb();
+			dseg->byte_count = byte_count;
+		} else {
+			/*
+			 * This data segment starts at the beginning of a new
+			 * TXBB, so we need to postpone its byte_count writing
+			 * for later.
+			 */
+			pv[pv_counter].dseg = dseg;
+			pv[pv_counter++].val = byte_count;
+		}
 	}
-	dseg->lkey = rte_cpu_to_be_32(lkey);
-	/*
-	 * Need a barrier here before writing the byte_count field to
-	 * make sure that all the data is visible before the
-	 * byte_count field is set.  Otherwise, if the segment begins
-	 * a new cacheline, the HCA prefetcher could grab the 64-byte
-	 * chunk and get a valid (!= * 0xffffffff) byte count but
-	 * stale data, and end up sending the wrong data.
-	 */
-	rte_io_wmb();
-	if (likely(pkt->data_len))
-		dseg->byte_count = rte_cpu_to_be_32(pkt->data_len);
-	else
-		/*
-		 * Zero length segment is treated as inline segment
-		 * with zero data.
-		 */
-		dseg->byte_count = RTE_BE32(0x80000000);
-	/*
-	 * Fill the control parameters for this packet.
-	 * For raw Ethernet, the SOLICIT flag is used to indicate that no icrc
-	 * should be calculated
-	 */
-	ctrl->srcrb_flags =
-		rte_cpu_to_be_32(MLX4_WQE_CTRL_SOLICIT |
-				 (send_flags & MLX4_WQE_CTRL_CQ_UPDATE));
+	/* Write the first DWORD of each TXBB save earlier. */
+	if (pv_counter) {
+		/* Need a barrier here before writing the byte_count. */
+		rte_io_wmb();
+		for (--pv_counter; pv_counter  >= 0; pv_counter--)
+			pv[pv_counter].dseg->byte_count = pv[pv_counter].val;
+	}
+	/* Fill the control parameters for this packet. */
 	ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
 	/*
 	 * The caller should prepare "imm" in advance in order to support
 	 * VF to VF communication (when the device is a virtual-function
 	 * device (VF)).
-	 */
+	*/
 	ctrl->imm = 0;
 	/*
+	 * For raw Ethernet, the SOLICIT flag is used to indicate that no icrc
+	 * should be calculated.
+	 */
+	txq->elts_comp_cd -= nr_txbbs;
+	if (unlikely(txq->elts_comp_cd <= 0)) {
+		txq->elts_comp_cd = txq->elts_comp_cd_init;
+		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
+				       MLX4_WQE_CTRL_CQ_UPDATE);
+	} else {
+		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
+	}
+	ctrl->srcrb_flags = srcrb_flags;
+	/*
 	 * Make sure descriptor is fully written before
 	 * setting ownership bit (because HW can start
 	 * executing as soon as we do).
 	 */
-	rte_wmb();
-	ctrl->owner_opcode =
-		rte_cpu_to_be_32(MLX4_OPCODE_SEND |
-				 ((sq->head & sq->txbb_cnt) ?
-				  MLX4_BIT_WQE_OWN : 0));
+	 rte_wmb();
+	 ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode |
+					       ((sq->head & sq->txbb_cnt) ?
+					       MLX4_BIT_WQE_OWN : 0));
 	sq->head += nr_txbbs;
 	return 0;
 err:

@@ -428,14 +474,13 @@
 	struct txq *txq = (struct txq *)dpdk_txq;
 	unsigned int elts_head = txq->elts_head;
 	const unsigned int elts_n = txq->elts_n;
-	unsigned int elts_comp_cd = txq->elts_comp_cd;
 	unsigned int elts_comp = 0;
 	unsigned int bytes_sent = 0;
 	unsigned int i;
 	unsigned int max;
 	int err;
 
-	assert(elts_comp_cd != 0);
+	assert(txq->elts_comp_cd != 0);
 	mlx4_txq_complete(txq);
 	max = (elts_n - (elts_head - txq->elts_tail));
 	if (max > elts_n)

@@ -454,8 +499,6 @@
 			(((elts_head + 1) == elts_n) ? 0 : elts_head + 1);
 		struct txq_elt *elt_next = &(*txq->elts)[elts_head_next];
 		struct txq_elt *elt = &(*txq->elts)[elts_head];
-		unsigned int segs = buf->nb_segs;
-		uint32_t send_flags = 0;
 
 		/* Clean up old buffer. */
 		if (likely(elt->buf != NULL)) {

@@ -473,34 +516,16 @@
 				tmp = next;
 			} while (tmp != NULL);
 		}
-		/* Request Tx completion. */
-		if (unlikely(--elts_comp_cd == 0)) {
-			elts_comp_cd = txq->elts_comp_cd_init;
-			++elts_comp;
-			send_flags |= MLX4_WQE_CTRL_CQ_UPDATE;
-		}
-		if (likely(segs == 1)) {
-			/* Update element. */
-			elt->buf = buf;
-			RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
-			/* post the pkt for sending */
-			err = mlx4_post_send(txq, buf, send_flags);
-			if (unlikely(err)) {
-				if (unlikely(send_flags &
-					     MLX4_WQE_CTRL_CQ_UPDATE)) {
-					elts_comp_cd = 1;
-					--elts_comp;
-				}
-				elt->buf = NULL;
-				goto stop;
-			}
-			elt->buf = buf;
-			bytes_sent += buf->pkt_len;
-		} else {
-			err = -EINVAL;
-			rte_errno = -err;
+		RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
+		/* post the packet for sending. */
+		err = mlx4_post_send(txq, buf);
+		if (unlikely(err)) {
+			elt->buf = NULL;
 			goto stop;
 		}
+		elt->buf = buf;
+		bytes_sent += buf->pkt_len;
+		++elts_comp;
 		elts_head = elts_head_next;
 	}
 stop:

@@ -516,7 +541,6 @@
 	rte_write32(txq->msq.doorbell_qpn, txq->msq.db);
 	txq->elts_head = elts_head;
 	txq->elts_comp += elts_comp;
-	txq->elts_comp_cd = elts_comp_cd;
 	return i;
 }
 

index df83552..1b90533 100644
@@ -103,13 +103,15 @@ struct txq {
 	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
-	unsigned int elts_comp; /**< Number of completion requests. */
-	unsigned int elts_comp_cd; /**< Countdown for next completion. */
+	unsigned int elts_comp; /**< Number of pkts waiting for completion. */
+	int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
 	unsigned int elts_n; /**< (*elts)[] length. */
 	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	uint32_t max_inline; /**< Max inline send size. */
+	char *bounce_buf;
+	/**< memory used for storing the first DWORD of data TXBBs. */
 	struct {
 		const struct rte_mempool *mp; /**< Cached memory pool. */
 		struct ibv_mr *mr; /**< Memory region (for mp). */

index 492779f..9333311 100644
@@ -83,8 +83,14 @@
 		rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq->ctrl.socket);
 	int ret = 0;
 
-	if (elts == NULL) {
-		ERROR("%p: can't allocate packets array", (void *)txq);
+	/* Allocate Bounce-buf memory */
+	txq->bounce_buf = (char *)rte_zmalloc_socket("TXQ",
+						     MLX4_MAX_WQE_SIZE,
+						     RTE_CACHE_LINE_MIN_SIZE,
+						     txq->ctrl.socket);
+
+	if ((elts == NULL) || (txq->bounce_buf == NULL)) {
+		ERROR("%p: can't allocate TXQ memory", (void *)txq);
 		ret = ENOMEM;
 		goto error;
 	}

@@ -110,6 +116,7 @@
 	assert(ret == 0);
 	return 0;
 error:
+	rte_free(txq->bounce_buf);
 	rte_free(elts);
 	DEBUG("%p: failed, freed everything", (void *)txq);
 	assert(ret > 0);

@@ -303,7 +310,6 @@ struct txq_mp2mr_mbuf_check_data {
 	struct mlx4dv_obj mlxdv;
 	struct mlx4dv_qp dv_qp;
 	struct mlx4dv_cq dv_cq;
-
 	struct txq tmpl = {
 		.ctrl = {
 			.priv = priv,

index b084903..a0e76ee 100644
@@ -397,6 +397,7 @@ struct mlx4_conf {
 		.ports.present = 0,
 	};
 	unsigned int vf;
+	unsigned int tunnel_en;
 	int i;
 
 	(void)pci_drv;

@@ -456,6 +457,9 @@ struct mlx4_conf {
 		rte_errno = ENODEV;
 		goto error;
 	}
+	/* Only cx3-pro supports L3 tunneling */
+	tunnel_en = (device_attr.vendor_part_id ==
+		     PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO);
 	INFO("%u port(s) detected", device_attr.phys_port_cnt);
 	conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1;
 	if (mlx4_args(pci_dev->device.devargs, &conf)) {

@@ -529,6 +533,11 @@ struct mlx4_conf {
 		priv->pd = pd;
 		priv->mtu = ETHER_MTU;
 		priv->vf = vf;
+		priv->hw_csum =
+		     !!(device_attr.device_cap_flags & IBV_DEVICE_RAW_IP_CSUM);
+		priv->hw_csum_l2tun = tunnel_en;
+		DEBUG("L2 tunnel checksum offloads are %ssupported",
+		      (priv->hw_csum_l2tun ? "" : "not "));
 		/* Configure the first MAC address by default. */
 		if (mlx4_get_mac(priv, &mac.addr_bytes)) {
 			ERROR("cannot get MAC address, is mlx4_en loaded?"

index b6e1ef2..d0bce91 100644
@@ -93,6 +93,8 @@ struct priv {
 	unsigned int vf:1; /* This is a VF device. */
 	unsigned int intr_alarm:1; /* An interrupt alarm is scheduled. */
 	unsigned int isolated:1; /* Toggle isolated mode. */
+	unsigned int hw_csum:1; /* Checksum offload is supported. */
+	unsigned int hw_csum_l2tun:1; /* Checksum support for L2 tunnels. */
 	struct rte_intr_handle intr_handle; /* Port interrupt handle. */
 	struct rte_flow_drop *flow_drop_queue; /* Flow drop queue. */
 	LIST_HEAD(mlx4_flows, rte_flow) flows;

index a9e8059..95cc6e4 100644
@@ -553,6 +553,12 @@
 	info->max_mac_addrs = 1;
 	info->rx_offload_capa = 0;
 	info->tx_offload_capa = 0;
+	if (priv->hw_csum)
+		info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
+					  DEV_TX_OFFLOAD_UDP_CKSUM  |
+					  DEV_TX_OFFLOAD_TCP_CKSUM);
+	if (priv->hw_csum_l2tun)
+		info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 	if (mlx4_get_ifname(priv, &ifname) == 0)
 		info->if_index = if_nametoindex(ifname);
 	info->speed_capa =

index 6d1800a..57f5a46 100644
@@ -64,6 +64,8 @@
 
 /* Work queue element (WQE) flags. */
 #define MLX4_BIT_WQE_OWN 0x80000000
+#define MLX4_WQE_CTRL_IIP_HDR_CSUM (1 << 28)
+#define MLX4_WQE_CTRL_IL4_HDR_CSUM (1 << 27)
 
 #define MLX4_SIZE_TO_TXBBS(size) \
 		(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))

index 4200716..2757aec 100644
@@ -433,12 +433,29 @@ struct pv {
 	txq->elts_comp_cd -= nr_txbbs;
 	if (unlikely(txq->elts_comp_cd <= 0)) {
 		txq->elts_comp_cd = txq->elts_comp_cd_init;
-		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
-				       MLX4_WQE_CTRL_CQ_UPDATE);
+		srcrb_flags = MLX4_WQE_CTRL_SOLICIT | MLX4_WQE_CTRL_CQ_UPDATE;
 	} else {
-		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
+		srcrb_flags = MLX4_WQE_CTRL_SOLICIT;
 	}
-	ctrl->srcrb_flags = srcrb_flags;
+	/* Enable HW checksum offload if requested */
+	if (txq->csum &&
+	    (pkt->ol_flags &
+	     (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))) {
+		const uint64_t is_tunneled = pkt->ol_flags &
+					     (PKT_TX_TUNNEL_GRE |
+					      PKT_TX_TUNNEL_VXLAN);
+
+		if (is_tunneled && txq->csum_l2tun) {
+			owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
+					MLX4_WQE_CTRL_IL4_HDR_CSUM;
+			if (pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM;
+		} else {
+			srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM |
+				      MLX4_WQE_CTRL_TCP_UDP_CSUM;
+		}
+	}
+	ctrl->srcrb_flags = rte_cpu_to_be_32(srcrb_flags);
 	/*
 	 * Make sure descriptor is fully written before
 	 * setting ownership bit (because HW can start

index 1b90533..dc283e1 100644
@@ -110,6 +110,8 @@ struct txq {
 	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	uint32_t max_inline; /**< Max inline send size. */
+	uint32_t csum:1; /**< Checksum is supported and enabled */
+	uint32_t csum_l2tun:1; /**< L2 tun Checksum is supported and enabled */
 	char *bounce_buf;
 	/**< memory used for storing the first DWORD of data TXBBs. */
 	struct {

index 9333311..2d776eb 100644
@@ -340,6 +340,8 @@ struct txq_mp2mr_mbuf_check_data {
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	tmpl.csum = priv->hw_csum;
+	tmpl.csum_l2tun = priv->hw_csum_l2tun;
 	DEBUG("priv->device_attr.max_qp_wr is %d",
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",

index a0e76ee..865ffdd 100644
@@ -535,6 +535,8 @@ struct mlx4_conf {
 		priv->vf = vf;
 		priv->hw_csum =
 		     !!(device_attr.device_cap_flags & IBV_DEVICE_RAW_IP_CSUM);
+		DEBUG("checksum offloading is %ssupported",
+		      (priv->hw_csum ? "" : "not "));
 		priv->hw_csum_l2tun = tunnel_en;
 		DEBUG("L2 tunnel checksum offloads are %ssupported",
 		      (priv->hw_csum_l2tun ? "" : "not "));

index 95cc6e4..6dbf273 100644
@@ -553,10 +553,14 @@
 	info->max_mac_addrs = 1;
 	info->rx_offload_capa = 0;
 	info->tx_offload_capa = 0;
-	if (priv->hw_csum)
+	if (priv->hw_csum) {
 		info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
-					  DEV_TX_OFFLOAD_UDP_CKSUM  |
+					  DEV_TX_OFFLOAD_UDP_CKSUM |
 					  DEV_TX_OFFLOAD_TCP_CKSUM);
+		info->rx_offload_capa |= (DEV_RX_OFFLOAD_IPV4_CKSUM |
+					  DEV_RX_OFFLOAD_UDP_CKSUM |
+					  DEV_RX_OFFLOAD_TCP_CKSUM);
+	}
 	if (priv->hw_csum_l2tun)
 		info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 	if (mlx4_get_ifname(priv, &ifname) == 0)

index 57f5a46..b4c14b9 100644
@@ -70,6 +70,25 @@
 #define MLX4_SIZE_TO_TXBBS(size) \
 		(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))
 
+/* Generic macro to convert MLX4 to IBV flags. */
+#define MLX4_TRANSPOSE(val, from, to) \
+		(__extension__({ \
+			typeof(val) _val = (val); \
+			typeof(from) _from = (from); \
+			typeof(to) _to = (to); \
+			(((_from) >= (_to)) ? \
+			(((_val) & (_from)) / ((_from) / (_to))) : \
+			(((_val) & (_from)) * ((_to) / (_from)))); \
+		}))
+
+/* CQE checksum flags */
+enum {
+	MLX4_CQE_L2_TUNNEL_IPV4    = (int)(1U << 25),
+	MLX4_CQE_L2_TUNNEL_L4_CSUM = (int)(1U << 26),
+	MLX4_CQE_L2_TUNNEL         = (int)(1U << 27),
+	MLX4_CQE_L2_TUNNEL_IPOK    = (int)(1U << 31),
+};
+
 /* Send queue information. */
 struct mlx4_sq {
 	char *buf; /**< SQ buffer. */

index d7447c4..053318f 100644
@@ -267,6 +267,11 @@
 	int ret;
 
 	(void)conf; /* Thresholds configuration (ignored). */
+	/* Toggle Rx checksum offload if hardware supports it. */
+	if (priv->hw_csum)
+		tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+	if (priv->hw_csum_l2tun)
+		tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
 	mb_len = rte_pktmbuf_data_room_size(mp);
 	/* Enable scattered packets support for this queue if necessary. */
 	assert(mb_len >= RTE_PKTMBUF_HEADROOM);

144 lines not on this page · raw message

index dc283e1..75c98c1 100644
@@ -80,6 +80,8 @@ struct rxq {
 	} hw;
 	struct mlx4_cq mcq;  /**< Info for directly manipulating the CQ. */
 	unsigned int sge_n; /**< Log 2 of SGEs number. */
+	unsigned int csum:1; /**< Enable checksum offloading. */
+	unsigned int csum_l2tun:1; /**< Enable checksum for L2 tunnels. */
 	struct mlx4_rxq_stats stats; /**< Rx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
 };

index 1e91aaf..85fb6d7 100644
@@ -320,10 +320,13 @@ struct pv {
 	struct mlx4_wqe_data_seg *dseg;
 	struct mlx4_sq *sq = &txq->msq;
 	struct rte_mbuf *buf;
+	union {
+		uint32_t flags;
+		uint16_t flags16[2];
+	} srcrb;
 	uint32_t head_idx = sq->head & sq->txbb_cnt_mask;
 	uint32_t lkey;
 	uintptr_t addr;
-	uint32_t srcrb_flags;
 	uint32_t owner_opcode = MLX4_OPCODE_SEND;
 	uint32_t byte_count;
 	int wqe_real_size;

@@ -421,21 +424,15 @@ struct pv {
 	/* Fill the control parameters for this packet. */
 	ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
 	/*
-	 * The caller should prepare "imm" in advance in order to support
-	 * VF to VF communication (when the device is a virtual-function
-	 * device (VF)).
-	*/
-	ctrl->imm = 0;
-	/*
 	 * For raw Ethernet, the SOLICIT flag is used to indicate that no icrc
 	 * should be calculated.
 	 */
 	txq->elts_comp_cd -= nr_txbbs;
 	if (unlikely(txq->elts_comp_cd <= 0)) {
 		txq->elts_comp_cd = txq->elts_comp_cd_init;
-		srcrb_flags = MLX4_WQE_CTRL_SOLICIT | MLX4_WQE_CTRL_CQ_UPDATE;
+		srcrb.flags = MLX4_WQE_CTRL_SOLICIT | MLX4_WQE_CTRL_CQ_UPDATE;
 	} else {
-		srcrb_flags = MLX4_WQE_CTRL_SOLICIT;
+		srcrb.flags = MLX4_WQE_CTRL_SOLICIT;
 	}
 	/* Enable HW checksum offload if requested */
 	if (txq->csum &&

@@ -449,13 +446,30 @@ struct pv {
 			owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
 					MLX4_WQE_CTRL_IL4_HDR_CSUM;
 			if (pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM)
-				srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM;
+				srcrb.flags |= MLX4_WQE_CTRL_IP_HDR_CSUM;
 		} else {
-			srcrb_flags |= MLX4_WQE_CTRL_IP_HDR_CSUM |
+			srcrb.flags |= MLX4_WQE_CTRL_IP_HDR_CSUM |
 				      MLX4_WQE_CTRL_TCP_UDP_CSUM;
 		}
 	}
-	ctrl->srcrb_flags = rte_cpu_to_be_32(srcrb_flags);
+	/*
+	 * convert flags to BE before adding the mac address (if at all)
+	 * to it
+	 */
+	srcrb.flags = rte_cpu_to_be_32(srcrb.flags);
+	if (txq->lb) {
+		/*
+		 * Copy destination mac address to the wqe,
+		 * this allows loopback in eSwitch, so that VFs and PF
+		 * can communicate with each other.
+		 */
+		srcrb.flags16[0] = *(rte_pktmbuf_mtod(pkt, uint16_t *));
+		ctrl->imm = *(rte_pktmbuf_mtod_offset(pkt, uint32_t *,
+						      sizeof(uint16_t)));
+	} else {
+		ctrl->imm = 0;
+	}
+	ctrl->srcrb_flags = srcrb.flags;
 	/*
 	 * Make sure descriptor is fully written before
 	 * setting ownership bit (because HW can start

index 75c98c1..6f33d1c 100644
@@ -114,6 +114,8 @@ struct txq {
 	uint32_t max_inline; /**< Max inline send size. */
 	uint32_t csum:1; /**< Checksum is supported and enabled */
 	uint32_t csum_l2tun:1; /**< L2 tun Checksum is supported and enabled */
+	uint32_t lb:1;
+	/**< Whether pkts should be looped-back by eswitch or not */
 	char *bounce_buf;
 	/**< memory used for storing the first DWORD of data TXBBs. */
 	struct {

index 2d776eb..fd1dce0 100644
@@ -415,6 +415,8 @@ struct txq_mp2mr_mbuf_check_data {
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	/* If a VF device - need to loopback xmitted packets */
+	tmpl.lb = !!(priv->vf);
 	/* Clean up txq in case we're reinitializing it. */
 	DEBUG("%p: cleaning-up old txq just in case", (void *)txq);
 	mlx4_txq_cleanup(txq);

new file mode 100644
index 0000000..085a595
@@ -0,0 +1,120 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MLX4_PRM_H_
+#define MLX4_PRM_H_
+
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_byteorder.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx4dv.h>
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* ConnectX-3 Tx queue basic block. */
+#define MLX4_TXBB_SHIFT 6
+#define MLX4_TXBB_SIZE (1 << MLX4_TXBB_SHIFT)
+
+/* Typical TSO descriptor with 16 gather entries is 352 bytes. */
+#define MLX4_MAX_WQE_SIZE 512
+#define MLX4_MAX_WQE_TXBBS (MLX4_MAX_WQE_SIZE / MLX4_TXBB_SIZE)
+
+/* Send queue stamping/invalidating information. */
+#define MLX4_SQ_STAMP_STRIDE 64
+#define MLX4_SQ_STAMP_DWORDS (MLX4_SQ_STAMP_STRIDE / 4)
+#define MLX4_SQ_STAMP_SHIFT 31
+#define MLX4_SQ_STAMP_VAL 0x7fffffff
+
+/* Work queue element (WQE) flags. */
+#define MLX4_BIT_WQE_OWN 0x80000000
+
+#define MLX4_SIZE_TO_TXBBS(size) \
+	(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))
+
+/* Send queue information. */
+struct mlx4_sq {
+	uint8_t *buf; /**< SQ buffer. */
+	uint8_t *eob; /**< End of SQ buffer */
+	uint32_t head; /**< SQ head counter in units of TXBBS. */
+	uint32_t tail; /**< SQ tail counter in units of TXBBS. */
+	uint32_t txbb_cnt; /**< Num of WQEBB in the Q (should be ^2). */
+	uint32_t txbb_cnt_mask; /**< txbbs_cnt mask (txbb_cnt is ^2). */
+	uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free. */
+	uint32_t *db; /**< Pointer to the doorbell. */
+	uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */
+};
+
+#define mlx4_get_send_wqe(sq, n) ((sq)->buf + ((n) * (MLX4_TXBB_SIZE)))
+
+/* Completion queue information. */
+struct mlx4_cq {
+	uint8_t *buf; /**< Pointer to the completion queue buffer. */
+	uint32_t cqe_cnt; /**< Number of entries in the queue. */
+	uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */
+	uint32_t cons_index; /**< Last queue entry that was handled. */
+	uint32_t *set_ci_db; /**< Pointer to the completion queue doorbell. */
+};
+
+/**
+ * Retrieve a CQE entry from a CQ.
+ *
+ * cqe = cq->buf + cons_index * cqe_size + cqe_offset
+ *
+ * Where cqe_size is 32 or 64 bytes and cqe_offset is 0 or 32 (depending on
+ * cqe_size).
+ *
+ * @param cq
+ *   CQ to retrieve entry from.
+ * @param index
+ *   Entry index.
+ *
+ * @return
+ *   Pointer to CQE entry.
+ */
+static inline struct mlx4_cqe *
+mlx4_get_cqe(struct mlx4_cq *cq, uint32_t index)
+{
+	return (struct mlx4_cqe *)(cq->buf +
+				   ((index & (cq->cqe_cnt - 1)) <<
+				    (5 + cq->cqe_64)) +
+				   (cq->cqe_64 << 5));
+}
+
+#endif /* MLX4_PRM_H_ */

411 lines not on this page · raw message

index fec998a..cc5951c 100644
@@ -40,6 +40,7 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"

@@ -50,6 +51,7 @@
 #include <rte_mempool.h>
 
 #include "mlx4.h"
+#include "mlx4_prm.h"
 
 /** Rx queue counters. */
 struct mlx4_rxq_stats {

@@ -85,8 +87,6 @@ struct rxq {
 
 /** Tx element. */
 struct txq_elt {
-	struct ibv_send_wr wr; /* Work request. */
-	struct ibv_sge sge; /* Scatter/gather element. */
 	struct rte_mbuf *buf; /**< Buffer. */
 };
 

@@ -100,24 +100,26 @@ struct mlx4_txq_stats {
 
 /** Tx queue descriptor. */
 struct txq {
-	struct priv *priv; /**< Back pointer to private data. */
-	struct {
-		const struct rte_mempool *mp; /**< Cached memory pool. */
-		struct ibv_mr *mr; /**< Memory region (for mp). */
-		uint32_t lkey; /**< mr->lkey copy. */
-	} mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
-	struct ibv_cq *cq; /**< Completion queue. */
-	struct ibv_qp *qp; /**< Queue pair. */
-	uint32_t max_inline; /**< Max inline send size. */
-	unsigned int elts_n; /**< (*elts)[] length. */
-	struct txq_elt (*elts)[]; /**< Tx elements. */
+	struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
+	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
 	unsigned int elts_comp; /**< Number of completion requests. */
 	unsigned int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
+	unsigned int elts_n; /**< (*elts)[] length. */
+	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
+	uint32_t max_inline; /**< Max inline send size. */
+	struct {
+		const struct rte_mempool *mp; /**< Cached memory pool. */
+		struct ibv_mr *mr; /**< Memory region (for mp). */
+		uint32_t lkey; /**< mr->lkey copy. */
+	} mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
+	struct priv *priv; /**< Back pointer to private data. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
+	struct ibv_cq *cq; /**< Completion queue. */
+	struct ibv_qp *qp; /**< Queue pair. */
 };
 
 /* mlx4_rxq.c */

75 lines not on this page · raw message

index 409983f..44d095d 100644
@@ -51,6 +51,7 @@
 #pragma GCC diagnostic error "-Wpedantic"
 #endif
 
+#include <rte_byteorder.h>
 #include <rte_common.h>
 #include <rte_errno.h>
 #include <rte_ethdev.h>

@@ -77,20 +78,18 @@ static int
 mlx4_rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n)
 {
 	unsigned int i;
-	struct rxq_elt (*elts)[elts_n] =
-		rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0,
-				  rxq->socket);
+	const uint32_t sges_n = 1 << rxq->sges_n;
+	struct rte_mbuf *(*elts)[elts_n] =
+		rte_calloc_socket("RXQ", 1, sizeof(*elts), 0, rxq->socket);
 
+	assert(rte_is_power_of_2(elts_n));
 	if (elts == NULL) {
 		rte_errno = ENOMEM;
 		ERROR("%p: can't allocate packets array", (void *)rxq);
 		goto error;
 	}
-	/* For each WR (packet). */
 	for (i = 0; (i != elts_n); ++i) {
-		struct rxq_elt *elt = &(*elts)[i];
-		struct ibv_recv_wr *wr = &elt->wr;
-		struct ibv_sge *sge = &(*elts)[i].sge;
+		volatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[i];
 		struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp);
 
 		if (buf == NULL) {

@@ -98,37 +97,35 @@ mlx4_rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n)
 			ERROR("%p: empty mbuf pool", (void *)rxq);
 			goto error;
 		}
-		elt->buf = buf;
-		wr->next = &(*elts)[(i + 1)].wr;
-		wr->sg_list = sge;
-		wr->num_sge = 1;
 		/* Headroom is reserved by rte_pktmbuf_alloc(). */
 		assert(buf->data_off == RTE_PKTMBUF_HEADROOM);
 		/* Buffer is supposed to be empty. */
 		assert(rte_pktmbuf_data_len(buf) == 0);
 		assert(rte_pktmbuf_pkt_len(buf) == 0);
-		/* sge->addr must be able to store a pointer. */
-		assert(sizeof(sge->addr) >= sizeof(uintptr_t));
-		/* SGE keeps its headroom. */
-		sge->addr = (uintptr_t)
-			((uint8_t *)buf->buf_addr + RTE_PKTMBUF_HEADROOM);
-		sge->length = (buf->buf_len - RTE_PKTMBUF_HEADROOM);
-		sge->lkey = rxq->mr->lkey;
-		/* Redundant check for tailroom. */
-		assert(sge->length == rte_pktmbuf_tailroom(buf));
+		/* Only the first segment keeps headroom. */
+		if (i % sges_n)
+			buf->data_off = 0;
+		buf->port = rxq->port_id;
+		buf->data_len = rte_pktmbuf_tailroom(buf);
+		buf->pkt_len = rte_pktmbuf_tailroom(buf);
+		buf->nb_segs = 1;
+		*scat = (struct mlx4_wqe_data_seg){
+			.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
+								  uintptr_t)),
+			.byte_count = rte_cpu_to_be_32(buf->data_len),
+			.lkey = rte_cpu_to_be_32(rxq->mr->lkey),
+		};
+		(*elts)[i] = buf;
 	}
-	/* The last WR pointer must be NULL. */
-	(*elts)[(i - 1)].wr.next = NULL;
-	DEBUG("%p: allocated and configured %u single-segment WRs",
-	      (void *)rxq, elts_n);
-	rxq->elts_n = elts_n;
-	rxq->elts_head = 0;
+	DEBUG("%p: allocated and configured %u segments (max %u packets)",
+	      (void *)rxq, elts_n, elts_n >> rxq->sges_n);
+	rxq->elts_n = log2above(elts_n);
 	rxq->elts = elts;
 	return 0;
 error:
 	if (elts != NULL) {
 		for (i = 0; (i != RTE_DIM(*elts)); ++i)
-			rte_pktmbuf_free_seg((*elts)[i].buf);
+			rte_pktmbuf_free_seg((*rxq->elts)[i]);
 		rte_free(elts);
 	}
 	DEBUG("%p: failed, freed everything", (void *)rxq);

@@ -146,17 +143,16 @@ static void
 mlx4_rxq_free_elts(struct rxq *rxq)
 {
 	unsigned int i;
-	unsigned int elts_n = rxq->elts_n;
-	struct rxq_elt (*elts)[elts_n] = rxq->elts;
 
-	DEBUG("%p: freeing WRs", (void *)rxq);
+	if (rxq->elts == NULL)
+		return;
+	DEBUG("%p: freeing Rx queue elements", (void *)rxq);
+	for (i = 0; i != (1u << rxq->elts_n); ++i)
+		if ((*rxq->elts)[i] != NULL)
+			rte_pktmbuf_free_seg((*rxq->elts)[i]);
+	rte_free(rxq->elts);
 	rxq->elts_n = 0;
 	rxq->elts = NULL;
-	if (elts == NULL)
-		return;
-	for (i = 0; (i != RTE_DIM(*elts)); ++i)
-		rte_pktmbuf_free_seg((*elts)[i].buf);
-	rte_free(elts);
 }
 
 /**

@@ -193,12 +189,15 @@ mlx4_rxq_cleanup(struct rxq *rxq)
  *   Completion queue to associate with QP.
  * @param desc
  *   Number of descriptors in QP (hint only).
+ * @param sges_n
+ *   Maximum number of segments per packet.
  *
  * @return
  *   QP pointer or NULL in case of error and rte_errno is set.
  */
 static struct ibv_qp *
-mlx4_rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc)
+mlx4_rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
+		  uint32_t sges_n)
 {
 	struct ibv_qp *qp;
 	struct ibv_qp_init_attr attr = {

@@ -211,8 +210,8 @@ mlx4_rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc)
 			.max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ?
 					priv->device_attr.max_qp_wr :
 					desc),
-			/* Max number of scatter/gather elements in a WR. */
-			.max_recv_sge = 1,
+			/* Maximum number of segments per packet. */
+			.max_recv_sge = sges_n,
 		},
 		.qp_type = IBV_QPT_RAW_PACKET,
 	};

@@ -248,13 +247,15 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	       struct rte_mempool *mp)
 {
 	struct priv *priv = dev->data->dev_private;
+	struct mlx4dv_obj mlxdv;
+	struct mlx4dv_qp dv_qp;
+	struct mlx4dv_cq dv_cq;
 	struct rxq tmpl = {
 		.priv = priv,
 		.mp = mp,
 		.socket = socket
 	};
 	struct ibv_qp_attr mod;
-	struct ibv_recv_wr *bad_wr;
 	unsigned int mb_len;
 	int ret;
 

@@ -269,11 +270,31 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	assert(mb_len >= RTE_PKTMBUF_HEADROOM);
 	if (dev->data->dev_conf.rxmode.max_rx_pkt_len <=
 	    (mb_len - RTE_PKTMBUF_HEADROOM)) {
-		;
+		tmpl.sges_n = 0;
 	} else if (dev->data->dev_conf.rxmode.enable_scatter) {
-		WARN("%p: scattered mode has been requested but is"
-		     " not supported, this may lead to packet loss",
-		     (void *)dev);
+		uint32_t size =
+			RTE_PKTMBUF_HEADROOM +
+			dev->data->dev_conf.rxmode.max_rx_pkt_len;
+		uint32_t sges_n;
+
+		/*
+		 * Determine the number of SGEs needed for a full packet
+		 * and round it to the next power of two.
+		 */
+		sges_n = log2above((size / mb_len) + !!(size % mb_len));
+		tmpl.sges_n = sges_n;
+		/* Make sure sges_n did not overflow. */
+		size = mb_len * (1 << tmpl.sges_n);
+		size -= RTE_PKTMBUF_HEADROOM;
+		if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) {
+			rte_errno = EOVERFLOW;
+			ERROR("%p: too many SGEs (%u) needed to handle"
+			      " requested maximum packet size %u",
+			      (void *)dev,
+			      1 << sges_n,
+			      dev->data->dev_conf.rxmode.max_rx_pkt_len);
+			goto error;
+		}
 	} else {
 		WARN("%p: the requested maximum Rx packet size (%u) is"
 		     " larger than a single mbuf (%u) and scattered"

@@ -282,6 +303,17 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		     dev->data->dev_conf.rxmode.max_rx_pkt_len,
 		     mb_len - RTE_PKTMBUF_HEADROOM);
 	}
+	DEBUG("%p: maximum number of segments per packet: %u",
+	      (void *)dev, 1 << tmpl.sges_n);
+	if (desc % (1 << tmpl.sges_n)) {
+		rte_errno = EINVAL;
+		ERROR("%p: number of RX queue descriptors (%u) is not a"
+		      " multiple of maximum segments per packet (%u)",
+		      (void *)dev,
+		      desc,
+		      1 << tmpl.sges_n);
+		goto error;
+	}
 	/* Use the entire Rx mempool as the memory region. */
 	tmpl.mr = mlx4_mp2mr(priv->pd, mp);
 	if (tmpl.mr == NULL) {

@@ -306,7 +338,8 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 			goto error;
 		}
 	}
-	tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0);
+	tmpl.cq = ibv_create_cq(priv->ctx, desc >> tmpl.sges_n, NULL,
+				tmpl.channel, 0);
 	if (tmpl.cq == NULL) {
 		rte_errno = ENOMEM;
 		ERROR("%p: CQ creation failure: %s",

@@ -317,7 +350,8 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",
 	      priv->device_attr.max_sge);
-	tmpl.qp = mlx4_rxq_setup_qp(priv, tmpl.cq, desc);
+	tmpl.qp = mlx4_rxq_setup_qp(priv, tmpl.cq, desc >> tmpl.sges_n,
+				    1 << tmpl.sges_n);
 	if (tmpl.qp == NULL) {
 		ERROR("%p: QP creation failure: %s",
 		      (void *)dev, strerror(rte_errno));

@@ -336,21 +370,6 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
-	ret = mlx4_rxq_alloc_elts(&tmpl, desc);
-	if (ret) {
-		ERROR("%p: RXQ allocation failed: %s",
-		      (void *)dev, strerror(rte_errno));
-		goto error;
-	}
-	ret = ibv_post_recv(tmpl.qp, &(*tmpl.elts)[0].wr, &bad_wr);
-	if (ret) {
-		rte_errno = ret;
-		ERROR("%p: ibv_post_recv() failed for WR %p: %s",
-		      (void *)dev,
-		      (void *)bad_wr,
-		      strerror(rte_errno));
-		goto error;
-	}
 	mod = (struct ibv_qp_attr){
 		.qp_state = IBV_QPS_RTR
 	};

@@ -361,14 +380,43 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	/* Retrieve device queue information. */
+	mlxdv.cq.in = tmpl.cq;
+	mlxdv.cq.out = &dv_cq;
+	mlxdv.qp.in = tmpl.qp;
+	mlxdv.qp.out = &dv_qp;
+	ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_QP | MLX4DV_OBJ_CQ);
+	if (ret) {
+		ERROR("%p: failed to obtain device information", (void *)dev);
+		goto error;
+	}
+	tmpl.wqes =
+		(volatile struct mlx4_wqe_data_seg (*)[])
+		((uintptr_t)dv_qp.buf.buf + dv_qp.rq.offset);
+	tmpl.rq_db = dv_qp.rdb;
+	tmpl.rq_ci = 0;
+	tmpl.mcq.buf = dv_cq.buf.buf;
+	tmpl.mcq.cqe_cnt = dv_cq.cqe_cnt;
+	tmpl.mcq.set_ci_db = dv_cq.set_ci_db;
+	tmpl.mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0;
 	/* Save port ID. */
 	tmpl.port_id = dev->data->port_id;
 	DEBUG("%p: RTE port ID: %u", (void *)rxq, tmpl.port_id);
+	ret = mlx4_rxq_alloc_elts(&tmpl, desc);
+	if (ret) {
+		ERROR("%p: RXQ allocation failed: %s",
+		      (void *)dev, strerror(rte_errno));
+		goto error;
+	}
 	/* Clean up rxq in case we're reinitializing it. */
 	DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq);
 	mlx4_rxq_cleanup(rxq);
 	*rxq = tmpl;
 	DEBUG("%p: rxq updated with %p", (void *)rxq, (void *)&tmpl);
+	/* Update doorbell counter. */
+	rxq->rq_ci = desc >> rxq->sges_n;
+	rte_wmb();
+	*rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
 	return 0;
 error:
 	ret = rte_errno;

@@ -406,6 +454,12 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	struct rxq *rxq = dev->data->rx_queues[idx];
 	int ret;
 
+	if (!rte_is_power_of_2(desc)) {
+		desc = 1 << log2above(desc);
+		WARN("%p: increased number of descriptors in RX queue %u"
+		     " to the next power of two (%d)",
+		     (void *)dev, idx, desc);
+	}
 	DEBUG("%p: configuring queue %u for %u descriptors",
 	      (void *)dev, idx, desc);
 	if (idx >= dev->data->nb_rx_queues) {

249 lines not on this page · raw message

index cc5951c..ac84177 100644
@@ -62,13 +62,6 @@ struct mlx4_rxq_stats {
 	uint64_t rx_nombuf; /**< Total of Rx mbuf allocation failures. */
 };
 
-/** Rx element. */
-struct rxq_elt {
-	struct ibv_recv_wr wr; /**< Work request. */
-	struct ibv_sge sge; /**< Scatter/gather element. */
-	struct rte_mbuf *buf; /**< Buffer. */
-};
-
 /** Rx queue descriptor. */
 struct rxq {
 	struct priv *priv; /**< Back pointer to private data. */

@@ -77,10 +70,14 @@ struct rxq {
 	struct ibv_cq *cq; /**< Completion queue. */
 	struct ibv_qp *qp; /**< Queue pair. */
 	struct ibv_comp_channel *channel; /**< Rx completion channel. */
-	unsigned int port_id; /**< Port ID for incoming packets. */
-	unsigned int elts_n; /**< (*elts)[] length. */
-	unsigned int elts_head; /**< Current index in (*elts)[]. */
-	struct rxq_elt (*elts)[]; /**< Rx elements. */
+	uint16_t rq_ci; /**< Saved RQ consumer index. */
+	uint16_t port_id; /**< Port ID for incoming packets. */
+	uint16_t sges_n; /**< Number of segments per packet (log2 value). */
+	uint16_t elts_n; /**< Mbuf queue size (log2 value). */
+	struct rte_mbuf *(*elts)[]; /**< Rx elements. */
+	volatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */
+	volatile uint32_t *rq_db; /**< RQ doorbell record. */
+	struct mlx4_cq mcq;  /**< Info for directly manipulating the CQ. */
 	struct mlx4_rxq_stats stats; /**< Rx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
 };

index 0fbdc71..d6f729f 100644
@@ -108,4 +108,24 @@ pmd_drv_log_basename(const char *s)
 
 int mlx4_fd_set_non_blocking(int fd);
 
+/**
+ * Return nearest power of two above input value.
+ *
+ * @param v
+ *   Input value.
+ *
+ * @return
+ *   Nearest power of two above input value.
+ */
+static inline unsigned int
+log2above(unsigned int v)
+{
+	unsigned int l;
+	unsigned int r;
+
+	for (l = 0, r = 0; (v >> 1); ++l, v >>= 1)
+		r |= (v & 1);
+	return l + r;
+}
+
 #endif /* MLX4_UTILS_H_ */

index fd8ef7b..cc0baaa 100644
@@ -63,6 +63,15 @@
 #include "mlx4_utils.h"
 
 /**
+ * Pointer-value pair structure used in tx_post_send for saving the first
+ * DWORD (32 byte) of a TXBB.
+ */
+struct pv {
+	struct mlx4_wqe_data_seg *dseg;
+	uint32_t val;
+};
+
+/**
  * Stamp a WQE so it won't be reused by the HW.
  *
  * Routine is used when freeing WQE used by the chip or when failing

@@ -291,24 +300,28 @@ mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
  *   Target Tx queue.
  * @param pkt
  *   Packet to transmit.
- * @param send_flags
- *   @p MLX4_WQE_CTRL_CQ_UPDATE to request completion on this packet.
  *
  * @return
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 static inline int
-mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt, uint32_t send_flags)
+mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt)
 {
 	struct mlx4_wqe_ctrl_seg *ctrl;
 	struct mlx4_wqe_data_seg *dseg;
 	struct mlx4_sq *sq = &txq->msq;
+	struct rte_mbuf *buf;
 	uint32_t head_idx = sq->head & sq->txbb_cnt_mask;
 	uint32_t lkey;
 	uintptr_t addr;
+	uint32_t srcrb_flags;
+	uint32_t owner_opcode = MLX4_OPCODE_SEND;
+	uint32_t byte_count;
 	int wqe_real_size;
 	int nr_txbbs;
 	int rc;
+	struct pv *pv = (struct pv *)txq->bounce_buf;
+	int pv_counter = 0;
 
 	/* Calculate the needed work queue entry size for this packet. */
 	wqe_real_size = sizeof(struct mlx4_wqe_ctrl_seg) +

@@ -324,56 +337,81 @@ mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt, uint32_t send_flags)
 		rc = ENOSPC;
 		goto err;
 	}
-	/* Get the control and single-data entries of the WQE. */
+	/* Get the control and data entries of the WQE. */
 	ctrl = (struct mlx4_wqe_ctrl_seg *)mlx4_get_send_wqe(sq, head_idx);
 	dseg = (struct mlx4_wqe_data_seg *)((uintptr_t)ctrl +
 					    sizeof(struct mlx4_wqe_ctrl_seg));
-	/* Fill the data segment with buffer information. */
-	addr = rte_pktmbuf_mtod(pkt, uintptr_t);
-	rte_prefetch0((volatile void *)addr);
-	dseg->addr = rte_cpu_to_be_64(addr);
-	/* Memory region key for this memory pool. */
-	lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(pkt));
-	if (unlikely(lkey == (uint32_t)-1)) {
-		/* MR does not exist. */
-		DEBUG("%p: unable to get MP <-> MR association", (void *)txq);
+	/* Fill the data segments with buffer information. */
+	for (buf = pkt; buf != NULL; buf = buf->next, dseg++) {
+		addr = rte_pktmbuf_mtod(buf, uintptr_t);
+		rte_prefetch0((volatile void *)addr);
+		/* Handle WQE wraparound. */
+		if (unlikely(dseg >= (struct mlx4_wqe_data_seg *)sq->eob))
+			dseg = (struct mlx4_wqe_data_seg *)sq->buf;
+		dseg->addr = rte_cpu_to_be_64(addr);
+		/* Memory region key for this memory pool. */
+		lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
+		if (unlikely(lkey == (uint32_t)-1)) {
+			/* MR does not exist. */
+			DEBUG("%p: unable to get MP <-> MR association",
+			      (void *)txq);
+			/*
+			 * Restamp entry in case of failure.
+			 * Make sure that size is written correctly
+			 * Note that we give ownership to the SW, not the HW.
+			 */
+			ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
+			mlx4_txq_stamp_freed_wqe(sq, head_idx,
+				     (sq->head & sq->txbb_cnt) ? 0 : 1);
+			rc = EFAULT;
+			goto err;
+		}
+		dseg->lkey = rte_cpu_to_be_32(lkey);
+		if (likely(buf->data_len)) {
+			byte_count = rte_cpu_to_be_32(buf->data_len);
+		} else {
+			/*
+			 * Zero length segment is treated as inline segment
+			 * with zero data.
+			 */
+			byte_count = RTE_BE32(0x80000000);
+		}
 		/*
-		 * Restamp entry in case of failure, make sure that size is
-		 * written correctly.
-		 * Note that we give ownership to the SW, not the HW.
+		 * If the data segment is not at the beginning of a
+		 * Tx basic block (TXBB) then write the byte count,
+		 * else postpone the writing to just before updating the
+		 * control segment.
 		 */
-		ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
-		mlx4_txq_stamp_freed_wqe(sq, head_idx,
-					 (sq->head & sq->txbb_cnt) ? 0 : 1);
-		rc = EFAULT;
-		goto err;
+		if ((uintptr_t)dseg & (uintptr_t)(MLX4_TXBB_SIZE - 1)) {
+			/*
+			 * Need a barrier here before writing the byte_count
+			 * fields to make sure that all the data is visible
+			 * before the byte_count field is set.
+			 * Otherwise, if the segment begins a new cacheline,
+			 * the HCA prefetcher could grab the 64-byte chunk and
+			 * get a valid (!= 0xffffffff) byte count but stale
+			 * data, and end up sending the wrong data.
+			 */
+			rte_io_wmb();
+			dseg->byte_count = byte_count;
+		} else {
+			/*
+			 * This data segment starts at the beginning of a new
+			 * TXBB, so we need to postpone its byte_count writing
+			 * for later.
+			 */
+			pv[pv_counter].dseg = dseg;
+			pv[pv_counter++].val = byte_count;
+		}
 	}
-	dseg->lkey = rte_cpu_to_be_32(lkey);
-	/*
-	 * Need a barrier here before writing the byte_count field to
-	 * make sure that all the data is visible before the
-	 * byte_count field is set. Otherwise, if the segment begins
-	 * a new cache line, the HCA prefetcher could grab the 64-byte
-	 * chunk and get a valid (!= 0xffffffff) byte count but
-	 * stale data, and end up sending the wrong data.
-	 */
-	rte_io_wmb();
-	if (likely(pkt->data_len))
-		dseg->byte_count = rte_cpu_to_be_32(pkt->data_len);
-	else
-		/*
-		 * Zero length segment is treated as inline segment
-		 * with zero data.
-		 */
-		dseg->byte_count = RTE_BE32(0x80000000);
-	/*
-	 * Fill the control parameters for this packet.
-	 * For raw Ethernet, the SOLICIT flag is used to indicate that no ICRC
-	 * should be calculated.
-	 */
-	ctrl->srcrb_flags =
-		rte_cpu_to_be_32(MLX4_WQE_CTRL_SOLICIT |
-				 (send_flags & MLX4_WQE_CTRL_CQ_UPDATE));
+	/* Write the first DWORD of each TXBB save earlier. */
+	if (pv_counter) {
+		/* Need a barrier here before writing the byte_count. */
+		rte_io_wmb();
+		for (--pv_counter; pv_counter  >= 0; pv_counter--)
+			pv[pv_counter].dseg->byte_count = pv[pv_counter].val;
+	}
+	/* Fill the control parameters for this packet. */
 	ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
 	/*
 	 * The caller should prepare "imm" in advance in order to support

@@ -382,14 +420,27 @@ mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt, uint32_t send_flags)
 	 */
 	ctrl->imm = 0;
 	/*
-	 * Make sure descriptor is fully written before setting ownership
-	 * bit (because HW can start executing as soon as we do).
+	 * For raw Ethernet, the SOLICIT flag is used to indicate that no ICRC
+	 * should be calculated.
+	 */
+	txq->elts_comp_cd -= nr_txbbs;
+	if (unlikely(txq->elts_comp_cd <= 0)) {
+		txq->elts_comp_cd = txq->elts_comp_cd_init;
+		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
+				       MLX4_WQE_CTRL_CQ_UPDATE);
+	} else {
+		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
+	}
+	ctrl->srcrb_flags = srcrb_flags;
+	/*
+	 * Make sure descriptor is fully written before
+	 * setting ownership bit (because HW can start
+	 * executing as soon as we do).
 	 */
 	rte_wmb();
-	ctrl->owner_opcode =
-		rte_cpu_to_be_32(MLX4_OPCODE_SEND |
-				 ((sq->head & sq->txbb_cnt) ?
-				  MLX4_BIT_WQE_OWN : 0));
+	ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode |
+					      ((sq->head & sq->txbb_cnt) ?
+					       MLX4_BIT_WQE_OWN : 0));
 	sq->head += nr_txbbs;
 	return 0;
 err:

@@ -416,14 +467,13 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	struct txq *txq = (struct txq *)dpdk_txq;
 	unsigned int elts_head = txq->elts_head;
 	const unsigned int elts_n = txq->elts_n;
-	unsigned int elts_comp_cd = txq->elts_comp_cd;
 	unsigned int elts_comp = 0;
 	unsigned int bytes_sent = 0;
 	unsigned int i;
 	unsigned int max;
 	int err;
 
-	assert(elts_comp_cd != 0);
+	assert(txq->elts_comp_cd != 0);
 	mlx4_txq_complete(txq);
 	max = (elts_n - (elts_head - txq->elts_tail));
 	if (max > elts_n)

@@ -442,8 +492,6 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 			(((elts_head + 1) == elts_n) ? 0 : elts_head + 1);
 		struct txq_elt *elt_next = &(*txq->elts)[elts_head_next];
 		struct txq_elt *elt = &(*txq->elts)[elts_head];
-		unsigned int segs = buf->nb_segs;
-		uint32_t send_flags = 0;
 
 		/* Clean up old buffer. */
 		if (likely(elt->buf != NULL)) {

@@ -461,34 +509,16 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 				tmp = next;
 			} while (tmp != NULL);
 		}
-		/* Request Tx completion. */
-		if (unlikely(--elts_comp_cd == 0)) {
-			elts_comp_cd = txq->elts_comp_cd_init;
-			++elts_comp;
-			send_flags |= MLX4_WQE_CTRL_CQ_UPDATE;
-		}
-		if (likely(segs == 1)) {
-			/* Update element. */
-			elt->buf = buf;
-			RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
-			/* Post the packet for sending. */
-			err = mlx4_post_send(txq, buf, send_flags);
-			if (unlikely(err)) {
-				if (unlikely(send_flags &
-					     MLX4_WQE_CTRL_CQ_UPDATE)) {
-					elts_comp_cd = 1;
-					--elts_comp;
-				}
-				elt->buf = NULL;
-				goto stop;
-			}
-			elt->buf = buf;
-			bytes_sent += buf->pkt_len;
-		} else {
-			err = -EINVAL;
-			rte_errno = -err;
+		RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
+		/* Post the packet for sending. */
+		err = mlx4_post_send(txq, buf);
+		if (unlikely(err)) {
+			elt->buf = NULL;
 			goto stop;
 		}
+		elt->buf = buf;
+		bytes_sent += buf->pkt_len;
+		++elts_comp;
 		elts_head = elts_head_next;
 	}
 stop:

@@ -504,7 +534,6 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 	rte_write32(txq->msq.doorbell_qpn, txq->msq.db);
 	txq->elts_head = elts_head;
 	txq->elts_comp += elts_comp;
-	txq->elts_comp_cd = elts_comp_cd;
 	return i;
 }
 

index ac84177..528e286 100644
@@ -101,13 +101,15 @@ struct txq {
 	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
-	unsigned int elts_comp; /**< Number of completion requests. */
-	unsigned int elts_comp_cd; /**< Countdown for next completion. */
+	unsigned int elts_comp; /**< Number of packets awaiting completion. */
+	int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
 	unsigned int elts_n; /**< (*elts)[] length. */
 	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	uint32_t max_inline; /**< Max inline send size. */
+	uint8_t *bounce_buf;
+	/**< Memory used for storing the first DWORD of data TXBBs. */
 	struct {
 		const struct rte_mempool *mp; /**< Cached memory pool. */
 		struct ibv_mr *mr; /**< Memory region (for mp). */

index fb28ef2..7552a88 100644
@@ -83,8 +83,13 @@ mlx4_txq_alloc_elts(struct txq *txq, unsigned int elts_n)
 		rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq->socket);
 	int ret = 0;
 
-	if (elts == NULL) {
-		ERROR("%p: can't allocate packets array", (void *)txq);
+	/* Allocate bounce buffer. */
+	txq->bounce_buf = rte_zmalloc_socket("TXQ",
+					     MLX4_MAX_WQE_SIZE,
+					     RTE_CACHE_LINE_MIN_SIZE,
+					     txq->socket);
+	if (!elts || !txq->bounce_buf) {
+		ERROR("%p: can't allocate TXQ memory", (void *)txq);
 		ret = ENOMEM;
 		goto error;
 	}

@@ -110,6 +115,8 @@ mlx4_txq_alloc_elts(struct txq *txq, unsigned int elts_n)
 	assert(ret == 0);
 	return 0;
 error:
+	rte_free(txq->bounce_buf);
+	txq->bounce_buf = NULL;
 	rte_free(elts);
 	DEBUG("%p: failed, freed everything", (void *)txq);
 	assert(ret > 0);

@@ -175,6 +182,7 @@ mlx4_txq_cleanup(struct txq *txq)
 		claim_zero(ibv_destroy_qp(txq->qp));
 	if (txq->cq != NULL)
 		claim_zero(ibv_destroy_cq(txq->cq));
+	rte_free(txq->bounce_buf);
 	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
 		if (txq->mp2mr[i].mp == NULL)
 			break;

index b084903..385ddaa 100644
@@ -529,6 +529,17 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		priv->pd = pd;
 		priv->mtu = ETHER_MTU;
 		priv->vf = vf;
+		priv->hw_csum =	!!(device_attr.device_cap_flags &
+				   IBV_DEVICE_RAW_IP_CSUM);
+		DEBUG("checksum offloading is %ssupported",
+		      (priv->hw_csum ? "" : "not "));
+		/* Only ConnectX-3 Pro supports tunneling. */
+		priv->hw_csum_l2tun =
+			priv->hw_csum &&
+			(device_attr.vendor_part_id ==
+			 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO);
+		DEBUG("L2 tunnel checksum offloads are %ssupported",
+		      (priv->hw_csum_l2tun ? "" : "not "));
 		/* Configure the first MAC address by default. */
 		if (mlx4_get_mac(priv, &mac.addr_bytes)) {
 			ERROR("cannot get MAC address, is mlx4_en loaded?"

index 93e5502..0b71867 100644
@@ -104,6 +104,8 @@ struct priv {
 	unsigned int vf:1; /* This is a VF device. */
 	unsigned int intr_alarm:1; /* An interrupt alarm is scheduled. */
 	unsigned int isolated:1; /* Toggle isolated mode. */
+	unsigned int hw_csum:1; /* Checksum offload is supported. */
+	unsigned int hw_csum_l2tun:1; /* Checksum support for L2 tunnels. */
 	struct rte_intr_handle intr_handle; /* Port interrupt handle. */
 	struct rte_flow_drop *flow_drop_queue; /* Flow drop queue. */
 	LIST_HEAD(mlx4_flows, rte_flow) flows;

index a9e8059..bec1787 100644
@@ -553,6 +553,12 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 	info->max_mac_addrs = 1;
 	info->rx_offload_capa = 0;
 	info->tx_offload_capa = 0;
+	if (priv->hw_csum)
+		info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
+					  DEV_TX_OFFLOAD_UDP_CKSUM |
+					  DEV_TX_OFFLOAD_TCP_CKSUM);
+	if (priv->hw_csum_l2tun)
+		info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 	if (mlx4_get_ifname(priv, &ifname) == 0)
 		info->if_index = if_nametoindex(ifname);
 	info->speed_capa =

index 085a595..df5a6b4 100644
@@ -64,6 +64,8 @@
 
 /* Work queue element (WQE) flags. */
 #define MLX4_BIT_WQE_OWN 0x80000000
+#define MLX4_WQE_CTRL_IIP_HDR_CSUM (1 << 28)
+#define MLX4_WQE_CTRL_IL4_HDR_CSUM (1 << 27)
 
 #define MLX4_SIZE_TO_TXBBS(size) \
 	(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))

index cc0baaa..fe7d5d0 100644
@@ -431,6 +431,25 @@ mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt)
 	} else {
 		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
 	}
+	/* Enable HW checksum offload if requested */
+	if (txq->csum &&
+	    (pkt->ol_flags &
+	     (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))) {
+		const uint64_t is_tunneled = (pkt->ol_flags &
+					      (PKT_TX_TUNNEL_GRE |
+					       PKT_TX_TUNNEL_VXLAN));
+
+		if (is_tunneled && txq->csum_l2tun) {
+			owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
+					MLX4_WQE_CTRL_IL4_HDR_CSUM;
+			if (pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM)
+				srcrb_flags |=
+					RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM);
+		} else {
+			srcrb_flags |= RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM |
+						MLX4_WQE_CTRL_TCP_UDP_CSUM);
+		}
+	}
 	ctrl->srcrb_flags = srcrb_flags;
 	/*
 	 * Make sure descriptor is fully written before

index 528e286..a742f61 100644
@@ -108,6 +108,8 @@ struct txq {
 	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
 	uint32_t max_inline; /**< Max inline send size. */
+	uint32_t csum:1; /**< Enable checksum offloading. */
+	uint32_t csum_l2tun:1; /**< Same for L2 tunnels. */
 	uint8_t *bounce_buf;
 	/**< Memory used for storing the first DWORD of data TXBBs. */
 	struct {

index 7552a88..96429bc 100644
@@ -338,6 +338,8 @@ mlx4_txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	tmpl.csum = priv->hw_csum;
+	tmpl.csum_l2tun = priv->hw_csum_l2tun;
 	DEBUG("priv->device_attr.max_qp_wr is %d",
 	      priv->device_attr.max_qp_wr);
 	DEBUG("priv->device_attr.max_sge is %d",

index bec1787..6dbf273 100644
@@ -553,10 +553,14 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 	info->max_mac_addrs = 1;
 	info->rx_offload_capa = 0;
 	info->tx_offload_capa = 0;
-	if (priv->hw_csum)
+	if (priv->hw_csum) {
 		info->tx_offload_capa |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
 					  DEV_TX_OFFLOAD_UDP_CKSUM |
 					  DEV_TX_OFFLOAD_TCP_CKSUM);
+		info->rx_offload_capa |= (DEV_RX_OFFLOAD_IPV4_CKSUM |
+					  DEV_RX_OFFLOAD_UDP_CKSUM |
+					  DEV_RX_OFFLOAD_TCP_CKSUM);
+	}
 	if (priv->hw_csum_l2tun)
 		info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
 	if (mlx4_get_ifname(priv, &ifname) == 0)

index df5a6b4..0d76a73 100644
@@ -70,6 +70,14 @@
 #define MLX4_SIZE_TO_TXBBS(size) \
 	(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))
 
+/* CQE checksum flags. */
+enum {
+	MLX4_CQE_L2_TUNNEL_IPV4 = (int)(1u << 25),
+	MLX4_CQE_L2_TUNNEL_L4_CSUM = (int)(1u << 26),
+	MLX4_CQE_L2_TUNNEL = (int)(1u << 27),
+	MLX4_CQE_L2_TUNNEL_IPOK = (int)(1u << 31),
+};
+
 /* Send queue information. */
 struct mlx4_sq {
 	uint8_t *buf; /**< SQ buffer. */

@@ -119,4 +127,26 @@ mlx4_get_cqe(struct mlx4_cq *cq, uint32_t index)
 				   (cq->cqe_64 << 5));
 }
 
+/**
+ * Transpose a flag in a value.
+ *
+ * @param val
+ *   Input value.
+ * @param from
+ *   Flag to retrieve from input value.
+ * @param to
+ *   Flag to set in output value.
+ *
+ * @return
+ *   Output value with transposed flag enabled if present on input.
+ */
+static inline uint64_t
+mlx4_transpose(uint64_t val, uint64_t from, uint64_t to)
+{
+	return (from >= to ?
+		(val & from) / (from / to) :
+		(val & from) * (to / from));
+}
+
+
 #endif /* MLX4_PRM_H_ */

index 44d095d..a021a32 100644
@@ -260,6 +260,11 @@ mlx4_rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
 	int ret;
 
 	(void)conf; /* Thresholds configuration (ignored). */
+	/* Toggle Rx checksum offload if hardware supports it. */
+	if (priv->hw_csum)
+		tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
+	if (priv->hw_csum_l2tun)
+		tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum;
 	mb_len = rte_pktmbuf_data_room_size(mp);
 	if (desc == 0) {
 		rte_errno = EINVAL;

130 lines not on this page · raw message

index a742f61..6aad41a 100644
@@ -77,6 +77,8 @@ struct rxq {
 	struct rte_mbuf *(*elts)[]; /**< Rx elements. */
 	volatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */
 	volatile uint32_t *rq_db; /**< RQ doorbell record. */
+	uint32_t csum:1; /**< Enable checksum offloading. */
+	uint32_t csum_l2tun:1; /**< Same for L2 tunnels. */
 	struct mlx4_cq mcq;  /**< Info for directly manipulating the CQ. */
 	struct mlx4_rxq_stats stats; /**< Rx queue counters. */
 	unsigned int socket; /**< CPU socket ID for allocations. */

index 87c5261..36173ad 100644
@@ -311,10 +311,13 @@ mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt)
 	struct mlx4_wqe_data_seg *dseg;
 	struct mlx4_sq *sq = &txq->msq;
 	struct rte_mbuf *buf;
+	union {
+		uint32_t flags;
+		uint16_t flags16[2];
+	} srcrb;
 	uint32_t head_idx = sq->head & sq->txbb_cnt_mask;
 	uint32_t lkey;
 	uintptr_t addr;
-	uint32_t srcrb_flags;
 	uint32_t owner_opcode = MLX4_OPCODE_SEND;
 	uint32_t byte_count;
 	int wqe_real_size;

@@ -414,22 +417,16 @@ mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt)
 	/* Fill the control parameters for this packet. */
 	ctrl->fence_size = (wqe_real_size >> 4) & 0x3f;
 	/*
-	 * The caller should prepare "imm" in advance in order to support
-	 * VF to VF communication (when the device is a virtual-function
-	 * device (VF)).
-	 */
-	ctrl->imm = 0;
-	/*
 	 * For raw Ethernet, the SOLICIT flag is used to indicate that no ICRC
 	 * should be calculated.
 	 */
 	txq->elts_comp_cd -= nr_txbbs;
 	if (unlikely(txq->elts_comp_cd <= 0)) {
 		txq->elts_comp_cd = txq->elts_comp_cd_init;
-		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
+		srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
 				       MLX4_WQE_CTRL_CQ_UPDATE);
 	} else {
-		srcrb_flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
+		srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
 	}
 	/* Enable HW checksum offload if requested */
 	if (txq->csum &&

@@ -443,14 +440,26 @@ mlx4_post_send(struct txq *txq, struct rte_mbuf *pkt)
 			owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
 					MLX4_WQE_CTRL_IL4_HDR_CSUM;
 			if (pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM)
-				srcrb_flags |=
+				srcrb.flags |=
 					RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM);
 		} else {
-			srcrb_flags |= RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM |
+			srcrb.flags |= RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM |
 						MLX4_WQE_CTRL_TCP_UDP_CSUM);
 		}
 	}
-	ctrl->srcrb_flags = srcrb_flags;
+	if (txq->lb) {
+		/*
+		 * Copy destination MAC address to the WQE, this allows
+		 * loopback in eSwitch, so that VFs and PF can communicate
+		 * with each other.
+		 */
+		srcrb.flags16[0] = *(rte_pktmbuf_mtod(pkt, uint16_t *));
+		ctrl->imm = *(rte_pktmbuf_mtod_offset(pkt, uint32_t *,
+						      sizeof(uint16_t)));
+	} else {
+		ctrl->imm = 0;
+	}
+	ctrl->srcrb_flags = srcrb.flags;
 	/*
 	 * Make sure descriptor is fully written before
 	 * setting ownership bit (because HW can start

index 6aad41a..37f31f4 100644
@@ -112,6 +112,7 @@ struct txq {
 	uint32_t max_inline; /**< Max inline send size. */
 	uint32_t csum:1; /**< Enable checksum offloading. */
 	uint32_t csum_l2tun:1; /**< Same for L2 tunnels. */
+	uint32_t lb:1; /**< Whether packets should be looped back by eSwitch. */
 	uint8_t *bounce_buf;
 	/**< Memory used for storing the first DWORD of data TXBBs. */
 	struct {

index 96429bc..9d1be95 100644
@@ -412,6 +412,8 @@ mlx4_txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc,
 		      (void *)dev, strerror(rte_errno));
 		goto error;
 	}
+	/* Enable Tx loopback for VF devices. */
+	tmpl.lb = !!(priv->vf);
 	/* Clean up txq in case we're reinitializing it. */
 	DEBUG("%p: cleaning-up old txq just in case", (void *)txq);
 	mlx4_txq_cleanup(txq);

new file mode 100644
index 0000000..085a595
@@ -0,0 +1,120 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright 2017 6WIND S.A.
+ *   Copyright 2017 Mellanox
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of 6WIND S.A. nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MLX4_PRM_H_
+#define MLX4_PRM_H_
+
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_byteorder.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/mlx4dv.h>
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+/* ConnectX-3 Tx queue basic block. */
+#define MLX4_TXBB_SHIFT 6
+#define MLX4_TXBB_SIZE (1 << MLX4_TXBB_SHIFT)
+
+/* Typical TSO descriptor with 16 gather entries is 352 bytes. */
+#define MLX4_MAX_WQE_SIZE 512
+#define MLX4_MAX_WQE_TXBBS (MLX4_MAX_WQE_SIZE / MLX4_TXBB_SIZE)
+
+/* Send queue stamping/invalidating information. */
+#define MLX4_SQ_STAMP_STRIDE 64
+#define MLX4_SQ_STAMP_DWORDS (MLX4_SQ_STAMP_STRIDE / 4)
+#define MLX4_SQ_STAMP_SHIFT 31
+#define MLX4_SQ_STAMP_VAL 0x7fffffff
+
+/* Work queue element (WQE) flags. */
+#define MLX4_BIT_WQE_OWN 0x80000000
+
+#define MLX4_SIZE_TO_TXBBS(size) \
+	(RTE_ALIGN((size), (MLX4_TXBB_SIZE)) >> (MLX4_TXBB_SHIFT))
+
+/* Send queue information. */
+struct mlx4_sq {
+	uint8_t *buf; /**< SQ buffer. */
+	uint8_t *eob; /**< End of SQ buffer */
+	uint32_t head; /**< SQ head counter in units of TXBBS. */
+	uint32_t tail; /**< SQ tail counter in units of TXBBS. */
+	uint32_t txbb_cnt; /**< Num of WQEBB in the Q (should be ^2). */
+	uint32_t txbb_cnt_mask; /**< txbbs_cnt mask (txbb_cnt is ^2). */
+	uint32_t headroom_txbbs; /**< Num of txbbs that should be kept free. */
+	uint32_t *db; /**< Pointer to the doorbell. */
+	uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */
+};
+
+#define mlx4_get_send_wqe(sq, n) ((sq)->buf + ((n) * (MLX4_TXBB_SIZE)))
+
+/* Completion queue information. */
+struct mlx4_cq {
+	uint8_t *buf; /**< Pointer to the completion queue buffer. */
+	uint32_t cqe_cnt; /**< Number of entries in the queue. */
+	uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */
+	uint32_t cons_index; /**< Last queue entry that was handled. */
+	uint32_t *set_ci_db; /**< Pointer to the completion queue doorbell. */
+};
+
+/**
+ * Retrieve a CQE entry from a CQ.
+ *
+ * cqe = cq->buf + cons_index * cqe_size + cqe_offset
+ *
+ * Where cqe_size is 32 or 64 bytes and cqe_offset is 0 or 32 (depending on
+ * cqe_size).
+ *
+ * @param cq
+ *   CQ to retrieve entry from.
+ * @param index
+ *   Entry index.
+ *
+ * @return
+ *   Pointer to CQE entry.
+ */
+static inline struct mlx4_cqe *
+mlx4_get_cqe(struct mlx4_cq *cq, uint32_t index)
+{
+	return (struct mlx4_cqe *)(cq->buf +
+				   ((index & (cq->cqe_cnt - 1)) <<
+				    (5 + cq->cqe_64)) +
+				   (cq->cqe_64 << 5));
+}
+
+#endif /* MLX4_PRM_H_ */

411 lines not on this page · raw message

index fec998a..cc5951c 100644
@@ -40,6 +40,7 @@
 #ifdef PEDANTIC
 #pragma GCC diagnostic ignored "-Wpedantic"
 #endif
+#include <infiniband/mlx4dv.h>
 #include <infiniband/verbs.h>
 #ifdef PEDANTIC
 #pragma GCC diagnostic error "-Wpedantic"

@@ -50,6 +51,7 @@
 #include <rte_mempool.h>
 
 #include "mlx4.h"
+#include "mlx4_prm.h"
 
 /** Rx queue counters. */
 struct mlx4_rxq_stats {

@@ -85,8 +87,6 @@ struct rxq {
 
 /** Tx element. */
 struct txq_elt {
-	struct ibv_send_wr wr; /* Work request. */
-	struct ibv_sge sge; /* Scatter/gather element. */
 	struct rte_mbuf *buf; /**< Buffer. */
 };
 

@@ -100,24 +100,26 @@ struct mlx4_txq_stats {
 
 /** Tx queue descriptor. */
 struct txq {
-	struct priv *priv; /**< Back pointer to private data. */
-	struct {
-		const struct rte_mempool *mp; /**< Cached memory pool. */
-		struct ibv_mr *mr; /**< Memory region (for mp). */
-		uint32_t lkey; /**< mr->lkey copy. */
-	} mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
-	struct ibv_cq *cq; /**< Completion queue. */
-	struct ibv_qp *qp; /**< Queue pair. */
-	uint32_t max_inline; /**< Max inline send size. */
-	unsigned int elts_n; /**< (*elts)[] length. */
-	struct txq_elt (*elts)[]; /**< Tx elements. */
+	struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
+	struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
 	unsigned int elts_head; /**< Current index in (*elts)[]. */
 	unsigned int elts_tail; /**< First element awaiting completion. */
 	unsigned int elts_comp; /**< Number of completion requests. */
 	unsigned int elts_comp_cd; /**< Countdown for next completion. */
 	unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
+	unsigned int elts_n; /**< (*elts)[] length. */
+	struct txq_elt (*elts)[]; /**< Tx elements. */
 	struct mlx4_txq_stats stats; /**< Tx queue counters. */
+	uint32_t max_inline; /**< Max inline send size. */
+	struct {
+		const struct rte_mempool *mp; /**< Cached memory pool. */
+		struct ibv_mr *mr; /**< Memory region (for mp). */
+		uint32_t lkey; /**< mr->lkey copy. */
+	} mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
+	struct priv *priv; /**< Back pointer to private data. */
 	unsigned int socket; /**< CPU socket ID for allocations. */
+	struct ibv_cq *cq; /**< Completion queue. */
+	struct ibv_qp *qp; /**< Queue pair. */
 };
 
 /* mlx4_rxq.c */