Introduce Microsoft Azure Network Adatper (MANA) PMD

704 lines not on this page · raw message

209 lines not on this page · raw message

235 lines not on this page · raw message

728 lines not on this page · raw message

207 lines not on this page · raw message

241 lines not on this page · raw message

725 lines not on this page · raw message

102 lines not on this page · raw message

241 lines not on this page · raw message

index 1c317418b6..64069e2b9f 100644
@@ -42,7 +42,85 @@ static rte_spinlock_t mana_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 int mana_logtype_driver;
 int mana_logtype_init;
 
+/*
+ * Callback from rdma-core to allocate a buffer for a queue.
+ */
+void *
+mana_alloc_verbs_buf(size_t size, void *data)
+{
+	void *ret;
+	size_t alignment = rte_mem_page_size();
+	int socket = (int)(uintptr_t)data;
+
+	DRV_LOG(DEBUG, "size=%zu socket=%d", size, socket);
+
+	if (alignment == (size_t)-1) {
+		DRV_LOG(ERR, "Failed to get mem page size");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	ret = rte_zmalloc_socket("mana_verb_buf", size, alignment, socket);
+	if (!ret && size)
+		rte_errno = ENOMEM;
+	return ret;
+}
+
+void
+mana_free_verbs_buf(void *ptr, void *data __rte_unused)
+{
+	rte_free(ptr);
+}
+
+static int
+mana_dev_configure(struct rte_eth_dev *dev)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+
+	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
+		dev_conf->rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
+
+	if (dev->data->nb_rx_queues != dev->data->nb_tx_queues) {
+		DRV_LOG(ERR, "Only support equal number of rx/tx queues");
+		return -EINVAL;
+	}
+
+	if (!rte_is_power_of_2(dev->data->nb_rx_queues)) {
+		DRV_LOG(ERR, "number of TX/RX queues must be power of 2");
+		return -EINVAL;
+	}
+
+	priv->num_queues = dev->data->nb_rx_queues;
+
+	manadv_set_context_attr(priv->ib_ctx, MANADV_CTX_ATTR_BUF_ALLOCATORS,
+				(void *)((uintptr_t)&(struct manadv_ctx_allocators){
+					.alloc = &mana_alloc_verbs_buf,
+					.free = &mana_free_verbs_buf,
+					.data = 0,
+				}));
+
+	return 0;
+}
+
+static int
+mana_dev_close(struct rte_eth_dev *dev)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	int ret;
+
+	ret = ibv_close_device(priv->ib_ctx);
+	if (ret) {
+		ret = errno;
+		return ret;
+	}
+
+	return 0;
+}
+
 static const struct eth_dev_ops mana_dev_ops = {
+	.dev_configure		= mana_dev_configure,
+	.dev_close		= mana_dev_close,
 };
 
 static const struct eth_dev_ops mana_dev_secondary_ops = {

@@ -649,8 +727,7 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 static int
 mana_dev_uninit(struct rte_eth_dev *dev)
 {
-	RTE_SET_USED(dev);
-	return 0;
+	return mana_dev_close(dev);
 }
 
 /*

index a2021ceb4a..0b4a828b0a 100644
@@ -28,6 +28,7 @@ struct mana_process_priv {
 struct mana_priv {
 	struct rte_eth_dev_data *dev_data;
 	struct mana_process_priv *process_priv;
+	int num_queues;
 
 	/* DPDK port */
 	uint16_t port_id;

@@ -99,4 +100,7 @@ int mana_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev);
 
 void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 
+void *mana_alloc_verbs_buf(size_t size, void *data);
+void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
+
 #endif

index 64069e2b9f..5fc656110f 100644
@@ -118,9 +118,26 @@ mana_dev_close(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static const uint32_t *
+mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_L4_FRAG,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	return ptypes;
+}
+
 static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
+	.dev_supported_ptypes_get = mana_supported_ptypes,
 };
 
 static const struct eth_dev_ops mana_dev_secondary_ops = {

index 5fc656110f..90e713318a 100644
@@ -134,10 +134,28 @@ mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
 	return ptypes;
 }
 
+static int
+mana_dev_link_update(struct rte_eth_dev *dev,
+		     int wait_to_complete __rte_unused)
+{
+	struct rte_eth_link link;
+
+	/* MANA has no concept of carrier state, always reporting UP */
+	link = (struct rte_eth_link) {
+		.link_duplex = RTE_ETH_LINK_FULL_DUPLEX,
+		.link_autoneg = RTE_ETH_LINK_SPEED_FIXED,
+		.link_speed = RTE_ETH_SPEED_NUM_100G,
+		.link_status = RTE_ETH_LINK_UP,
+	};
+
+	return rte_eth_linkstatus_set(dev, &link);
+}
+
 static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
+	.link_update		= mana_dev_link_update,
 };
 
 static const struct eth_dev_ops mana_dev_secondary_ops = {

127 lines not on this page · raw message

index 0b4a828b0a..d9eb19559d 100644
@@ -40,6 +40,7 @@ struct mana_priv {
 	struct ibv_pd *ib_pd;
 	struct ibv_pd *ib_parent_pd;
 	void *db_page;
+	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;
 	int max_rx_desc;

100 lines not on this page · raw message

index d9eb19559d..2d280785ec 100644
@@ -21,6 +21,34 @@ struct mana_shared_data {
 	unsigned int secondary_cnt;
 };
 
+#define MIN_RX_BUF_SIZE	1024
+#define MAX_FRAME_SIZE	RTE_ETHER_MAX_LEN
+#define MANA_MAX_MAC_ADDR 1
+
+#define MANA_DEV_RX_OFFLOAD_SUPPORT ( \
+		DEV_RX_OFFLOAD_CHECKSUM | \
+		DEV_RX_OFFLOAD_RSS_HASH)
+
+#define MANA_DEV_TX_OFFLOAD_SUPPORT ( \
+		RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
+		RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \
+		RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \
+		RTE_ETH_TX_OFFLOAD_UDP_CKSUM)
+
+#define INDIRECTION_TABLE_NUM_ELEMENTS 64
+#define TOEPLITZ_HASH_KEY_SIZE_IN_BYTES 40
+#define MANA_ETH_RSS_SUPPORT ( \
+	ETH_RSS_IPV4 |	     \
+	ETH_RSS_NONFRAG_IPV4_TCP | \
+	ETH_RSS_NONFRAG_IPV4_UDP | \
+	ETH_RSS_IPV6 |	     \
+	ETH_RSS_NONFRAG_IPV6_TCP | \
+	ETH_RSS_NONFRAG_IPV6_UDP)
+
+#define MIN_BUFFERS_PER_QUEUE		64
+#define MAX_RECEIVE_BUFFERS_PER_QUEUE	256
+#define MAX_SEND_BUFFERS_PER_QUEUE	256
+
 struct mana_process_priv {
 	void *db_page;
 };

79 lines not on this page · raw message

index 2d280785ec..8d8be3d4eb 100644
@@ -68,6 +68,7 @@ struct mana_priv {
 	struct ibv_pd *ib_pd;
 	struct ibv_pd *ib_parent_pd;
 	void *db_page;
+	struct rte_eth_rss_conf rss_conf;
 	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;

index bbaaa8c233..1c5416fac5 100644
@@ -205,6 +205,17 @@ mana_dev_info_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void
+mana_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
+		       struct rte_eth_rxq_info *qinfo)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[queue_id];
+
+	qinfo->mp = rxq->mp;
+	qinfo->nb_desc = rxq->num_desc;
+	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
+}
+
 static const uint32_t *
 mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
 {

@@ -282,9 +293,66 @@ mana_rss_hash_conf_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int
+mana_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+			uint16_t nb_desc, unsigned int socket_id,
+			const struct rte_eth_rxconf *rx_conf __rte_unused,
+			struct rte_mempool *mp)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct mana_rxq *rxq;
+	int ret;
+
+	rxq = rte_zmalloc_socket("mana_rxq", sizeof(*rxq), 0, socket_id);
+	if (!rxq) {
+		DRV_LOG(ERR, "failed to allocate rxq");
+		return -ENOMEM;
+	}
+
+	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u",
+		queue_idx, nb_desc, socket_id);
+
+	rxq->socket = socket_id;
+
+	rxq->desc_ring = rte_zmalloc_socket("mana_rx_mbuf_ring",
+					    sizeof(struct mana_rxq_desc) *
+						nb_desc,
+					    RTE_CACHE_LINE_SIZE, socket_id);
+
+	if (!rxq->desc_ring) {
+		DRV_LOG(ERR, "failed to allocate rxq desc_ring");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	rxq->desc_ring_head = 0;
+	rxq->desc_ring_tail = 0;
+
+	rxq->priv = priv;
+	rxq->num_desc = nb_desc;
+	rxq->mp = mp;
+	dev->data->rx_queues[queue_idx] = rxq;
+
+	return 0;
+
+fail:
+	rte_free(rxq->desc_ring);
+	rte_free(rxq);
+	return ret;
+}
+
+static void
+mana_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[qid];
+
+	rte_free(rxq->desc_ring);
+	rte_free(rxq);
+}
+
 static int
 mana_dev_link_update(struct rte_eth_dev *dev,
-				int wait_to_complete __rte_unused)
+		     int wait_to_complete __rte_unused)
 {
 	struct rte_eth_link link;
 

@@ -303,9 +371,12 @@ static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
+	.rxq_info_get		= mana_dev_rx_queue_info,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
 	.rss_hash_update	= mana_rss_hash_update,
 	.rss_hash_conf_get	= mana_rss_hash_conf_get,
+	.rx_queue_setup		= mana_dev_rx_queue_setup,
+	.rx_queue_release	= mana_dev_rx_queue_release,
 	.link_update		= mana_dev_link_update,
 };
 

index 8d8be3d4eb..a436d6cfc9 100644
@@ -80,6 +80,27 @@ struct mana_priv {
 	uint64_t max_mr_size;
 };
 
+struct mana_rxq_desc {
+	struct rte_mbuf *pkt;
+	uint32_t wqe_size_in_bu;
+};
+
+struct mana_rxq {
+	struct mana_priv *priv;
+	uint32_t num_desc;
+	struct rte_mempool *mp;
+
+	/* For storing pending requests */
+	struct mana_rxq_desc *desc_ring;
+
+	/* desc_ring_head is where we put pending requests to ring,
+	 * completion pull off desc_ring_tail
+	 */
+	uint32_t desc_ring_head, desc_ring_tail;
+
+	unsigned int socket;
+};
+
 extern int mana_logtype_driver;
 extern int mana_logtype_init;
 

index 1c5416fac5..a8a792d5f8 100644
@@ -205,6 +205,16 @@ mana_dev_info_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void
+mana_dev_tx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
+		       struct rte_eth_txq_info *qinfo)
+{
+	struct mana_txq *txq = dev->data->tx_queues[queue_id];
+
+	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
+	qinfo->nb_desc = txq->num_desc;
+}
+
 static void
 mana_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
 		       struct rte_eth_rxq_info *qinfo)

@@ -293,6 +303,60 @@ mana_rss_hash_conf_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int
+mana_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+			uint16_t nb_desc, unsigned int socket_id,
+			const struct rte_eth_txconf *tx_conf __rte_unused)
+
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct mana_txq *txq;
+	int ret;
+
+	txq = rte_zmalloc_socket("mana_txq", sizeof(*txq), 0, socket_id);
+	if (!txq) {
+		DRV_LOG(ERR, "failed to allocate txq");
+		return -ENOMEM;
+	}
+
+	txq->socket = socket_id;
+
+	txq->desc_ring = rte_malloc_socket("mana_tx_desc_ring",
+					   sizeof(struct mana_txq_desc) *
+						nb_desc,
+					   RTE_CACHE_LINE_SIZE, socket_id);
+	if (!txq->desc_ring) {
+		DRV_LOG(ERR, "failed to allocate txq desc_ring");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u txq->desc_ring %p",
+		queue_idx, nb_desc, socket_id, txq->desc_ring);
+
+	txq->desc_ring_head = 0;
+	txq->desc_ring_tail = 0;
+	txq->priv = priv;
+	txq->num_desc = nb_desc;
+	dev->data->tx_queues[queue_idx] = txq;
+
+	return 0;
+
+fail:
+	rte_free(txq->desc_ring);
+	rte_free(txq);
+	return ret;
+}
+
+static void
+mana_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct mana_txq *txq = dev->data->tx_queues[qid];
+
+	rte_free(txq->desc_ring);
+	rte_free(txq);
+}
+
 static int
 mana_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 			uint16_t nb_desc, unsigned int socket_id,

@@ -371,10 +435,13 @@ static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
+	.txq_info_get		= mana_dev_tx_queue_info,
 	.rxq_info_get		= mana_dev_rx_queue_info,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
 	.rss_hash_update	= mana_rss_hash_update,
 	.rss_hash_conf_get	= mana_rss_hash_conf_get,
+	.tx_queue_setup		= mana_dev_tx_queue_setup,
+	.tx_queue_release	= mana_dev_tx_queue_release,
 	.rx_queue_setup		= mana_dev_rx_queue_setup,
 	.rx_queue_release	= mana_dev_rx_queue_release,
 	.link_update		= mana_dev_link_update,

index a436d6cfc9..817ac75f69 100644
@@ -80,11 +80,31 @@ struct mana_priv {
 	uint64_t max_mr_size;
 };
 
+struct mana_txq_desc {
+	struct rte_mbuf *pkt;
+	uint32_t wqe_size_in_bu;
+};
+
 struct mana_rxq_desc {
 	struct rte_mbuf *pkt;
 	uint32_t wqe_size_in_bu;
 };
 
+struct mana_txq {
+	struct mana_priv *priv;
+	uint32_t num_desc;
+
+	/* For storing pending requests */
+	struct mana_txq_desc *desc_ring;
+
+	/* desc_ring_head is where we put pending requests to ring,
+	 * completion pull off desc_ring_tail
+	 */
+	uint32_t desc_ring_head, desc_ring_tail;
+
+	unsigned int socket;
+};
+
 struct mana_rxq {
 	struct mana_priv *priv;
 	uint32_t num_desc;

index a8a792d5f8..a3c949d408 100644
@@ -111,6 +111,8 @@ mana_dev_close(struct rte_eth_dev *dev)
 	struct mana_priv *priv = dev->data->dev_private;
 	int ret;
 
+	mana_remove_all_mr(priv);
+
 	ret = mana_intr_uninstall(priv);
 	if (ret)
 		return ret;

@@ -331,6 +333,13 @@ mana_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		goto fail;
 	}
 
+	ret = mana_mr_btree_init(&txq->mr_btree,
+				 MANA_MR_BTREE_PER_QUEUE_N, socket_id);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to init TXQ MR btree");
+		goto fail;
+	}
+
 	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u txq->desc_ring %p",
 		queue_idx, nb_desc, socket_id, txq->desc_ring);
 

@@ -353,6 +362,8 @@ mana_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 {
 	struct mana_txq *txq = dev->data->tx_queues[qid];
 
+	mana_mr_btree_free(&txq->mr_btree);
+
 	rte_free(txq->desc_ring);
 	rte_free(txq);
 }

@@ -392,6 +403,13 @@ mana_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	rxq->desc_ring_head = 0;
 	rxq->desc_ring_tail = 0;
 
+	ret = mana_mr_btree_init(&rxq->mr_btree,
+				 MANA_MR_BTREE_PER_QUEUE_N, socket_id);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to init RXQ MR btree");
+		goto fail;
+	}
+
 	rxq->priv = priv;
 	rxq->num_desc = nb_desc;
 	rxq->mp = mp;

@@ -410,6 +428,8 @@ mana_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 {
 	struct mana_rxq *rxq = dev->data->rx_queues[qid];
 
+	mana_mr_btree_free(&rxq->mr_btree);
+
 	rte_free(rxq->desc_ring);
 	rte_free(rxq);
 }

84 lines not on this page · raw message

110 lines not on this page · raw message

348 lines not on this page · raw message

new file mode 100644
index 0000000000..3f937d6c93
@@ -0,0 +1,301 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2022 Microsoft Corporation
+ */
+
+#include <ethdev_driver.h>
+#include <rte_io.h>
+
+#include "mana.h"
+
+uint8_t *
+gdma_get_wqe_pointer(struct mana_gdma_queue *queue)
+{
+	uint32_t offset_in_bytes =
+		(queue->head * GDMA_WQE_ALIGNMENT_UNIT_SIZE) &
+		(queue->size - 1);
+
+	DRV_LOG(DEBUG, "txq sq_head %u sq_size %u offset_in_bytes %u",
+		queue->head, queue->size, offset_in_bytes);
+
+	if (offset_in_bytes + GDMA_WQE_ALIGNMENT_UNIT_SIZE > queue->size)
+		DRV_LOG(ERR, "fatal error: offset_in_bytes %u too big",
+			offset_in_bytes);
+
+	return ((uint8_t *)queue->buffer) + offset_in_bytes;
+}
+
+static uint32_t
+write_dma_client_oob(uint8_t *work_queue_buffer_pointer,
+		     const struct gdma_work_request *work_request,
+		     uint32_t client_oob_size)
+{
+	uint8_t *p = work_queue_buffer_pointer;
+
+	struct gdma_wqe_dma_oob *header = (struct gdma_wqe_dma_oob *)p;
+
+	memset(header, 0, sizeof(struct gdma_wqe_dma_oob));
+	header->num_sgl_entries = work_request->num_sgl_elements;
+	header->inline_client_oob_size_in_dwords =
+		client_oob_size / sizeof(uint32_t);
+	header->client_data_unit = work_request->client_data_unit;
+
+	DRV_LOG(DEBUG, "queue buf %p sgl %u oob_h %u du %u oob_buf %p oob_b %u",
+		work_queue_buffer_pointer, header->num_sgl_entries,
+		header->inline_client_oob_size_in_dwords,
+		header->client_data_unit, work_request->inline_oob_data,
+		work_request->inline_oob_size_in_bytes);
+
+	p += sizeof(struct gdma_wqe_dma_oob);
+	if (work_request->inline_oob_data &&
+	    work_request->inline_oob_size_in_bytes > 0) {
+		memcpy(p, work_request->inline_oob_data,
+		       work_request->inline_oob_size_in_bytes);
+		if (client_oob_size > work_request->inline_oob_size_in_bytes)
+			memset(p + work_request->inline_oob_size_in_bytes, 0,
+			       client_oob_size -
+			       work_request->inline_oob_size_in_bytes);
+	}
+
+	return sizeof(struct gdma_wqe_dma_oob) + client_oob_size;
+}
+
+static uint32_t
+write_scatter_gather_list(uint8_t *work_queue_head_pointer,
+			  uint8_t *work_queue_end_pointer,
+			  uint8_t *work_queue_cur_pointer,
+			  struct gdma_work_request *work_request)
+{
+	struct gdma_sgl_element *sge_list;
+	struct gdma_sgl_element dummy_sgl[1];
+	uint8_t *address;
+	uint32_t size;
+	uint32_t num_sge;
+	uint32_t size_to_queue_end;
+	uint32_t sge_list_size;
+
+	DRV_LOG(DEBUG, "work_queue_cur_pointer %p work_request->flags %x",
+		work_queue_cur_pointer, work_request->flags);
+
+	num_sge = work_request->num_sgl_elements;
+	sge_list = work_request->sgl;
+	size_to_queue_end = (uint32_t)(work_queue_end_pointer -
+				       work_queue_cur_pointer);
+
+	if (num_sge == 0) {
+		/* Per spec, the case of an empty SGL should be handled as
+		 * follows to avoid corrupted WQE errors:
+		 * Write one dummy SGL entry
+		 * Set the address to 1, leave the rest as 0
+		 */
+		dummy_sgl[num_sge].address = 1;
+		dummy_sgl[num_sge].size = 0;
+		dummy_sgl[num_sge].memory_key = 0;
+		num_sge++;
+		sge_list = dummy_sgl;
+	}
+
+	sge_list_size = 0;
+	{
+		address = (uint8_t *)sge_list;
+		size = sizeof(struct gdma_sgl_element) * num_sge;
+		if (size_to_queue_end < size) {
+			memcpy(work_queue_cur_pointer, address,
+			       size_to_queue_end);
+			work_queue_cur_pointer = work_queue_head_pointer;
+			address += size_to_queue_end;
+			size -= size_to_queue_end;
+		}
+
+		memcpy(work_queue_cur_pointer, address, size);
+		sge_list_size = size;
+	}
+
+	DRV_LOG(DEBUG, "sge %u address 0x%" PRIx64 " size %u key %u list_s %u",
+		num_sge, sge_list->address, sge_list->size,
+		sge_list->memory_key, sge_list_size);
+
+	return sge_list_size;
+}
+
+/*
+ * Post a work request to queue.
+ */
+int
+gdma_post_work_request(struct mana_gdma_queue *queue,
+		       struct gdma_work_request *work_req,
+		       struct gdma_posted_wqe_info *wqe_info)
+{
+	uint32_t client_oob_size =
+		work_req->inline_oob_size_in_bytes >
+				INLINE_OOB_SMALL_SIZE_IN_BYTES ?
+			INLINE_OOB_LARGE_SIZE_IN_BYTES :
+			INLINE_OOB_SMALL_SIZE_IN_BYTES;
+
+	uint32_t sgl_data_size = sizeof(struct gdma_sgl_element) *
+			RTE_MAX((uint32_t)1, work_req->num_sgl_elements);
+	uint32_t wqe_size =
+		RTE_ALIGN(sizeof(struct gdma_wqe_dma_oob) +
+				client_oob_size + sgl_data_size,
+			  GDMA_WQE_ALIGNMENT_UNIT_SIZE);
+	uint8_t *wq_buffer_pointer;
+	uint32_t queue_free_units = queue->count - (queue->head - queue->tail);
+
+	if (wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE > queue_free_units) {
+		DRV_LOG(DEBUG, "WQE size %u queue count %u head %u tail %u",
+			wqe_size, queue->count, queue->head, queue->tail);
+		return -EBUSY;
+	}
+
+	DRV_LOG(DEBUG, "client_oob_size %u sgl_data_size %u wqe_size %u",
+		client_oob_size, sgl_data_size, wqe_size);
+
+	if (wqe_info) {
+		wqe_info->wqe_index =
+			((queue->head * GDMA_WQE_ALIGNMENT_UNIT_SIZE) &
+			 (queue->size - 1)) / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+		wqe_info->unmasked_queue_offset = queue->head;
+		wqe_info->wqe_size_in_bu =
+			wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+	}
+
+	wq_buffer_pointer = gdma_get_wqe_pointer(queue);
+	wq_buffer_pointer += write_dma_client_oob(wq_buffer_pointer, work_req,
+						  client_oob_size);
+	if (wq_buffer_pointer >= ((uint8_t *)queue->buffer) + queue->size)
+		wq_buffer_pointer -= queue->size;
+
+	write_scatter_gather_list((uint8_t *)queue->buffer,
+				  (uint8_t *)queue->buffer + queue->size,
+				  wq_buffer_pointer, work_req);
+
+	queue->head += wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+
+	return 0;
+}
+
+union gdma_doorbell_entry {
+	uint64_t     as_uint64;
+
+	struct {
+		uint64_t id	  : 24;
+		uint64_t reserved    : 8;
+		uint64_t tail_ptr    : 31;
+		uint64_t arm	 : 1;
+	} cq;
+
+	struct {
+		uint64_t id	  : 24;
+		uint64_t wqe_cnt     : 8;
+		uint64_t tail_ptr    : 32;
+	} rq;
+
+	struct {
+		uint64_t id	  : 24;
+		uint64_t reserved    : 8;
+		uint64_t tail_ptr    : 32;
+	} sq;
+
+	struct {
+		uint64_t id	  : 16;
+		uint64_t reserved    : 16;
+		uint64_t tail_ptr    : 31;
+		uint64_t arm	 : 1;
+	} eq;
+}; /* HW DATA */
+
+#define DOORBELL_OFFSET_SQ      0x0
+#define DOORBELL_OFFSET_RQ      0x400
+#define DOORBELL_OFFSET_CQ      0x800
+#define DOORBELL_OFFSET_EQ      0xFF8
+
+/*
+ * Write to hardware doorbell to notify new activity.
+ */
+int
+mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
+		   uint32_t queue_id, uint32_t tail)
+{
+	uint8_t *addr = db_page;
+	union gdma_doorbell_entry e = {};
+
+	switch (queue_type) {
+	case GDMA_QUEUE_SEND:
+		e.sq.id = queue_id;
+		e.sq.tail_ptr = tail;
+		addr += DOORBELL_OFFSET_SQ;
+		break;
+
+	case GDMA_QUEUE_RECEIVE:
+		e.rq.id = queue_id;
+		e.rq.tail_ptr = tail;
+		e.rq.wqe_cnt = 1;
+		addr += DOORBELL_OFFSET_RQ;
+		break;
+
+	case GDMA_QUEUE_COMPLETION:
+		e.cq.id = queue_id;
+		e.cq.tail_ptr = tail;
+		e.cq.arm = 1;
+		addr += DOORBELL_OFFSET_CQ;
+		break;
+
+	default:
+		DRV_LOG(ERR, "Unsupported queue type %d", queue_type);
+		return -1;
+	}
+
+	/* Ensure all writes are done before ringing doorbell */
+	rte_wmb();
+
+	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u",
+		db_page, addr, queue_id, queue_type, tail);
+
+	rte_write64(e.as_uint64, addr);
+	return 0;
+}
+
+/*
+ * Poll completion queue for completions.
+ */
+int
+gdma_poll_completion_queue(struct mana_gdma_queue *cq, struct gdma_comp *comp)
+{
+	struct gdma_hardware_completion_entry *cqe;
+	uint32_t head = cq->head % cq->count;
+	uint32_t new_owner_bits, old_owner_bits;
+	uint32_t cqe_owner_bits;
+	struct gdma_hardware_completion_entry *buffer = cq->buffer;
+
+	cqe = &buffer[head];
+	new_owner_bits = (cq->head / cq->count) & COMPLETION_QUEUE_OWNER_MASK;
+	old_owner_bits = (cq->head / cq->count - 1) &
+				COMPLETION_QUEUE_OWNER_MASK;
+	cqe_owner_bits = cqe->owner_bits;
+
+	DRV_LOG(DEBUG, "comp cqe bits 0x%x owner bits 0x%x",
+		cqe_owner_bits, old_owner_bits);
+
+	if (cqe_owner_bits == old_owner_bits)
+		return 0; /* No new entry */
+
+	if (cqe_owner_bits != new_owner_bits) {
+		DRV_LOG(ERR, "CQ overflowed, ID %u cqe 0x%x new 0x%x",
+			cq->id, cqe_owner_bits, new_owner_bits);
+		return -1;
+	}
+
+	/* Ensure checking owner bits happens before reading from CQE */
+	rte_rmb();
+
+	comp->work_queue_number = cqe->wq_num;
+	comp->send_work_queue = cqe->is_sq;
+
+	memcpy(comp->completion_data, cqe->dma_client_data, GDMA_COMP_DATA_SIZE);
+
+	cq->head++;
+
+	DRV_LOG(DEBUG, "comp new 0x%x old 0x%x cqe 0x%x wq %u sq %u head %u",
+		new_owner_bits, old_owner_bits, cqe_owner_bits,
+		comp->work_queue_number, comp->send_work_queue, cq->head);
+	return 1;
+}

215 lines not on this page · raw message

index f4fb6e8a37..0b0c6ad122 100644
@@ -293,6 +293,13 @@ struct mana_gdma_queue {
 struct mana_txq {
 	struct mana_priv *priv;
 	uint32_t num_desc;
+	struct ibv_cq *cq;
+	struct ibv_qp *qp;
+
+	struct mana_gdma_queue gdma_sq;
+	struct mana_gdma_queue gdma_cq;
+
+	uint32_t tx_vp_offset;
 
 	/* For storing pending requests */
 	struct mana_txq_desc *desc_ring;

@@ -354,6 +361,10 @@ uint16_t mana_tx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 int gdma_poll_completion_queue(struct mana_gdma_queue *cq,
 			       struct gdma_comp *comp);
 
+int mana_start_tx_queues(struct rte_eth_dev *dev);
+
+int mana_stop_tx_queues(struct rte_eth_dev *dev);
+
 struct mana_mr_cache *mana_find_pmd_mr(struct mana_mr_btree *local_tree,
 				       struct mana_priv *priv,
 				       struct rte_mbuf *mbuf);

166 lines not on this page · raw message

index 0b0c6ad122..5312c2f93c 100644
@@ -254,6 +254,8 @@ struct mana_priv {
 	struct ibv_context *ib_ctx;
 	struct ibv_pd *ib_pd;
 	struct ibv_pd *ib_parent_pd;
+	struct ibv_rwq_ind_table *ind_table;
+	struct ibv_qp *rwq_qp;
 	void *db_page;
 	struct rte_eth_rss_conf rss_conf;
 	struct rte_intr_handle *intr_handle;

@@ -279,6 +281,13 @@ struct mana_rxq_desc {
 	uint32_t wqe_size_in_bu;
 };
 
+struct mana_stats {
+	uint64_t packets;
+	uint64_t bytes;
+	uint64_t errors;
+	uint64_t nombuf;
+};
+
 struct mana_gdma_queue {
 	void *buffer;
 	uint32_t count;	/* in entries */

@@ -317,6 +326,8 @@ struct mana_rxq {
 	struct mana_priv *priv;
 	uint32_t num_desc;
 	struct rte_mempool *mp;
+	struct ibv_cq *cq;
+	struct ibv_wq *wq;
 
 	/* For storing pending requests */
 	struct mana_rxq_desc *desc_ring;

@@ -326,6 +337,10 @@ struct mana_rxq {
 	 */
 	uint32_t desc_ring_head, desc_ring_tail;
 
+	struct mana_gdma_queue gdma_rq;
+	struct mana_gdma_queue gdma_cq;
+
+	struct mana_stats stats;
 	struct mana_mr_btree mr_btree;
 
 	unsigned int socket;

@@ -346,6 +361,7 @@ extern int mana_logtype_init;
 
 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 		       uint32_t queue_id, uint32_t tail);
+int mana_rq_ring_doorbell(struct mana_rxq *rxq);
 
 int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_work_request *work_req,

@@ -361,8 +377,10 @@ uint16_t mana_tx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 int gdma_poll_completion_queue(struct mana_gdma_queue *cq,
 			       struct gdma_comp *comp);
 
+int mana_start_rx_queues(struct rte_eth_dev *dev);
 int mana_start_tx_queues(struct rte_eth_dev *dev);
 
+int mana_stop_rx_queues(struct rte_eth_dev *dev);
 int mana_stop_tx_queues(struct rte_eth_dev *dev);
 
 struct mana_mr_cache *mana_find_pmd_mr(struct mana_mr_btree *local_tree,

354 lines not on this page · raw message

index a3c949d408..51aa01a642 100644
@@ -991,6 +991,8 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev,
 				/* fd is no not used after mapping doorbell */
 				close(fd);
 
+				eth_dev->rx_pkt_burst = mana_rx_burst;
+
 				rte_spinlock_lock(&mana_shared_data->lock);
 				mana_shared_data->secondary_cnt++;
 				mana_local_data.secondary_cnt++;

index 5312c2f93c..ef0743df2d 100644
@@ -177,6 +177,11 @@ struct gdma_work_request {
 
 enum mana_cqe_type {
 	CQE_INVALID                     = 0,
+
+	CQE_RX_OKAY                     = 1,
+	CQE_RX_COALESCED_4              = 2,
+	CQE_RX_OBJECT_FENCE             = 3,
+	CQE_RX_TRUNCATED                = 4,
 };
 
 struct mana_cqe_header {

@@ -201,6 +206,35 @@ struct mana_cqe_header {
 	(NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 |      \
 	 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
 
+struct mana_rx_comp_per_packet_info {
+	uint32_t packet_length	: 16;
+	uint32_t reserved0	: 16;
+	uint32_t reserved1;
+	uint32_t packet_hash;
+}; /* HW DATA */
+#define RX_COM_OOB_NUM_PACKETINFO_SEGMENTS 4
+
+struct mana_rx_comp_oob {
+	struct mana_cqe_header cqe_hdr;
+
+	uint32_t rx_vlan_id				: 12;
+	uint32_t rx_vlan_tag_present			: 1;
+	uint32_t rx_outer_ip_header_checksum_succeeded	: 1;
+	uint32_t rx_outer_ip_header_checksum_failed	: 1;
+	uint32_t reserved				: 1;
+	uint32_t rx_hash_type				: 9;
+	uint32_t rx_ip_header_checksum_succeeded	: 1;
+	uint32_t rx_ip_header_checksum_failed		: 1;
+	uint32_t rx_tcp_checksum_succeeded		: 1;
+	uint32_t rx_tcp_checksum_failed			: 1;
+	uint32_t rx_udp_checksum_succeeded		: 1;
+	uint32_t rx_udp_checksum_failed			: 1;
+	uint32_t reserved1				: 1;
+	struct mana_rx_comp_per_packet_info
+		packet_info[RX_COM_OOB_NUM_PACKETINFO_SEGMENTS];
+	uint32_t received_wqe_offset;
+}; /* HW DATA */
+
 struct gdma_wqe_dma_oob {
 	uint32_t reserved:24;
 	uint32_t last_v_bytes:8;

@@ -368,6 +402,9 @@ int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_posted_wqe_info *wqe_info);
 uint8_t *gdma_get_wqe_pointer(struct mana_gdma_queue *queue);
 
+uint16_t mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **rx_pkts,
+		       uint16_t pkts_n);
+
 uint16_t mana_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 			       uint16_t pkts_n);
 

index a3b5ede559..feda30623a 100644
@@ -141,6 +141,8 @@ mana_mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
 	case MANA_MP_REQ_START_RXTX:
 		DRV_LOG(INFO, "Port %u starting datapath", dev->data->port_id);
 
+		dev->rx_pkt_burst = mana_rx_burst;
+
 		rte_mb();
 
 		res->result = 0;

108 lines not on this page · raw message

index 51aa01a642..d0214725e6 100644
@@ -991,6 +991,7 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev,
 				/* fd is no not used after mapping doorbell */
 				close(fd);
 
+				eth_dev->tx_pkt_burst = mana_tx_burst;
 				eth_dev->rx_pkt_burst = mana_rx_burst;
 
 				rte_spinlock_lock(&mana_shared_data->lock);

96 lines not on this page · raw message

index feda30623a..92432c431d 100644
@@ -141,6 +141,7 @@ mana_mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
 	case MANA_MP_REQ_START_RXTX:
 		DRV_LOG(INFO, "Port %u starting datapath", dev->data->port_id);
 
+		dev->tx_pkt_burst = mana_tx_burst;
 		dev->rx_pkt_burst = mana_rx_burst;
 
 		rte_mb();

251 lines not on this page · raw message

index d0214725e6..c43212950f 100644
@@ -105,6 +105,81 @@ mana_dev_configure(struct rte_eth_dev *dev)
 
 static int mana_intr_uninstall(struct mana_priv *priv);
 
+static int
+mana_dev_start(struct rte_eth_dev *dev)
+{
+	int ret;
+	struct mana_priv *priv = dev->data->dev_private;
+
+	rte_spinlock_init(&priv->mr_btree_lock);
+	ret = mana_mr_btree_init(&priv->mr_btree, MANA_MR_BTREE_CACHE_N,
+				 dev->device->numa_node);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to init device MR btree %d", ret);
+		return ret;
+	}
+
+	ret = mana_start_tx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to start tx queues %d", ret);
+		goto failed_tx;
+	}
+
+	ret = mana_start_rx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to start rx queues %d", ret);
+		goto failed_rx;
+	}
+
+	rte_wmb();
+
+	dev->tx_pkt_burst = mana_tx_burst;
+	dev->rx_pkt_burst = mana_rx_burst;
+
+	DRV_LOG(INFO, "TX/RX queues have started");
+
+	/* Enable datapath for secondary processes */
+	mana_mp_req_on_rxtx(dev, MANA_MP_REQ_START_RXTX);
+
+	return 0;
+
+failed_rx:
+	mana_stop_tx_queues(dev);
+
+failed_tx:
+	mana_mr_btree_free(&priv->mr_btree);
+
+	return ret;
+}
+
+static int
+mana_dev_stop(struct rte_eth_dev *dev __rte_unused)
+{
+	int ret;
+
+	dev->tx_pkt_burst = mana_tx_burst_removed;
+	dev->rx_pkt_burst = mana_rx_burst_removed;
+
+	/* Stop datapath on secondary processes */
+	mana_mp_req_on_rxtx(dev, MANA_MP_REQ_STOP_RXTX);
+
+	rte_wmb();
+
+	ret = mana_stop_tx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to stop tx queues");
+		return ret;
+	}
+
+	ret = mana_stop_rx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to stop tx queues");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 mana_dev_close(struct rte_eth_dev *dev)
 {

@@ -453,6 +528,8 @@ mana_dev_link_update(struct rte_eth_dev *dev,
 
 static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
+	.dev_start		= mana_dev_start,
+	.dev_stop		= mana_dev_stop,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
 	.txq_info_get		= mana_dev_tx_queue_info,

92 lines not on this page · raw message

index 3f937d6c93..c67c5af2f9 100644
@@ -213,7 +213,7 @@ union gdma_doorbell_entry {
  */
 int
 mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
-		   uint32_t queue_id, uint32_t tail)
+		   uint32_t queue_id, uint32_t tail, uint8_t arm)
 {
 	uint8_t *addr = db_page;
 	union gdma_doorbell_entry e = {};

@@ -228,14 +228,14 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 	case GDMA_QUEUE_RECEIVE:
 		e.rq.id = queue_id;
 		e.rq.tail_ptr = tail;
-		e.rq.wqe_cnt = 1;
+		e.rq.wqe_cnt = arm;
 		addr += DOORBELL_OFFSET_RQ;
 		break;
 
 	case GDMA_QUEUE_COMPLETION:
 		e.cq.id = queue_id;
 		e.cq.tail_ptr = tail;
-		e.cq.arm = 1;
+		e.cq.arm = arm;
 		addr += DOORBELL_OFFSET_CQ;
 		break;
 

@@ -247,8 +247,8 @@ mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 	/* Ensure all writes are done before ringing doorbell */
 	rte_wmb();
 
-	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u",
-		db_page, addr, queue_id, queue_type, tail);
+	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u",
+		db_page, addr, queue_id, queue_type, tail, arm);
 
 	rte_write64(e.as_uint64, addr);
 	return 0;

205 lines not on this page · raw message

index 9c576a82fa..14d813466e 100644
@@ -425,6 +425,7 @@ struct mana_rxq {
 	uint32_t num_desc;
 	struct rte_mempool *mp;
 	struct ibv_cq *cq;
+	struct ibv_comp_channel *channel;
 	struct ibv_wq *wq;
 
 	/* For storing pending requests */

@@ -458,8 +459,8 @@ extern int mana_logtype_init;
 #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
-		       uint32_t queue_id, uint32_t tail);
-int mana_rq_ring_doorbell(struct mana_rxq *rxq);
+		       uint32_t queue_id, uint32_t tail, uint8_t arm);
+int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm);
 
 int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_work_request *work_req,

@@ -539,4 +540,8 @@ void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 void *mana_alloc_verbs_buf(size_t size, void *data);
 void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
 
+int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mana_fd_set_non_blocking(int fd);
+
 #endif

145 lines not on this page · raw message

index 0884681c30..a92d895e54 100644
@@ -406,7 +406,8 @@ mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		ret = mana_ring_doorbell(db_page, GDMA_QUEUE_SEND,
 					 txq->gdma_sq.id,
 					 txq->gdma_sq.head *
-						GDMA_WQE_ALIGNMENT_UNIT_SIZE);
+						GDMA_WQE_ALIGNMENT_UNIT_SIZE,
+					 0);
 	if (ret)
 		DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret);
 

732 lines not on this page · raw message

97 lines not on this page · raw message

241 lines not on this page · raw message

index 6a7b8a419d..f5084b4a3b 100644
@@ -42,7 +42,85 @@ static rte_spinlock_t mana_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 int mana_logtype_driver;
 int mana_logtype_init;
 
+/*
+ * Callback from rdma-core to allocate a buffer for a queue.
+ */
+void *
+mana_alloc_verbs_buf(size_t size, void *data)
+{
+	void *ret;
+	size_t alignment = rte_mem_page_size();
+	int socket = (int)(uintptr_t)data;
+
+	DRV_LOG(DEBUG, "size=%zu socket=%d", size, socket);
+
+	if (alignment == (size_t)-1) {
+		DRV_LOG(ERR, "Failed to get mem page size");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	ret = rte_zmalloc_socket("mana_verb_buf", size, alignment, socket);
+	if (!ret && size)
+		rte_errno = ENOMEM;
+	return ret;
+}
+
+void
+mana_free_verbs_buf(void *ptr, void *data __rte_unused)
+{
+	rte_free(ptr);
+}
+
+static int
+mana_dev_configure(struct rte_eth_dev *dev)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+
+	if (dev_conf->rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
+		dev_conf->rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
+
+	if (dev->data->nb_rx_queues != dev->data->nb_tx_queues) {
+		DRV_LOG(ERR, "Only support equal number of rx/tx queues");
+		return -EINVAL;
+	}
+
+	if (!rte_is_power_of_2(dev->data->nb_rx_queues)) {
+		DRV_LOG(ERR, "number of TX/RX queues must be power of 2");
+		return -EINVAL;
+	}
+
+	priv->num_queues = dev->data->nb_rx_queues;
+
+	manadv_set_context_attr(priv->ib_ctx, MANADV_CTX_ATTR_BUF_ALLOCATORS,
+				(void *)((uintptr_t)&(struct manadv_ctx_allocators){
+					.alloc = &mana_alloc_verbs_buf,
+					.free = &mana_free_verbs_buf,
+					.data = 0,
+				}));
+
+	return 0;
+}
+
+static int
+mana_dev_close(struct rte_eth_dev *dev)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	int ret;
+
+	ret = ibv_close_device(priv->ib_ctx);
+	if (ret) {
+		ret = errno;
+		return ret;
+	}
+
+	return 0;
+}
+
 static const struct eth_dev_ops mana_dev_ops = {
+	.dev_configure		= mana_dev_configure,
+	.dev_close		= mana_dev_close,
 };
 
 static const struct eth_dev_ops mana_dev_secondary_ops = {

@@ -655,8 +733,7 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 static int
 mana_dev_uninit(struct rte_eth_dev *dev)
 {
-	RTE_SET_USED(dev);
-	return 0;
+	return mana_dev_close(dev);
 }
 
 /*

index 291dd83f27..d5f9b2661d 100644
@@ -23,6 +23,7 @@ struct mana_process_priv {
 struct mana_priv {
 	struct rte_eth_dev_data *dev_data;
 	struct mana_process_priv *process_priv;
+	int num_queues;
 
 	/* DPDK port */
 	uint16_t port_id;

@@ -94,4 +95,7 @@ int mana_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev);
 
 void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 
+void *mana_alloc_verbs_buf(size_t size, void *data);
+void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
+
 #endif

index f5084b4a3b..16fcfe0e99 100644
@@ -118,9 +118,26 @@ mana_dev_close(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static const uint32_t *
+mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_L4_FRAG,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	return ptypes;
+}
+
 static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
+	.dev_supported_ptypes_get = mana_supported_ptypes,
 };
 
 static const struct eth_dev_ops mana_dev_secondary_ops = {

index 16fcfe0e99..2b1f6fcf1e 100644
@@ -134,10 +134,28 @@ mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
 	return ptypes;
 }
 
+static int
+mana_dev_link_update(struct rte_eth_dev *dev,
+		     int wait_to_complete __rte_unused)
+{
+	struct rte_eth_link link;
+
+	/* MANA has no concept of carrier state, always reporting UP */
+	link = (struct rte_eth_link) {
+		.link_duplex = RTE_ETH_LINK_FULL_DUPLEX,
+		.link_autoneg = RTE_ETH_LINK_SPEED_FIXED,
+		.link_speed = RTE_ETH_SPEED_NUM_100G,
+		.link_status = RTE_ETH_LINK_UP,
+	};
+
+	return rte_eth_linkstatus_set(dev, &link);
+}
+
 static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
+	.link_update		= mana_dev_link_update,
 };
 
 static const struct eth_dev_ops mana_dev_secondary_ops = {

127 lines not on this page · raw message

index d5f9b2661d..f249ef1f66 100644
@@ -35,6 +35,7 @@ struct mana_priv {
 	struct ibv_pd *ib_pd;
 	struct ibv_pd *ib_parent_pd;
 	void *db_page;
+	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;
 	int max_rx_desc;

100 lines not on this page · raw message

index f249ef1f66..a3165616ce 100644
@@ -16,6 +16,34 @@ struct mana_shared_data {
 	unsigned int secondary_cnt;
 };
 
+#define MIN_RX_BUF_SIZE	1024
+#define MAX_FRAME_SIZE	RTE_ETHER_MAX_LEN
+#define MANA_MAX_MAC_ADDR 1
+
+#define MANA_DEV_RX_OFFLOAD_SUPPORT ( \
+		RTE_ETH_RX_OFFLOAD_CHECKSUM | \
+		RTE_ETH_RX_OFFLOAD_RSS_HASH)
+
+#define MANA_DEV_TX_OFFLOAD_SUPPORT ( \
+		RTE_ETH_TX_OFFLOAD_MULTI_SEGS | \
+		RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | \
+		RTE_ETH_TX_OFFLOAD_TCP_CKSUM | \
+		RTE_ETH_TX_OFFLOAD_UDP_CKSUM)
+
+#define INDIRECTION_TABLE_NUM_ELEMENTS 64
+#define TOEPLITZ_HASH_KEY_SIZE_IN_BYTES 40
+#define MANA_ETH_RSS_SUPPORT ( \
+	RTE_ETH_RSS_IPV4 |	     \
+	RTE_ETH_RSS_NONFRAG_IPV4_TCP | \
+	RTE_ETH_RSS_NONFRAG_IPV4_UDP | \
+	RTE_ETH_RSS_IPV6 |	     \
+	RTE_ETH_RSS_NONFRAG_IPV6_TCP | \
+	RTE_ETH_RSS_NONFRAG_IPV6_UDP)
+
+#define MIN_BUFFERS_PER_QUEUE		64
+#define MAX_RECEIVE_BUFFERS_PER_QUEUE	256
+#define MAX_SEND_BUFFERS_PER_QUEUE	256
+
 struct mana_process_priv {
 	void *db_page;
 };

75 lines not on this page · raw message

index a3165616ce..e67719e11d 100644
@@ -63,6 +63,7 @@ struct mana_priv {
 	struct ibv_pd *ib_pd;
 	struct ibv_pd *ib_parent_pd;
 	void *db_page;
+	struct rte_eth_rss_conf rss_conf;
 	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;

index 2bfda50f37..d19be6f852 100644
@@ -205,6 +205,17 @@ mana_dev_info_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void
+mana_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
+		       struct rte_eth_rxq_info *qinfo)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[queue_id];
+
+	qinfo->mp = rxq->mp;
+	qinfo->nb_desc = rxq->num_desc;
+	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
+}
+
 static const uint32_t *
 mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
 {

@@ -282,6 +293,63 @@ mana_rss_hash_conf_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int
+mana_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
+			uint16_t nb_desc, unsigned int socket_id,
+			const struct rte_eth_rxconf *rx_conf __rte_unused,
+			struct rte_mempool *mp)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct mana_rxq *rxq;
+	int ret;
+
+	rxq = rte_zmalloc_socket("mana_rxq", sizeof(*rxq), 0, socket_id);
+	if (!rxq) {
+		DRV_LOG(ERR, "failed to allocate rxq");
+		return -ENOMEM;
+	}
+
+	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u",
+		queue_idx, nb_desc, socket_id);
+
+	rxq->socket = socket_id;
+
+	rxq->desc_ring = rte_zmalloc_socket("mana_rx_mbuf_ring",
+					    sizeof(struct mana_rxq_desc) *
+						nb_desc,
+					    RTE_CACHE_LINE_SIZE, socket_id);
+
+	if (!rxq->desc_ring) {
+		DRV_LOG(ERR, "failed to allocate rxq desc_ring");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	rxq->desc_ring_head = 0;
+	rxq->desc_ring_tail = 0;
+
+	rxq->priv = priv;
+	rxq->num_desc = nb_desc;
+	rxq->mp = mp;
+	dev->data->rx_queues[queue_idx] = rxq;
+
+	return 0;
+
+fail:
+	rte_free(rxq->desc_ring);
+	rte_free(rxq);
+	return ret;
+}
+
+static void
+mana_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[qid];
+
+	rte_free(rxq->desc_ring);
+	rte_free(rxq);
+}
+
 static int
 mana_dev_link_update(struct rte_eth_dev *dev,
 		     int wait_to_complete __rte_unused)

@@ -303,9 +371,12 @@ static const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
+	.rxq_info_get		= mana_dev_rx_queue_info,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
 	.rss_hash_update	= mana_rss_hash_update,
 	.rss_hash_conf_get	= mana_rss_hash_conf_get,
+	.rx_queue_setup		= mana_dev_rx_queue_setup,
+	.rx_queue_release	= mana_dev_rx_queue_release,
 	.link_update		= mana_dev_link_update,
 };
 

index e67719e11d..4cd225fe6e 100644
@@ -75,6 +75,27 @@ struct mana_priv {
 	uint64_t max_mr_size;
 };
 
+struct mana_rxq_desc {
+	struct rte_mbuf *pkt;
+	uint32_t wqe_size_in_bu;
+};
+
+struct mana_rxq {
+	struct mana_priv *priv;
+	uint32_t num_desc;
+	struct rte_mempool *mp;
+
+	/* For storing pending requests */
+	struct mana_rxq_desc *desc_ring;
+
+	/* desc_ring_head is where we put pending requests to ring,
+	 * completion pull off desc_ring_tail
+	 */
+	uint32_t desc_ring_head, desc_ring_tail;
+
+	unsigned int socket;
+};
+
 extern int mana_logtype_driver;
 extern int mana_logtype_init;