Introduce Microsoft Azure Network Adatper (MANA) PMD

704 lines not on this page · raw message

210 lines not on this page · raw message

235 lines not on this page · raw message

index cb59eb6882..147ab144d5 100644
@@ -40,7 +40,79 @@ static rte_spinlock_t mana_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 int mana_logtype_driver;
 int mana_logtype_init;
 
+void *mana_alloc_verbs_buf(size_t size, void *data)
+{
+	void *ret;
+	size_t alignment = rte_mem_page_size();
+	int socket = (int)(uintptr_t)data;
+
+	DRV_LOG(DEBUG, "size=%zu socket=%d", size, socket);
+
+	if (alignment == (size_t)-1) {
+		DRV_LOG(ERR, "Failed to get mem page size");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+
+	ret = rte_zmalloc_socket("mana_verb_buf", size, alignment, socket);
+	if (!ret && size)
+		rte_errno = ENOMEM;
+	return ret;
+}
+
+void mana_free_verbs_buf(void *ptr, void *data __rte_unused)
+{
+	rte_free(ptr);
+}
+
+static int mana_dev_configure(struct rte_eth_dev *dev)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+
+	if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
+		dev_conf->rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
+
+	if (dev->data->nb_rx_queues != dev->data->nb_tx_queues) {
+		DRV_LOG(ERR, "Only support equal number of rx/tx queues");
+		return -EINVAL;
+	}
+
+	if (!rte_is_power_of_2(dev->data->nb_rx_queues)) {
+		DRV_LOG(ERR, "number of TX/RX queues must be power of 2");
+		return -EINVAL;
+	}
+
+	priv->num_queues = dev->data->nb_rx_queues;
+
+	manadv_set_context_attr(priv->ib_ctx, MANADV_CTX_ATTR_BUF_ALLOCATORS,
+				(void *)((uintptr_t)&(struct manadv_ctx_allocators){
+					.alloc = &mana_alloc_verbs_buf,
+					.free = &mana_free_verbs_buf,
+					.data = 0,
+				}));
+
+	return 0;
+}
+
+static int
+mana_dev_close(struct rte_eth_dev *dev)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	int ret;
+
+	ret = ibv_close_device(priv->ib_ctx);
+	if (ret) {
+		ret = errno;
+		return ret;
+	}
+
+	return 0;
+}
+
 const struct eth_dev_ops mana_dev_ops = {
+	.dev_configure		= mana_dev_configure,
+	.dev_close		= mana_dev_close,
 };
 
 const struct eth_dev_ops mana_dev_sec_ops = {

@@ -627,8 +699,7 @@ static int mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
 static int mana_dev_uninit(struct rte_eth_dev *dev)
 {
-	RTE_SET_USED(dev);
-	return 0;
+	return mana_dev_close(dev);
 }
 
 static int mana_pci_remove(struct rte_pci_device *pci_dev)

index e30c030b4e..66873394b9 100644
@@ -207,4 +207,7 @@ int mana_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev);
 
 void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 
+void *mana_alloc_verbs_buf(size_t size, void *data);
+void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
+
 #endif

index 147ab144d5..4559632056 100644
@@ -110,9 +110,25 @@ mana_dev_close(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static const uint32_t *mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
+{
+	static const uint32_t ptypes[] = {
+		RTE_PTYPE_L2_ETHER,
+		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+		RTE_PTYPE_L4_FRAG,
+		RTE_PTYPE_L4_TCP,
+		RTE_PTYPE_L4_UDP,
+		RTE_PTYPE_UNKNOWN
+	};
+
+	return ptypes;
+}
+
 const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
+	.dev_supported_ptypes_get = mana_supported_ptypes,
 };
 
 const struct eth_dev_ops mana_dev_sec_ops = {

index 66873394b9..c433940022 100644
@@ -168,8 +168,6 @@ extern int mana_logtype_init;
 
 #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
-const uint32_t *mana_supported_ptypes(struct rte_eth_dev *dev);
-
 uint16_t mana_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 			       uint16_t pkts_n);
 

index 4559632056..b77d0c29b0 100644
@@ -125,10 +125,27 @@ static const uint32_t *mana_supported_ptypes(struct rte_eth_dev *dev __rte_unuse
 	return ptypes;
 }
 
+static int mana_dev_link_update(struct rte_eth_dev *dev,
+				int wait_to_complete __rte_unused)
+{
+	struct rte_eth_link link;
+
+	/* MANA has no concept of carrier state, always reporting UP */
+	link = (struct rte_eth_link) {
+		.link_duplex = RTE_ETH_LINK_FULL_DUPLEX,
+		.link_autoneg = RTE_ETH_LINK_SPEED_FIXED,
+		.link_speed = RTE_ETH_SPEED_NUM_200G,
+		.link_status = RTE_ETH_LINK_UP,
+	};
+
+	return rte_eth_linkstatus_set(dev, &link);
+}
+
 const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
+	.link_update		= mana_dev_link_update,
 };
 
 const struct eth_dev_ops mana_dev_sec_ops = {

121 lines not on this page · raw message

index c433940022..f97eed2e81 100644
@@ -72,6 +72,7 @@ struct mana_priv {
 	uint8_t ind_table_key[40];
 	struct ibv_qp *rwq_qp;
 	void *db_page;
+	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;
 	int max_rx_desc;

99 lines not on this page · raw message

73 lines not on this page · raw message

index f97eed2e81..33f68b3d1b 100644
@@ -72,6 +72,7 @@ struct mana_priv {
 	uint8_t ind_table_key[40];
 	struct ibv_qp *rwq_qp;
 	void *db_page;
+	struct rte_eth_rss_conf rss_conf;
 	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;

index cb136f24c1..d03adab041 100644
@@ -196,6 +196,16 @@ static int mana_dev_info_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void mana_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
+				   struct rte_eth_rxq_info *qinfo)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[queue_id];
+
+	qinfo->mp = rxq->mp;
+	qinfo->nb_desc = rxq->num_desc;
+	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
+}
+
 static const uint32_t *mana_supported_ptypes(struct rte_eth_dev *dev __rte_unused)
 {
 	static const uint32_t ptypes[] = {

@@ -270,6 +280,61 @@ static int mana_rss_hash_conf_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int mana_dev_rx_queue_setup(struct rte_eth_dev *dev,
+				   uint16_t queue_idx, uint16_t nb_desc,
+				   unsigned int socket_id,
+				   const struct rte_eth_rxconf *rx_conf __rte_unused,
+				   struct rte_mempool *mp)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct mana_rxq *rxq;
+	int ret;
+
+	rxq = rte_zmalloc_socket("mana_rxq", sizeof(*rxq), 0, socket_id);
+	if (!rxq) {
+		DRV_LOG(ERR, "failed to allocate rxq");
+		return -ENOMEM;
+	}
+
+	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u",
+		queue_idx, nb_desc, socket_id);
+
+	rxq->socket = socket_id;
+
+	rxq->desc_ring = rte_zmalloc_socket("mana_rx_mbuf_ring",
+					    sizeof(struct mana_rxq_desc) *
+						nb_desc,
+					    RTE_CACHE_LINE_SIZE, socket_id);
+
+	if (!rxq->desc_ring) {
+		DRV_LOG(ERR, "failed to allocate rxq desc_ring");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	rxq->num_desc = nb_desc;
+
+	rxq->priv = priv;
+	rxq->num_desc = nb_desc;
+	rxq->mp = mp;
+	dev->data->rx_queues[queue_idx] = rxq;
+
+	return 0;
+
+fail:
+	rte_free(rxq->desc_ring);
+	rte_free(rxq);
+	return ret;
+}
+
+static void mana_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct mana_rxq *rxq = dev->data->rx_queues[qid];
+
+	rte_free(rxq->desc_ring);
+	rte_free(rxq);
+}
+
 static int mana_dev_link_update(struct rte_eth_dev *dev,
 				int wait_to_complete __rte_unused)
 {

@@ -290,9 +355,12 @@ const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
+	.rxq_info_get		= mana_dev_rx_queue_info,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
 	.rss_hash_update	= mana_rss_hash_update,
 	.rss_hash_conf_get	= mana_rss_hash_conf_get,
+	.rx_queue_setup		= mana_dev_rx_queue_setup,
+	.rx_queue_release	= mana_dev_rx_queue_release,
 	.link_update		= mana_dev_link_update,
 };
 

index d03adab041..490686f404 100644
@@ -196,6 +196,15 @@ static int mana_dev_info_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static void mana_dev_tx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
+			    struct rte_eth_txq_info *qinfo)
+{
+	struct mana_txq *txq = dev->data->tx_queues[queue_id];
+
+	qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
+	qinfo->nb_desc = txq->num_desc;
+}
+
 static void mana_dev_rx_queue_info(struct rte_eth_dev *dev, uint16_t queue_id,
 				   struct rte_eth_rxq_info *qinfo)
 {

@@ -280,6 +289,59 @@ static int mana_rss_hash_conf_get(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static int mana_dev_tx_queue_setup(struct rte_eth_dev *dev,
+				   uint16_t queue_idx, uint16_t nb_desc,
+				   unsigned int socket_id,
+				   const struct rte_eth_txconf *tx_conf __rte_unused)
+
+{
+	struct mana_priv *priv = dev->data->dev_private;
+	struct mana_txq *txq;
+	int ret;
+
+	txq = rte_zmalloc_socket("mana_txq", sizeof(*txq), 0, socket_id);
+	if (!txq) {
+		DRV_LOG(ERR, "failed to allocate txq");
+		return -ENOMEM;
+	}
+
+	txq->socket = socket_id;
+
+	txq->desc_ring = rte_malloc_socket("mana_tx_desc_ring",
+					   sizeof(struct mana_txq_desc) *
+						nb_desc,
+					   RTE_CACHE_LINE_SIZE, socket_id);
+	if (!txq->desc_ring) {
+		DRV_LOG(ERR, "failed to allocate txq desc_ring");
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u txq->desc_ring %p",
+		queue_idx, nb_desc, socket_id, txq->desc_ring);
+
+	txq->desc_ring_head = 0;
+	txq->desc_ring_tail = 0;
+	txq->priv = priv;
+	txq->num_desc = nb_desc;
+	dev->data->tx_queues[queue_idx] = txq;
+
+	return 0;
+
+fail:
+	rte_free(txq->desc_ring);
+	rte_free(txq);
+	return ret;
+}
+
+static void mana_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct mana_txq *txq = dev->data->tx_queues[qid];
+
+	rte_free(txq->desc_ring);
+	rte_free(txq);
+}
+
 static int mana_dev_rx_queue_setup(struct rte_eth_dev *dev,
 				   uint16_t queue_idx, uint16_t nb_desc,
 				   unsigned int socket_id,

@@ -355,10 +417,13 @@ const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
+	.txq_info_get		= mana_dev_tx_queue_info,
 	.rxq_info_get		= mana_dev_rx_queue_info,
 	.dev_supported_ptypes_get = mana_supported_ptypes,
 	.rss_hash_update	= mana_rss_hash_update,
 	.rss_hash_conf_get	= mana_rss_hash_conf_get,
+	.tx_queue_setup		= mana_dev_tx_queue_setup,
+	.tx_queue_release	= mana_dev_tx_queue_release,
 	.rx_queue_setup		= mana_dev_rx_queue_setup,
 	.rx_queue_release	= mana_dev_rx_queue_release,
 	.link_update		= mana_dev_link_update,

index 490686f404..d18cc4ab0e 100644
@@ -103,6 +103,8 @@ mana_dev_close(struct rte_eth_dev *dev)
 	struct mana_priv *priv = dev->data->dev_private;
 	int ret;
 
+	mana_remove_all_mr(priv);
+
 	ret = mana_intr_uninstall(priv);
 	if (ret)
 		return ret;

@@ -317,6 +319,13 @@ static int mana_dev_tx_queue_setup(struct rte_eth_dev *dev,
 		goto fail;
 	}
 
+	ret = mana_mr_btree_init(&txq->mr_btree,
+				 MANA_MR_BTREE_PER_QUEUE_N, socket_id);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to init TXQ MR btree");
+		goto fail;
+	}
+
 	DRV_LOG(DEBUG, "idx %u nb_desc %u socket %u txq->desc_ring %p",
 		queue_idx, nb_desc, socket_id, txq->desc_ring);
 

@@ -338,6 +347,8 @@ static void mana_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 {
 	struct mana_txq *txq = dev->data->tx_queues[qid];
 
+	mana_mr_btree_free(&txq->mr_btree);
+
 	rte_free(txq->desc_ring);
 	rte_free(txq);
 }

@@ -374,6 +385,13 @@ static int mana_dev_rx_queue_setup(struct rte_eth_dev *dev,
 		goto fail;
 	}
 
+	ret = mana_mr_btree_init(&rxq->mr_btree,
+				 MANA_MR_BTREE_PER_QUEUE_N, socket_id);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to init RXQ MR btree");
+		goto fail;
+	}
+
 	rxq->num_desc = nb_desc;
 
 	rxq->priv = priv;

@@ -393,6 +411,8 @@ static void mana_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 {
 	struct mana_rxq *rxq = dev->data->rx_queues[qid];
 
+	mana_mr_btree_free(&rxq->mr_btree);
+
 	rte_free(rxq->desc_ring);
 	rte_free(rxq);
 }

index 33f68b3d1b..9e15b43275 100644
@@ -50,6 +50,22 @@ struct mana_shared_data {
 #define MAX_RECEIVE_BUFFERS_PER_QUEUE	256
 #define MAX_SEND_BUFFERS_PER_QUEUE	256
 
+struct mana_mr_cache {
+	uint32_t	lkey;
+	uintptr_t	addr;
+	size_t		len;
+	void		*verb_obj;
+};
+
+#define MANA_MR_BTREE_CACHE_N	512
+struct mana_mr_btree {
+	uint16_t	len;	/* Used entries */
+	uint16_t	size;	/* Total entries */
+	int		overflow;
+	int		socket;
+	struct mana_mr_cache *table;
+};
+
 struct mana_process_priv {
 	void *db_page;
 };

@@ -82,6 +98,8 @@ struct mana_priv {
 	int max_recv_sge;
 	int max_mr;
 	uint64_t max_mr_size;
+	struct mana_mr_btree mr_btree;
+	rte_spinlock_t	mr_btree_lock;
 };
 
 struct mana_txq_desc {

@@ -131,6 +149,7 @@ struct mana_txq {
 	uint32_t desc_ring_head, desc_ring_tail;
 
 	struct mana_stats stats;
+	struct mana_mr_btree mr_btree;
 	unsigned int socket;
 };
 

@@ -153,6 +172,7 @@ struct mana_rxq {
 	struct mana_gdma_queue gdma_cq;
 
 	struct mana_stats stats;
+	struct mana_mr_btree mr_btree;
 
 	unsigned int socket;
 };

@@ -176,6 +196,24 @@ uint16_t mana_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 uint16_t mana_tx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 			       uint16_t pkts_n);
 
+struct mana_mr_cache *mana_find_pmd_mr(struct mana_mr_btree *local_tree,
+				       struct mana_priv *priv,
+				       struct rte_mbuf *mbuf);
+int mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv,
+		    struct rte_mempool *pool);
+void mana_remove_all_mr(struct mana_priv *priv);
+void mana_del_pmd_mr(struct mana_mr_cache *mr);
+
+void mana_mempool_chunk_cb(struct rte_mempool *mp, void *opaque,
+			   struct rte_mempool_memhdr *memhdr, unsigned int idx);
+
+struct mana_mr_cache *mana_mr_btree_lookup(struct mana_mr_btree *bt,
+					   uint16_t *idx,
+					   uintptr_t addr, size_t len);
+int mana_mr_btree_insert(struct mana_mr_btree *bt, struct mana_mr_cache *entry);
+int mana_mr_btree_init(struct mana_mr_btree *bt, int n, int socket);
+void mana_mr_btree_free(struct mana_mr_btree *bt);
+
 /** Request timeout for IPC. */
 #define MANA_MP_REQ_TIMEOUT_SEC 5
 

@@ -204,6 +242,7 @@ int mana_mp_init_secondary(void);
 void mana_mp_uninit_primary(void);
 void mana_mp_uninit_secondary(void);
 int mana_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev);
+int mana_mp_req_mr_create(struct mana_priv *priv, uintptr_t addr, uint32_t len);
 
 void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 

103 lines not on this page · raw message

324 lines not on this page · raw message

new file mode 100644
index 0000000000..077ac7744b
@@ -0,0 +1,284 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2022 Microsoft Corporation
+ */
+
+#include <ethdev_driver.h>
+#include <rte_io.h>
+
+#include "mana.h"
+
+uint8_t *gdma_get_wqe_pointer(struct mana_gdma_queue *queue)
+{
+	uint32_t offset_in_bytes =
+		(queue->head * GDMA_WQE_ALIGNMENT_UNIT_SIZE) &
+		(queue->size - 1);
+
+	DRV_LOG(DEBUG, "txq sq_head %u sq_size %u offset_in_bytes %u",
+		queue->head, queue->size, offset_in_bytes);
+
+	if (offset_in_bytes + GDMA_WQE_ALIGNMENT_UNIT_SIZE > queue->size)
+		DRV_LOG(ERR, "fatal error: offset_in_bytes %u too big",
+			offset_in_bytes);
+
+	return ((uint8_t *)queue->buffer) + offset_in_bytes;
+}
+
+static uint32_t
+write_dma_client_oob(uint8_t *work_queue_buffer_pointer,
+		     const struct gdma_work_request *work_request,
+		     uint32_t client_oob_size)
+{
+	uint8_t *p = work_queue_buffer_pointer;
+
+	struct gdma_wqe_dma_oob *header = (struct gdma_wqe_dma_oob *)p;
+
+	memset(header, 0, sizeof(struct gdma_wqe_dma_oob));
+	header->num_sgl_entries = work_request->num_sgl_elements;
+	header->inline_client_oob_size_in_dwords =
+		client_oob_size / sizeof(uint32_t);
+	header->client_data_unit = work_request->client_data_unit;
+
+	DRV_LOG(DEBUG, "queue buf %p sgl %u oob_h %u du %u oob_buf %p oob_b %u",
+		work_queue_buffer_pointer, header->num_sgl_entries,
+		header->inline_client_oob_size_in_dwords,
+		header->client_data_unit, work_request->inline_oob_data,
+		work_request->inline_oob_size_in_bytes);
+
+	p += sizeof(struct gdma_wqe_dma_oob);
+	if (work_request->inline_oob_data &&
+	    work_request->inline_oob_size_in_bytes > 0) {
+		memcpy(p, work_request->inline_oob_data,
+		       work_request->inline_oob_size_in_bytes);
+		if (client_oob_size > work_request->inline_oob_size_in_bytes)
+			memset(p + work_request->inline_oob_size_in_bytes, 0,
+			       client_oob_size -
+			       work_request->inline_oob_size_in_bytes);
+	}
+
+	return sizeof(struct gdma_wqe_dma_oob) + client_oob_size;
+}
+
+static uint32_t
+write_scatter_gather_list(uint8_t *work_queue_head_pointer,
+			  uint8_t *work_queue_end_pointer,
+			  uint8_t *work_queue_cur_pointer,
+			  struct gdma_work_request *work_request)
+{
+	struct gdma_sgl_element *sge_list;
+	struct gdma_sgl_element dummy_sgl[1];
+	uint8_t *address;
+	uint32_t size;
+	uint32_t num_sge;
+	uint32_t size_to_queue_end;
+	uint32_t sge_list_size;
+
+	DRV_LOG(DEBUG, "work_queue_cur_pointer %p work_request->flags %x",
+		work_queue_cur_pointer, work_request->flags);
+
+	num_sge = work_request->num_sgl_elements;
+	sge_list = work_request->sgl;
+	size_to_queue_end = (uint32_t)(work_queue_end_pointer -
+				       work_queue_cur_pointer);
+
+	if (num_sge == 0) {
+		/* Per spec, the case of an empty SGL should be handled as
+		 * follows to avoid corrupted WQE errors:
+		 * Write one dummy SGL entry
+		 * Set the address to 1, leave the rest as 0
+		 */
+		dummy_sgl[num_sge].address = 1;
+		dummy_sgl[num_sge].size = 0;
+		dummy_sgl[num_sge].memory_key = 0;
+		num_sge++;
+		sge_list = dummy_sgl;
+	}
+
+	sge_list_size = 0;
+	{
+		address = (uint8_t *)sge_list;
+		size = sizeof(struct gdma_sgl_element) * num_sge;
+		if (size_to_queue_end < size) {
+			memcpy(work_queue_cur_pointer, address,
+			       size_to_queue_end);
+			work_queue_cur_pointer = work_queue_head_pointer;
+			address += size_to_queue_end;
+			size -= size_to_queue_end;
+		}
+
+		memcpy(work_queue_cur_pointer, address, size);
+		sge_list_size = size;
+	}
+
+	DRV_LOG(DEBUG, "sge %u address 0x%" PRIx64 " size %u key %u list_s %u",
+		num_sge, sge_list->address, sge_list->size,
+		sge_list->memory_key, sge_list_size);
+
+	return sge_list_size;
+}
+
+int gdma_post_work_request(struct mana_gdma_queue *queue,
+			   struct gdma_work_request *work_req,
+			   struct gdma_posted_wqe_info *wqe_info)
+{
+	uint32_t client_oob_size =
+		work_req->inline_oob_size_in_bytes >
+				INLINE_OOB_SMALL_SIZE_IN_BYTES ?
+			INLINE_OOB_LARGE_SIZE_IN_BYTES :
+			INLINE_OOB_SMALL_SIZE_IN_BYTES;
+
+	uint32_t sgl_data_size = sizeof(struct gdma_sgl_element) *
+			RTE_MAX((uint32_t)1, work_req->num_sgl_elements);
+	uint32_t wqe_size =
+		RTE_ALIGN(sizeof(struct gdma_wqe_dma_oob) +
+				client_oob_size + sgl_data_size,
+			  GDMA_WQE_ALIGNMENT_UNIT_SIZE);
+	uint8_t *wq_buffer_pointer;
+	uint32_t queue_free_units = queue->count - (queue->head - queue->tail);
+
+	if (wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE > queue_free_units) {
+		DRV_LOG(DEBUG, "WQE size %u queue count %u head %u tail %u",
+			wqe_size, queue->count, queue->head, queue->tail);
+		return -EBUSY;
+	}
+
+	DRV_LOG(DEBUG, "client_oob_size %u sgl_data_size %u wqe_size %u",
+		client_oob_size, sgl_data_size, wqe_size);
+
+	if (wqe_info) {
+		wqe_info->wqe_index =
+			((queue->head * GDMA_WQE_ALIGNMENT_UNIT_SIZE) &
+			 (queue->size - 1)) / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+		wqe_info->unmasked_queue_offset = queue->head;
+		wqe_info->wqe_size_in_bu =
+			wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+	}
+
+	wq_buffer_pointer = gdma_get_wqe_pointer(queue);
+	wq_buffer_pointer += write_dma_client_oob(wq_buffer_pointer, work_req,
+						  client_oob_size);
+	if (wq_buffer_pointer >= ((uint8_t *)queue->buffer) + queue->size)
+		wq_buffer_pointer -= queue->size;
+
+	write_scatter_gather_list((uint8_t *)queue->buffer,
+				  (uint8_t *)queue->buffer + queue->size,
+				  wq_buffer_pointer, work_req);
+
+	queue->head += wqe_size / GDMA_WQE_ALIGNMENT_UNIT_SIZE;
+
+	return 0;
+}
+
+union gdma_doorbell_entry {
+	uint64_t     as_uint64;
+
+	struct {
+		uint64_t id	  : 24;
+		uint64_t reserved    : 8;
+		uint64_t tail_ptr    : 31;
+		uint64_t arm	 : 1;
+	} cq;
+
+	struct {
+		uint64_t id	  : 24;
+		uint64_t wqe_cnt     : 8;
+		uint64_t tail_ptr    : 32;
+	} rq;
+
+	struct {
+		uint64_t id	  : 24;
+		uint64_t reserved    : 8;
+		uint64_t tail_ptr    : 32;
+	} sq;
+
+	struct {
+		uint64_t id	  : 16;
+		uint64_t reserved    : 16;
+		uint64_t tail_ptr    : 31;
+		uint64_t arm	 : 1;
+	} eq;
+}; /* HW DATA */
+
+#define DOORBELL_OFFSET_SQ      0x0
+#define DOORBELL_OFFSET_RQ      0x400
+#define DOORBELL_OFFSET_CQ      0x800
+#define DOORBELL_OFFSET_EQ      0xFF8
+
+int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
+		       uint32_t queue_id, uint32_t tail)
+{
+	uint8_t *addr = db_page;
+	union gdma_doorbell_entry e = {};
+
+	switch (queue_type) {
+	case gdma_queue_send:
+		e.sq.id = queue_id;
+		e.sq.tail_ptr = tail;
+		addr += DOORBELL_OFFSET_SQ;
+		break;
+
+	case gdma_queue_receive:
+		e.rq.id = queue_id;
+		e.rq.tail_ptr = tail;
+		e.rq.wqe_cnt = 1;
+		addr += DOORBELL_OFFSET_RQ;
+		break;
+
+	case gdma_queue_completion:
+		e.cq.id = queue_id;
+		e.cq.tail_ptr = tail;
+		e.cq.arm = 1;
+		addr += DOORBELL_OFFSET_CQ;
+		break;
+
+	default:
+		DRV_LOG(ERR, "Unsupported queue type %d", queue_type);
+		return -1;
+	}
+
+	rte_wmb();
+	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u",
+		db_page, addr, queue_id, queue_type, tail);
+
+	rte_write64(e.as_uint64, addr);
+	return 0;
+}
+
+int gdma_poll_completion_queue(struct mana_gdma_queue *cq,
+			       struct gdma_comp *comp)
+{
+	struct gdma_hardware_completion_entry *cqe;
+	uint32_t head = cq->head % cq->count;
+	uint32_t new_owner_bits, old_owner_bits;
+	uint32_t cqe_owner_bits;
+	struct gdma_hardware_completion_entry *buffer = cq->buffer;
+
+	cqe = &buffer[head];
+	new_owner_bits = (cq->head / cq->count) & COMPLETION_QUEUE_OWNER_MASK;
+	old_owner_bits = (cq->head / cq->count - 1) &
+				COMPLETION_QUEUE_OWNER_MASK;
+	cqe_owner_bits = cqe->owner_bits;
+
+	DRV_LOG(DEBUG, "comp cqe bits 0x%x owner bits 0x%x",
+		cqe_owner_bits, old_owner_bits);
+
+	if (cqe_owner_bits == old_owner_bits)
+		return 0; /* No new entry */
+
+	if (cqe_owner_bits != new_owner_bits) {
+		DRV_LOG(ERR, "CQ overflowed, ID %u cqe 0x%x new 0x%x",
+			cq->id, cqe_owner_bits, new_owner_bits);
+		return -1;
+	}
+
+	comp->work_queue_number = cqe->wq_num;
+	comp->send_work_queue = cqe->is_sq;
+
+	memcpy(comp->completion_data, cqe->dma_client_data, GDMA_COMP_DATA_SIZE);
+
+	cq->head++;
+
+	DRV_LOG(DEBUG, "comp new 0x%x old 0x%x cqe 0x%x wq %u sq %u head %u",
+		new_owner_bits, old_owner_bits, cqe_owner_bits,
+		comp->work_queue_number, comp->send_work_queue, cq->head);
+	return 1;
+}

200 lines not on this page · raw message

index 4733a62b87..9a2253fe1a 100644
@@ -378,6 +378,10 @@ uint16_t mana_tx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 int gdma_poll_completion_queue(struct mana_gdma_queue *cq,
 			       struct gdma_comp *comp);
 
+int mana_start_tx_queues(struct rte_eth_dev *dev);
+
+int mana_stop_tx_queues(struct rte_eth_dev *dev);
+
 struct mana_mr_cache *mana_find_pmd_mr(struct mana_mr_btree *local_tree,
 				       struct mana_priv *priv,
 				       struct rte_mbuf *mbuf);

157 lines not on this page · raw message

index 9a2253fe1a..1141464d74 100644
@@ -363,6 +363,7 @@ extern int mana_logtype_init;
 
 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 		       uint32_t queue_id, uint32_t tail);
+int mana_rq_ring_doorbell(struct mana_rxq *rxq);
 
 int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_work_request *work_req,

@@ -378,8 +379,10 @@ uint16_t mana_tx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 int gdma_poll_completion_queue(struct mana_gdma_queue *cq,
 			       struct gdma_comp *comp);
 
+int mana_start_rx_queues(struct rte_eth_dev *dev);
 int mana_start_tx_queues(struct rte_eth_dev *dev);
 
+int mana_stop_rx_queues(struct rte_eth_dev *dev);
 int mana_stop_tx_queues(struct rte_eth_dev *dev);
 
 struct mana_mr_cache *mana_find_pmd_mr(struct mana_mr_btree *local_tree,

346 lines not on this page · raw message

index d18cc4ab0e..c349822991 100644
@@ -950,6 +950,8 @@ static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv __rte_unused,
 				/* fd is no not used after mapping doorbell */
 				close(fd);
 
+				eth_dev->rx_pkt_burst = mana_rx_burst;
+
 				rte_spinlock_lock(&mana_shared_data->lock);
 				mana_shared_data->secondary_cnt++;
 				mana_local_data.secondary_cnt++;

index 1141464d74..10229600fe 100644
@@ -178,6 +178,11 @@ struct gdma_work_request {
 
 enum mana_cqe_type {
 	CQE_INVALID                     = 0,
+
+	CQE_RX_OKAY                     = 1,
+	CQE_RX_COALESCED_4              = 2,
+	CQE_RX_OBJECT_FENCE             = 3,
+	CQE_RX_TRUNCATED                = 4,
 };
 
 struct mana_cqe_header {

@@ -202,6 +207,35 @@ struct mana_cqe_header {
 	(NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 |      \
 	 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
 
+struct mana_rx_comp_per_packet_info {
+	uint32_t packet_length	: 16;
+	uint32_t reserved0	: 16;
+	uint32_t reserved1;
+	uint32_t packet_hash;
+}; /* HW DATA */
+#define RX_COM_OOB_NUM_PACKETINFO_SEGMENTS 4
+
+struct mana_rx_comp_oob {
+	struct mana_cqe_header cqe_hdr;
+
+	uint32_t rx_vlan_id				: 12;
+	uint32_t rx_vlan_tag_present			: 1;
+	uint32_t rx_outer_ip_header_checksum_succeeded	: 1;
+	uint32_t rx_outer_ip_header_checksum_failed	: 1;
+	uint32_t reserved				: 1;
+	uint32_t rx_hash_type				: 9;
+	uint32_t rx_ip_header_checksum_succeeded	: 1;
+	uint32_t rx_ip_header_checksum_failed		: 1;
+	uint32_t rx_tcp_checksum_succeeded		: 1;
+	uint32_t rx_tcp_checksum_failed			: 1;
+	uint32_t rx_udp_checksum_succeeded		: 1;
+	uint32_t rx_udp_checksum_failed			: 1;
+	uint32_t reserved1				: 1;
+	struct mana_rx_comp_per_packet_info
+		packet_info[RX_COM_OOB_NUM_PACKETINFO_SEGMENTS];
+	uint32_t received_wqe_offset;
+}; /* HW DATA */
+
 struct gdma_wqe_dma_oob {
 	uint32_t reserved:24;
 	uint32_t last_v_bytes:8;

@@ -370,6 +404,9 @@ int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_posted_wqe_info *wqe_info);
 uint8_t *gdma_get_wqe_pointer(struct mana_gdma_queue *queue);
 
+uint16_t mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **rx_pkts,
+		       uint16_t pkts_n);
+
 uint16_t mana_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts,
 			       uint16_t pkts_n);
 

index f4f78d2787..36a88c561a 100644
@@ -138,6 +138,8 @@ static int mana_mp_secondary_handle(const struct rte_mp_msg *mp_msg,
 	case MANA_MP_REQ_START_RXTX:
 		DRV_LOG(INFO, "Port %u starting datapath", dev->data->port_id);
 
+		dev->rx_pkt_burst = mana_rx_burst;
+
 		rte_mb();
 
 		res->result = 0;

107 lines not on this page · raw message

index c349822991..0dcd3f3124 100644
@@ -950,6 +950,7 @@ static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv __rte_unused,
 				/* fd is no not used after mapping doorbell */
 				close(fd);
 
+				eth_dev->tx_pkt_burst = mana_tx_burst;
 				eth_dev->rx_pkt_burst = mana_rx_burst;
 
 				rte_spinlock_lock(&mana_shared_data->lock);

89 lines not on this page · raw message

index 36a88c561a..da9c0f36a1 100644
@@ -138,6 +138,7 @@ static int mana_mp_secondary_handle(const struct rte_mp_msg *mp_msg,
 	case MANA_MP_REQ_START_RXTX:
 		DRV_LOG(INFO, "Port %u starting datapath", dev->data->port_id);
 
+		dev->tx_pkt_burst = mana_tx_burst;
 		dev->rx_pkt_burst = mana_rx_burst;
 
 		rte_mb();

244 lines not on this page · raw message

index 0dcd3f3124..119fdfd8bc 100644
@@ -97,6 +97,74 @@ static int mana_dev_configure(struct rte_eth_dev *dev)
 
 static int mana_intr_uninstall(struct mana_priv *priv);
 
+static int
+mana_dev_start(struct rte_eth_dev *dev)
+{
+	int ret;
+	struct mana_priv *priv = dev->data->dev_private;
+
+	rte_spinlock_init(&priv->mr_btree_lock);
+	ret = mana_mr_btree_init(&priv->mr_btree, MANA_MR_BTREE_CACHE_N,
+				 dev->device->numa_node);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to init device MR btree %d", ret);
+		return ret;
+	}
+
+	ret = mana_start_tx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to start tx queues %d", ret);
+		return ret;
+	}
+
+	ret = mana_start_rx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to start rx queues %d", ret);
+		mana_stop_tx_queues(dev);
+		return ret;
+	}
+
+	rte_wmb();
+
+	dev->tx_pkt_burst = mana_tx_burst;
+	dev->rx_pkt_burst = mana_rx_burst;
+
+	DRV_LOG(INFO, "TX/RX queues have started");
+
+	/* Enable datapath for secondary processes */
+	mana_mp_req_on_rxtx(dev, MANA_MP_REQ_START_RXTX);
+
+	return 0;
+}
+
+static int
+mana_dev_stop(struct rte_eth_dev *dev __rte_unused)
+{
+	int ret;
+
+	dev->tx_pkt_burst = mana_tx_burst_removed;
+	dev->rx_pkt_burst = mana_rx_burst_removed;
+
+	/* Stop datapath on secondary processes */
+	mana_mp_req_on_rxtx(dev, MANA_MP_REQ_STOP_RXTX);
+
+	rte_wmb();
+
+	ret = mana_stop_tx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to stop tx queues");
+		return ret;
+	}
+
+	ret = mana_stop_rx_queues(dev);
+	if (ret) {
+		DRV_LOG(ERR, "failed to stop tx queues");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 mana_dev_close(struct rte_eth_dev *dev)
 {

@@ -435,6 +503,8 @@ static int mana_dev_link_update(struct rte_eth_dev *dev,
 
 const struct eth_dev_ops mana_dev_ops = {
 	.dev_configure		= mana_dev_configure,
+	.dev_start		= mana_dev_start,
+	.dev_stop		= mana_dev_stop,
 	.dev_close		= mana_dev_close,
 	.dev_infos_get		= mana_dev_info_get,
 	.txq_info_get		= mana_dev_tx_queue_info,

91 lines not on this page · raw message

index 077ac7744b..d701c4bd31 100644
@@ -204,7 +204,7 @@ union gdma_doorbell_entry {
 #define DOORBELL_OFFSET_EQ      0xFF8
 
 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
-		       uint32_t queue_id, uint32_t tail)
+		       uint32_t queue_id, uint32_t tail, uint8_t arm)
 {
 	uint8_t *addr = db_page;
 	union gdma_doorbell_entry e = {};

@@ -219,14 +219,14 @@ int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 	case gdma_queue_receive:
 		e.rq.id = queue_id;
 		e.rq.tail_ptr = tail;
-		e.rq.wqe_cnt = 1;
+		e.rq.wqe_cnt = arm;
 		addr += DOORBELL_OFFSET_RQ;
 		break;
 
 	case gdma_queue_completion:
 		e.cq.id = queue_id;
 		e.cq.tail_ptr = tail;
-		e.cq.arm = 1;
+		e.cq.arm = arm;
 		addr += DOORBELL_OFFSET_CQ;
 		break;
 

@@ -236,8 +236,8 @@ int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
 	}
 
 	rte_wmb();
-	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u",
-		db_page, addr, queue_id, queue_type, tail);
+	DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u",
+		db_page, addr, queue_id, queue_type, tail, arm);
 
 	rte_write64(e.as_uint64, addr);
 	return 0;

200 lines not on this page · raw message

index 0581d17e33..3b1ce20dd4 100644
@@ -426,6 +426,7 @@ struct mana_rxq {
 	uint32_t num_desc;
 	struct rte_mempool *mp;
 	struct ibv_cq *cq;
+	struct ibv_comp_channel *channel;
 	struct ibv_wq *wq;
 
 	/* For storing pending requests */

@@ -459,8 +460,8 @@ extern int mana_logtype_init;
 #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type,
-		       uint32_t queue_id, uint32_t tail);
-int mana_rq_ring_doorbell(struct mana_rxq *rxq);
+		       uint32_t queue_id, uint32_t tail, uint8_t arm);
+int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm);
 
 int gdma_post_work_request(struct mana_gdma_queue *queue,
 			   struct gdma_work_request *work_req,

@@ -495,6 +496,10 @@ int mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv,
 void mana_remove_all_mr(struct mana_priv *priv);
 void mana_del_pmd_mr(struct mana_mr_cache *mr);
 
+void mana_intr_handler(void *arg);
+int mana_intr_install(struct rte_eth_dev *eth_dev, struct mana_priv *priv);
+int mana_intr_uninstall(struct mana_priv *priv);
+
 void mana_mempool_chunk_cb(struct rte_mempool *mp, void *opaque,
 			   struct rte_mempool_memhdr *memhdr, unsigned int idx);
 

@@ -540,4 +545,8 @@ void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type);
 void *mana_alloc_verbs_buf(size_t size, void *data);
 void mana_free_verbs_buf(void *ptr, void *data __rte_unused);
 
+int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int mana_fd_set_non_blocking(int fd);
+
 #endif

142 lines not on this page · raw message

index 26340311c9..68db131868 100644
@@ -390,7 +390,8 @@ uint16_t mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts,
 	ret = mana_ring_doorbell(db_page, gdma_queue_send,
 				 txq->gdma_sq.id,
 				 txq->gdma_sq.head *
-					GDMA_WQE_ALIGNMENT_UNIT_SIZE);
+					GDMA_WQE_ALIGNMENT_UNIT_SIZE,
+				 0);
 	if (ret)
 		DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret);