Add ODM DMA device

new file mode 100644
index 0000000000..aeeb6f9e9a
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#ifndef _ODM_H_
+#define _ODM_H_
+
+#include <rte_log.h>
+
+extern int odm_logtype;
+
+#define odm_err(...)                                                                               \
+	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_info(...)                                                                              \
+	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+
+struct __rte_cache_aligned odm_dev {
+	struct rte_pci_device *pci_dev;
+	uint8_t *rbase;
+	uint16_t vfid;
+	uint8_t max_qs;
+	uint8_t num_qs;
+};
+
+#endif /* _ODM_H_ */

74 lines not on this page · raw message

index aeeb6f9e9a..7564ffbed4 100644
@@ -9,6 +9,47 @@
 
 extern int odm_logtype;
 
+/* ODM VF register offsets from VF_BAR0 */
+#define ODM_VDMA_EN(x)		(0x00 | (x << 3))
+#define ODM_VDMA_REQQ_CTL(x)	(0x80 | (x << 3))
+#define ODM_VDMA_DBELL(x)	(0x100 | (x << 3))
+#define ODM_VDMA_RING_CFG(x)	(0x180 | (x << 3))
+#define ODM_VDMA_IRING_BADDR(x) (0x200 | (x << 3))
+#define ODM_VDMA_CRING_BADDR(x) (0x280 | (x << 3))
+#define ODM_VDMA_COUNTS(x)	(0x300 | (x << 3))
+#define ODM_VDMA_IRING_NADDR(x) (0x380 | (x << 3))
+#define ODM_VDMA_CRING_NADDR(x) (0x400 | (x << 3))
+#define ODM_VDMA_IRING_DBG(x)	(0x480 | (x << 3))
+#define ODM_VDMA_CNT(x)		(0x580 | (x << 3))
+#define ODM_VF_INT		(0x1000)
+#define ODM_VF_INT_W1S		(0x1008)
+#define ODM_VF_INT_ENA_W1C	(0x1010)
+#define ODM_VF_INT_ENA_W1S	(0x1018)
+#define ODM_MBOX_VF_PF_DATA(i)	(0x2000 | (i << 3))
+
+#define ODM_MBOX_RETRY_CNT	(0xfffffff)
+#define ODM_MBOX_ERR_CODE_MAX	(0xf)
+#define ODM_IRING_IDLE_WAIT_CNT (0xfffffff)
+
+/**
+ * Enumeration odm_hdr_xtype_e
+ *
+ * ODM Transfer Type Enumeration
+ * Enumerates the pointer type in ODM_DMA_INSTR_HDR_S[XTYPE]
+ */
+#define ODM_XTYPE_INTERNAL 2
+#define ODM_XTYPE_FILL0	   4
+#define ODM_XTYPE_FILL1	   5
+
+/**
+ *  ODM Header completion type enumeration
+ *  Enumerates the completion type in ODM_DMA_INSTR_HDR_S[CT]
+ */
+#define ODM_HDR_CT_CW_CA 0x0
+#define ODM_HDR_CT_CW_NC 0x1
+
+#define ODM_MAX_QUEUES_PER_DEV 16
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -18,6 +59,81 @@ extern int odm_logtype;
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
 
+/**
+ * Structure odm_instr_hdr_s for ODM
+ *
+ * ODM DMA Instruction Header Format
+ */
+union odm_instr_hdr_s {
+	uint64_t u;
+	struct odm_instr_hdr {
+		uint64_t nfst : 3;
+		uint64_t reserved_3 : 1;
+		uint64_t nlst : 3;
+		uint64_t reserved_7_9 : 3;
+		uint64_t ct : 2;
+		uint64_t stse : 1;
+		uint64_t reserved_13_28 : 16;
+		uint64_t sts : 1;
+		uint64_t reserved_30_49 : 20;
+		uint64_t xtype : 3;
+		uint64_t reserved_53_63 : 11;
+	} s;
+};
+
+/**
+ *  ODM Completion Entry Structure
+ *
+ */
+union odm_cmpl_ent_s {
+	uint32_t u;
+	struct odm_cmpl_ent {
+		uint32_t cmp_code : 8;
+		uint32_t rsvd : 23;
+		uint32_t valid : 1;
+	} s;
+};
+
+/**
+ * ODM DMA Ring Configuration Register
+ */
+union odm_vdma_ring_cfg_s {
+	uint64_t u;
+	struct {
+		uint64_t isize : 8;
+		uint64_t rsvd_8_15 : 8;
+		uint64_t csize : 8;
+		uint64_t rsvd_24_63 : 40;
+	} s;
+};
+
+/**
+ * ODM DMA Instruction Ring DBG
+ */
+union odm_vdma_iring_dbg_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell_cnt : 32;
+		uint64_t offset : 16;
+		uint64_t rsvd_48_62 : 15;
+		uint64_t iwbusy : 1;
+	} s;
+};
+
+/**
+ * ODM DMA Counts
+ */
+union odm_vdma_counts_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell : 32;
+		uint64_t buf_used_cnt : 9;
+		uint64_t rsvd_41_43 : 3;
+		uint64_t rsvd_buf_used_cnt : 3;
+		uint64_t rsvd_47_63 : 17;
+	} s;
+};
+
 struct __rte_cache_aligned odm_dev {
 	struct rte_pci_device *pci_dev;
 	uint8_t *rbase;

49 lines not on this page · raw message

97 lines not on this page · raw message

index 7564ffbed4..9fd3e30ad8 100644
@@ -5,6 +5,10 @@
 #ifndef _ODM_H_
 #define _ODM_H_
 
+#include <stdint.h>
+
+#include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_log.h>
 
 extern int odm_logtype;

@@ -50,6 +54,9 @@ extern int odm_logtype;
 
 #define ODM_MAX_QUEUES_PER_DEV 16
 
+#define odm_read64(addr)       rte_read64_relaxed((volatile void *)(addr))
+#define odm_write64(val, addr) rte_write64_relaxed((val), (volatile void *)(addr))
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -142,4 +149,7 @@ struct __rte_cache_aligned odm_dev {
 	uint8_t num_qs;
 };
 
+int odm_dev_init(struct odm_dev *odm);
+int odm_dev_fini(struct odm_dev *odm);
+
 #endif /* _ODM_H_ */

index cc3342cf7b..bef335c10c 100644
@@ -23,6 +23,7 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	char name[RTE_DEV_NAME_MAX_LEN];
 	struct odm_dev *odm = NULL;
 	struct rte_dma_dev *dmadev;
+	int rc;
 
 	if (!pci_dev->mem_resource[0].addr)
 		return -ENODEV;

@@ -37,8 +38,20 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	}
 
 	odm_info("DMA device %s probed", name);
+	odm = dmadev->data->dev_private;
+
+	odm->pci_dev = pci_dev;
+
+	rc = odm_dev_init(odm);
+	if (rc < 0)
+		goto dma_pmd_release;
 
 	return 0;
+
+dma_pmd_release:
+	rte_dma_pmd_release(name);
+
+	return rc;
 }
 
 static int

index c0963da451..6094ace9fd 100644
@@ -7,6 +7,7 @@
 #include <bus_pci_driver.h>
 
 #include <rte_io.h>
+#include <rte_malloc.h>
 
 #include "odm.h"
 #include "odm_priv.h"

@@ -14,8 +15,15 @@
 static void
 odm_vchan_resc_free(struct odm_dev *odm, int qno)
 {
-	RTE_SET_USED(odm);
-	RTE_SET_USED(qno);
+	struct odm_queue *vq = &odm->vq[qno];
+
+	rte_memzone_free(vq->iring_mz);
+	rte_memzone_free(vq->cring_mz);
+	rte_free(vq->extra_ins_sz);
+
+	vq->iring_mz = NULL;
+	vq->cring_mz = NULL;
+	vq->extra_ins_sz = NULL;
 }
 
 static int

@@ -53,6 +61,138 @@ send_mbox_to_pf(struct odm_dev *odm, union odm_mbox_msg *msg, union odm_mbox_msg
 	return 0;
 }
 
+static int
+odm_queue_ring_config(struct odm_dev *odm, int vchan, int isize, int csize)
+{
+	union odm_vdma_ring_cfg_s ring_cfg = {0};
+	struct odm_queue *vq = &odm->vq[vchan];
+
+	if (vq->iring_mz == NULL || vq->cring_mz == NULL)
+		return -EINVAL;
+
+	ring_cfg.s.isize = (isize / 1024) - 1;
+	ring_cfg.s.csize = (csize / 1024) - 1;
+
+	odm_write64(ring_cfg.u, odm->rbase + ODM_VDMA_RING_CFG(vchan));
+	odm_write64(vq->iring_mz->iova, odm->rbase + ODM_VDMA_IRING_BADDR(vchan));
+	odm_write64(vq->cring_mz->iova, odm->rbase + ODM_VDMA_CRING_BADDR(vchan));
+
+	return 0;
+}
+
+int
+odm_enable(struct odm_dev *odm)
+{
+	struct odm_queue *vq;
+	int qno, rc = 0;
+
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		vq = &odm->vq[qno];
+
+		vq->desc_idx = vq->stats.completed_offset;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+		vq->iring_head = 0;
+		vq->cring_head = 0;
+		vq->ins_ring_head = 0;
+		vq->iring_sz_available = vq->iring_max_words;
+
+		rc = odm_queue_ring_config(odm, qno, vq->iring_max_words * 8,
+					   vq->cring_max_entry * 4);
+		if (rc < 0)
+			break;
+
+		odm_write64(0x1, odm->rbase + ODM_VDMA_EN(qno));
+	}
+
+	return rc;
+}
+
+int
+odm_disable(struct odm_dev *odm)
+{
+	int qno, wait_cnt = ODM_IRING_IDLE_WAIT_CNT;
+	uint64_t val;
+
+	/* Disable the queue and wait for the queue to became idle */
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		odm_write64(0x0, odm->rbase + ODM_VDMA_EN(qno));
+		do {
+			val = odm_read64(odm->rbase + ODM_VDMA_IRING_BADDR(qno));
+		} while ((!(val & 1ULL << 63)) && (--wait_cnt > 0));
+	}
+
+	return 0;
+}
+
+int
+odm_vchan_setup(struct odm_dev *odm, int vchan, int nb_desc)
+{
+	struct odm_queue *vq = &odm->vq[vchan];
+	int isize, csize, max_nb_desc, rc = 0;
+	union odm_mbox_msg mbox_msg;
+	const struct rte_memzone *mz;
+	char name[32];
+
+	if (vq->iring_mz != NULL)
+		odm_vchan_resc_free(odm, vchan);
+
+	mbox_msg.u[0] = 0;
+	mbox_msg.u[1] = 0;
+
+	/* ODM PF driver expects vfid starts from index 0 */
+	mbox_msg.q.vfid = odm->vfid;
+	mbox_msg.q.cmd = ODM_QUEUE_OPEN;
+	mbox_msg.q.qidx = vchan;
+	rc = send_mbox_to_pf(odm, &mbox_msg, &mbox_msg);
+	if (rc < 0)
+		return rc;
+
+	/* Determine instruction & completion ring sizes. */
+
+	/* Create iring that can support nb_desc. Round up to a multiple of 1024. */
+	isize = RTE_ALIGN_CEIL(nb_desc * ODM_IRING_ENTRY_SIZE_MAX * 8, 1024);
+	isize = RTE_MIN(isize, ODM_IRING_MAX_SIZE);
+	snprintf(name, sizeof(name), "vq%d_iring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, isize, 0, ODM_MEMZONE_FLAGS, 1024);
+	if (mz == NULL)
+		return -ENOMEM;
+	vq->iring_mz = mz;
+	vq->iring_max_words = isize / 8;
+
+	/* Create cring that can support max instructions that can be inflight in hw. */
+	max_nb_desc = (isize / (ODM_IRING_ENTRY_SIZE_MIN * 8));
+	csize = RTE_ALIGN_CEIL(max_nb_desc * sizeof(union odm_cmpl_ent_s), 1024);
+	snprintf(name, sizeof(name), "vq%d_cring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, csize, 0, ODM_MEMZONE_FLAGS, 1024);
+	if (mz == NULL) {
+		rc = -ENOMEM;
+		goto iring_free;
+	}
+	vq->cring_mz = mz;
+	vq->cring_max_entry = csize / 4;
+
+	/* Allocate memory to track the size of each instruction. */
+	snprintf(name, sizeof(name), "vq%d_extra%d", odm->vfid, vchan);
+	vq->extra_ins_sz = rte_zmalloc(name, vq->cring_max_entry, 0);
+	if (vq->extra_ins_sz == NULL) {
+		rc = -ENOMEM;
+		goto cring_free;
+	}
+
+	vq->stats = (struct vq_stats){0};
+	return rc;
+
+cring_free:
+	rte_memzone_free(odm->vq[vchan].cring_mz);
+	vq->cring_mz = NULL;
+iring_free:
+	rte_memzone_free(odm->vq[vchan].iring_mz);
+	vq->iring_mz = NULL;
+
+	return rc;
+}
+
 int
 odm_dev_init(struct odm_dev *odm)
 {

90 lines not on this page · raw message

97 lines not on this page · raw message

index 8c705978fe..13b2588246 100644
@@ -87,14 +87,73 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
+	      uint32_t size)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+
+	if (size < sizeof(rte_stats))
+		return -EINVAL;
+	if (rte_stats == NULL)
+		return -EINVAL;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[vchan].stats;
+
+		*rte_stats = *stats;
+	} else {
+		int i;
+
+		for (i = 0; i < odm->num_qs; i++) {
+			struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[i].stats;
+
+			rte_stats->submitted += stats->submitted;
+			rte_stats->completed += stats->completed;
+			rte_stats->errors += stats->errors;
+		}
+	}
+
+	return 0;
+}
+
+static void
+odm_vq_stats_reset(struct vq_stats *vq_stats)
+{
+	vq_stats->completed_offset += vq_stats->completed;
+	vq_stats->completed = 0;
+	vq_stats->errors = 0;
+	vq_stats->submitted = 0;
+}
+
+static int
+odm_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+	struct vq_stats *vq_stats;
+	int i;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		vq_stats = &odm->vq[vchan].stats;
+		odm_vq_stats_reset(vq_stats);
+	} else {
+		for (i = 0; i < odm->num_qs; i++) {
+			vq_stats = &odm->vq[i].stats;
+			odm_vq_stats_reset(vq_stats);
+		}
+	}
+
+	return 0;
+}
+
 static const struct rte_dma_dev_ops odm_dmadev_ops = {
 	.dev_close = odm_dmadev_close,
 	.dev_configure = odm_dmadev_configure,
 	.dev_info_get = odm_dmadev_info_get,
 	.dev_start = odm_dmadev_start,
 	.dev_stop = odm_dmadev_stop,
-	.stats_get = NULL,
-	.stats_reset = NULL,
+	.stats_get = odm_stats_get,
+	.stats_reset = odm_stats_reset,
 	.vchan_setup = odm_dmadev_vchan_setup,
 };
 

index 13b2588246..327692426f 100644
@@ -9,6 +9,7 @@
 #include <rte_common.h>
 #include <rte_dmadev.h>
 #include <rte_dmadev_pmd.h>
+#include <rte_memcpy.h>
 #include <rte_pci.h>
 
 #include "odm.h"

@@ -87,6 +88,235 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, uint32_t length,
+		uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_sz_available, iring_head;
+	const int num_words = ODM_IRING_ENTRY_SIZE_MIN;
+	struct odm_dev *odm = dev_private;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	uint64_t h;
+
+	const union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+		.s.nfst = 1,
+		.s.nlst = 1,
+	};
+
+	vq = &odm->vq[vchan];
+
+	h = length;
+	h |= ((uint64_t)length << 32);
+
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_sz_available = vq->iring_sz_available;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+
+		iring_head_ptr[iring_head] = hdr.u;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = h;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = src;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = dst;
+		iring_head = (iring_head + 1) % max_iring_words;
+	} else {
+		iring_head_ptr[iring_head++] = hdr.u;
+		iring_head_ptr[iring_head++] = h;
+		iring_head_ptr[iring_head++] = src;
+		iring_head_ptr[iring_head++] = dst;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* No extra space to save. Skip entry in extra space ring. */
+	vq->ins_ring_head = (vq->ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
+static inline void
+odm_dmadev_fill_sg(uint64_t *cmd, const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
+		   uint16_t nb_src, uint16_t nb_dst, union odm_instr_hdr_s *hdr)
+{
+	int i = 0, j = 0;
+	uint64_t h = 0;
+
+	cmd[j++] = hdr->u;
+	/* When nb_src is even */
+	if (!(nb_src & 0x1)) {
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Fill the iring with dst pointers */
+		for (i = 1; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is odd */
+		if (nb_dst & 0x1) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	} else {
+		/* When nb_src is odd */
+
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Handle the last src pointer */
+		h = ((uint64_t)dst[0].length << 32) | src[nb_src - 1].length;
+		cmd[j++] = h;
+		cmd[j++] = src[nb_src - 1].addr;
+		cmd[j++] = dst[0].addr;
+
+		/* Fill the iring with dst pointers */
+		for (i = 2; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is even */
+		if (!(nb_dst & 0x1)) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	}
+}
+
+static int
+odm_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src,
+		   const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_head, ins_ring_head;
+	uint64_t cmd[ODM_IRING_ENTRY_SIZE_MAX];
+	struct odm_dev *odm = dev_private;
+	uint32_t s_sz = 0, d_sz = 0;
+	uint16_t iring_sz_available;
+	uint64_t *iring_head_ptr;
+	int i, nb, num_words;
+	struct odm_queue *vq;
+	union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+	};
+
+	vq = &odm->vq[vchan];
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+	iring_sz_available = vq->iring_sz_available;
+	ins_ring_head = vq->ins_ring_head;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+
+	if (unlikely(nb_src > 4 || nb_dst > 4))
+		return -EINVAL;
+
+	for (i = 0; i < nb_src; i++)
+		s_sz += src[i].length;
+
+	for (i = 0; i < nb_dst; i++)
+		d_sz += dst[i].length;
+
+	if (s_sz != d_sz)
+		return -EINVAL;
+
+	nb = nb_src + nb_dst;
+	hdr.s.nfst = nb_src;
+	hdr.s.nlst = nb_dst;
+	num_words = 1 + 3 * (nb / 2 + (nb & 0x1));
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+		int words_avail = max_iring_words - iring_head;
+
+		odm_dmadev_fill_sg(cmd, src, dst, nb_src, nb_dst, &hdr);
+		rte_memcpy((void *)&iring_head_ptr[iring_head], (void *)cmd, words_avail * 8);
+		iring_head = num_words - words_avail;
+		rte_memcpy((void *)iring_head_ptr, (void *)&cmd[words_avail], iring_head * 8);
+	} else {
+		odm_dmadev_fill_sg(&iring_head_ptr[iring_head], src, dst, nb_src, nb_dst, &hdr);
+		iring_head += num_words;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* Save extra space used for the instruction. */
+	vq->extra_ins_sz[ins_ring_head] = num_words - 4;
+
+	vq->ins_ring_head = (ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
 static int
 odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
 	      uint32_t size)

@@ -184,6 +414,9 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	dmadev->fp_obj->dev_private = odm;
 	dmadev->dev_ops = &odm_dmadev_ops;
 
+	dmadev->fp_obj->copy = odm_dmadev_copy;
+	dmadev->fp_obj->copy_sg = odm_dmadev_copy_sg;
+
 	odm->pci_dev = pci_dev;
 
 	rc = odm_dev_init(odm);

index e1373e0c7f..1d60d2d11a 100644
@@ -75,6 +75,10 @@ extern int odm_logtype;
 	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_debug(...)                                                                             \
+	rte_log(RTE_LOG_DEBUG, odm_logtype,                                                        \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
 
 #define ODM_MEMZONE_FLAGS                                                                          \
 	(RTE_MEMZONE_1GB | RTE_MEMZONE_16MB | RTE_MEMZONE_16GB | RTE_MEMZONE_256MB |               \

258 lines not on this page · raw message

new file mode 100644
index 0000000000..aeeb6f9e9a
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#ifndef _ODM_H_
+#define _ODM_H_
+
+#include <rte_log.h>
+
+extern int odm_logtype;
+
+#define odm_err(...)                                                                               \
+	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_info(...)                                                                              \
+	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+
+struct __rte_cache_aligned odm_dev {
+	struct rte_pci_device *pci_dev;
+	uint8_t *rbase;
+	uint16_t vfid;
+	uint8_t max_qs;
+	uint8_t num_qs;
+};
+
+#endif /* _ODM_H_ */

74 lines not on this page · raw message

index aeeb6f9e9a..7564ffbed4 100644
@@ -9,6 +9,47 @@
 
 extern int odm_logtype;
 
+/* ODM VF register offsets from VF_BAR0 */
+#define ODM_VDMA_EN(x)		(0x00 | (x << 3))
+#define ODM_VDMA_REQQ_CTL(x)	(0x80 | (x << 3))
+#define ODM_VDMA_DBELL(x)	(0x100 | (x << 3))
+#define ODM_VDMA_RING_CFG(x)	(0x180 | (x << 3))
+#define ODM_VDMA_IRING_BADDR(x) (0x200 | (x << 3))
+#define ODM_VDMA_CRING_BADDR(x) (0x280 | (x << 3))
+#define ODM_VDMA_COUNTS(x)	(0x300 | (x << 3))
+#define ODM_VDMA_IRING_NADDR(x) (0x380 | (x << 3))
+#define ODM_VDMA_CRING_NADDR(x) (0x400 | (x << 3))
+#define ODM_VDMA_IRING_DBG(x)	(0x480 | (x << 3))
+#define ODM_VDMA_CNT(x)		(0x580 | (x << 3))
+#define ODM_VF_INT		(0x1000)
+#define ODM_VF_INT_W1S		(0x1008)
+#define ODM_VF_INT_ENA_W1C	(0x1010)
+#define ODM_VF_INT_ENA_W1S	(0x1018)
+#define ODM_MBOX_VF_PF_DATA(i)	(0x2000 | (i << 3))
+
+#define ODM_MBOX_RETRY_CNT	(0xfffffff)
+#define ODM_MBOX_ERR_CODE_MAX	(0xf)
+#define ODM_IRING_IDLE_WAIT_CNT (0xfffffff)
+
+/**
+ * Enumeration odm_hdr_xtype_e
+ *
+ * ODM Transfer Type Enumeration
+ * Enumerates the pointer type in ODM_DMA_INSTR_HDR_S[XTYPE]
+ */
+#define ODM_XTYPE_INTERNAL 2
+#define ODM_XTYPE_FILL0	   4
+#define ODM_XTYPE_FILL1	   5
+
+/**
+ *  ODM Header completion type enumeration
+ *  Enumerates the completion type in ODM_DMA_INSTR_HDR_S[CT]
+ */
+#define ODM_HDR_CT_CW_CA 0x0
+#define ODM_HDR_CT_CW_NC 0x1
+
+#define ODM_MAX_QUEUES_PER_DEV 16
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -18,6 +59,81 @@ extern int odm_logtype;
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
 
+/**
+ * Structure odm_instr_hdr_s for ODM
+ *
+ * ODM DMA Instruction Header Format
+ */
+union odm_instr_hdr_s {
+	uint64_t u;
+	struct odm_instr_hdr {
+		uint64_t nfst : 3;
+		uint64_t reserved_3 : 1;
+		uint64_t nlst : 3;
+		uint64_t reserved_7_9 : 3;
+		uint64_t ct : 2;
+		uint64_t stse : 1;
+		uint64_t reserved_13_28 : 16;
+		uint64_t sts : 1;
+		uint64_t reserved_30_49 : 20;
+		uint64_t xtype : 3;
+		uint64_t reserved_53_63 : 11;
+	} s;
+};
+
+/**
+ *  ODM Completion Entry Structure
+ *
+ */
+union odm_cmpl_ent_s {
+	uint32_t u;
+	struct odm_cmpl_ent {
+		uint32_t cmp_code : 8;
+		uint32_t rsvd : 23;
+		uint32_t valid : 1;
+	} s;
+};
+
+/**
+ * ODM DMA Ring Configuration Register
+ */
+union odm_vdma_ring_cfg_s {
+	uint64_t u;
+	struct {
+		uint64_t isize : 8;
+		uint64_t rsvd_8_15 : 8;
+		uint64_t csize : 8;
+		uint64_t rsvd_24_63 : 40;
+	} s;
+};
+
+/**
+ * ODM DMA Instruction Ring DBG
+ */
+union odm_vdma_iring_dbg_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell_cnt : 32;
+		uint64_t offset : 16;
+		uint64_t rsvd_48_62 : 15;
+		uint64_t iwbusy : 1;
+	} s;
+};
+
+/**
+ * ODM DMA Counts
+ */
+union odm_vdma_counts_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell : 32;
+		uint64_t buf_used_cnt : 9;
+		uint64_t rsvd_41_43 : 3;
+		uint64_t rsvd_buf_used_cnt : 3;
+		uint64_t rsvd_47_63 : 17;
+	} s;
+};
+
 struct __rte_cache_aligned odm_dev {
 	struct rte_pci_device *pci_dev;
 	uint8_t *rbase;

49 lines not on this page · raw message

97 lines not on this page · raw message

index 7564ffbed4..9fd3e30ad8 100644
@@ -5,6 +5,10 @@
 #ifndef _ODM_H_
 #define _ODM_H_
 
+#include <stdint.h>
+
+#include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_log.h>
 
 extern int odm_logtype;

@@ -50,6 +54,9 @@ extern int odm_logtype;
 
 #define ODM_MAX_QUEUES_PER_DEV 16
 
+#define odm_read64(addr)       rte_read64_relaxed((volatile void *)(addr))
+#define odm_write64(val, addr) rte_write64_relaxed((val), (volatile void *)(addr))
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -142,4 +149,7 @@ struct __rte_cache_aligned odm_dev {
 	uint8_t num_qs;
 };
 
+int odm_dev_init(struct odm_dev *odm);
+int odm_dev_fini(struct odm_dev *odm);
+
 #endif /* _ODM_H_ */

index cc3342cf7b..bef335c10c 100644
@@ -23,6 +23,7 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	char name[RTE_DEV_NAME_MAX_LEN];
 	struct odm_dev *odm = NULL;
 	struct rte_dma_dev *dmadev;
+	int rc;
 
 	if (!pci_dev->mem_resource[0].addr)
 		return -ENODEV;

@@ -37,8 +38,20 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	}
 
 	odm_info("DMA device %s probed", name);
+	odm = dmadev->data->dev_private;
+
+	odm->pci_dev = pci_dev;
+
+	rc = odm_dev_init(odm);
+	if (rc < 0)
+		goto dma_pmd_release;
 
 	return 0;
+
+dma_pmd_release:
+	rte_dma_pmd_release(name);
+
+	return rc;
 }
 
 static int

index c0963da451..6094ace9fd 100644
@@ -7,6 +7,7 @@
 #include <bus_pci_driver.h>
 
 #include <rte_io.h>
+#include <rte_malloc.h>
 
 #include "odm.h"
 #include "odm_priv.h"

@@ -14,8 +15,15 @@
 static void
 odm_vchan_resc_free(struct odm_dev *odm, int qno)
 {
-	RTE_SET_USED(odm);
-	RTE_SET_USED(qno);
+	struct odm_queue *vq = &odm->vq[qno];
+
+	rte_memzone_free(vq->iring_mz);
+	rte_memzone_free(vq->cring_mz);
+	rte_free(vq->extra_ins_sz);
+
+	vq->iring_mz = NULL;
+	vq->cring_mz = NULL;
+	vq->extra_ins_sz = NULL;
 }
 
 static int

@@ -53,6 +61,138 @@ send_mbox_to_pf(struct odm_dev *odm, union odm_mbox_msg *msg, union odm_mbox_msg
 	return 0;
 }
 
+static int
+odm_queue_ring_config(struct odm_dev *odm, int vchan, int isize, int csize)
+{
+	union odm_vdma_ring_cfg_s ring_cfg = {0};
+	struct odm_queue *vq = &odm->vq[vchan];
+
+	if (vq->iring_mz == NULL || vq->cring_mz == NULL)
+		return -EINVAL;
+
+	ring_cfg.s.isize = (isize / 1024) - 1;
+	ring_cfg.s.csize = (csize / 1024) - 1;
+
+	odm_write64(ring_cfg.u, odm->rbase + ODM_VDMA_RING_CFG(vchan));
+	odm_write64(vq->iring_mz->iova, odm->rbase + ODM_VDMA_IRING_BADDR(vchan));
+	odm_write64(vq->cring_mz->iova, odm->rbase + ODM_VDMA_CRING_BADDR(vchan));
+
+	return 0;
+}
+
+int
+odm_enable(struct odm_dev *odm)
+{
+	struct odm_queue *vq;
+	int qno, rc = 0;
+
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		vq = &odm->vq[qno];
+
+		vq->desc_idx = vq->stats.completed_offset;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+		vq->iring_head = 0;
+		vq->cring_head = 0;
+		vq->ins_ring_head = 0;
+		vq->iring_sz_available = vq->iring_max_words;
+
+		rc = odm_queue_ring_config(odm, qno, vq->iring_max_words * 8,
+					   vq->cring_max_entry * 4);
+		if (rc < 0)
+			break;
+
+		odm_write64(0x1, odm->rbase + ODM_VDMA_EN(qno));
+	}
+
+	return rc;
+}
+
+int
+odm_disable(struct odm_dev *odm)
+{
+	int qno, wait_cnt = ODM_IRING_IDLE_WAIT_CNT;
+	uint64_t val;
+
+	/* Disable the queue and wait for the queue to became idle */
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		odm_write64(0x0, odm->rbase + ODM_VDMA_EN(qno));
+		do {
+			val = odm_read64(odm->rbase + ODM_VDMA_IRING_BADDR(qno));
+		} while ((!(val & 1ULL << 63)) && (--wait_cnt > 0));
+	}
+
+	return 0;
+}
+
+int
+odm_vchan_setup(struct odm_dev *odm, int vchan, int nb_desc)
+{
+	struct odm_queue *vq = &odm->vq[vchan];
+	int isize, csize, max_nb_desc, rc = 0;
+	union odm_mbox_msg mbox_msg;
+	const struct rte_memzone *mz;
+	char name[32];
+
+	if (vq->iring_mz != NULL)
+		odm_vchan_resc_free(odm, vchan);
+
+	mbox_msg.u[0] = 0;
+	mbox_msg.u[1] = 0;
+
+	/* ODM PF driver expects vfid starts from index 0 */
+	mbox_msg.q.vfid = odm->vfid;
+	mbox_msg.q.cmd = ODM_QUEUE_OPEN;
+	mbox_msg.q.qidx = vchan;
+	rc = send_mbox_to_pf(odm, &mbox_msg, &mbox_msg);
+	if (rc < 0)
+		return rc;
+
+	/* Determine instruction & completion ring sizes. */
+
+	/* Create iring that can support nb_desc. Round up to a multiple of 1024. */
+	isize = RTE_ALIGN_CEIL(nb_desc * ODM_IRING_ENTRY_SIZE_MAX * 8, 1024);
+	isize = RTE_MIN(isize, ODM_IRING_MAX_SIZE);
+	snprintf(name, sizeof(name), "vq%d_iring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, isize, 0, ODM_MEMZONE_FLAGS, 1024);
+	if (mz == NULL)
+		return -ENOMEM;
+	vq->iring_mz = mz;
+	vq->iring_max_words = isize / 8;
+
+	/* Create cring that can support max instructions that can be inflight in hw. */
+	max_nb_desc = (isize / (ODM_IRING_ENTRY_SIZE_MIN * 8));
+	csize = RTE_ALIGN_CEIL(max_nb_desc * sizeof(union odm_cmpl_ent_s), 1024);
+	snprintf(name, sizeof(name), "vq%d_cring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, csize, 0, ODM_MEMZONE_FLAGS, 1024);
+	if (mz == NULL) {
+		rc = -ENOMEM;
+		goto iring_free;
+	}
+	vq->cring_mz = mz;
+	vq->cring_max_entry = csize / 4;
+
+	/* Allocate memory to track the size of each instruction. */
+	snprintf(name, sizeof(name), "vq%d_extra%d", odm->vfid, vchan);
+	vq->extra_ins_sz = rte_zmalloc(name, vq->cring_max_entry, 0);
+	if (vq->extra_ins_sz == NULL) {
+		rc = -ENOMEM;
+		goto cring_free;
+	}
+
+	vq->stats = (struct vq_stats){0};
+	return rc;
+
+cring_free:
+	rte_memzone_free(odm->vq[vchan].cring_mz);
+	vq->cring_mz = NULL;
+iring_free:
+	rte_memzone_free(odm->vq[vchan].iring_mz);
+	vq->iring_mz = NULL;
+
+	return rc;
+}
+
 int
 odm_dev_init(struct odm_dev *odm)
 {

90 lines not on this page · raw message

97 lines not on this page · raw message

index 8c705978fe..13b2588246 100644
@@ -87,14 +87,73 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
+	      uint32_t size)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+
+	if (size < sizeof(rte_stats))
+		return -EINVAL;
+	if (rte_stats == NULL)
+		return -EINVAL;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[vchan].stats;
+
+		*rte_stats = *stats;
+	} else {
+		int i;
+
+		for (i = 0; i < odm->num_qs; i++) {
+			struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[i].stats;
+
+			rte_stats->submitted += stats->submitted;
+			rte_stats->completed += stats->completed;
+			rte_stats->errors += stats->errors;
+		}
+	}
+
+	return 0;
+}
+
+static void
+odm_vq_stats_reset(struct vq_stats *vq_stats)
+{
+	vq_stats->completed_offset += vq_stats->completed;
+	vq_stats->completed = 0;
+	vq_stats->errors = 0;
+	vq_stats->submitted = 0;
+}
+
+static int
+odm_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+	struct vq_stats *vq_stats;
+	int i;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		vq_stats = &odm->vq[vchan].stats;
+		odm_vq_stats_reset(vq_stats);
+	} else {
+		for (i = 0; i < odm->num_qs; i++) {
+			vq_stats = &odm->vq[i].stats;
+			odm_vq_stats_reset(vq_stats);
+		}
+	}
+
+	return 0;
+}
+
 static const struct rte_dma_dev_ops odm_dmadev_ops = {
 	.dev_close = odm_dmadev_close,
 	.dev_configure = odm_dmadev_configure,
 	.dev_info_get = odm_dmadev_info_get,
 	.dev_start = odm_dmadev_start,
 	.dev_stop = odm_dmadev_stop,
-	.stats_get = NULL,
-	.stats_reset = NULL,
+	.stats_get = odm_stats_get,
+	.stats_reset = odm_stats_reset,
 	.vchan_setup = odm_dmadev_vchan_setup,
 };
 

index 13b2588246..b21be83a89 100644
@@ -9,6 +9,7 @@
 #include <rte_common.h>
 #include <rte_dmadev.h>
 #include <rte_dmadev_pmd.h>
+#include <rte_memcpy.h>
 #include <rte_pci.h>
 
 #include "odm.h"

@@ -87,6 +88,238 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, uint32_t length,
+		uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_sz_available, iring_head;
+	const int num_words = ODM_IRING_ENTRY_SIZE_MIN;
+	struct odm_dev *odm = dev_private;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	uint64_t h;
+
+	const union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+		.s.nfst = 1,
+		.s.nlst = 1,
+	};
+
+	vq = &odm->vq[vchan];
+
+	h = length;
+	h |= ((uint64_t)length << 32);
+
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_sz_available = vq->iring_sz_available;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+
+		iring_head_ptr[iring_head] = hdr.u;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = h;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = src;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = dst;
+		iring_head = (iring_head + 1) % max_iring_words;
+	} else {
+		iring_head_ptr[iring_head++] = hdr.u;
+		iring_head_ptr[iring_head++] = h;
+		iring_head_ptr[iring_head++] = src;
+		iring_head_ptr[iring_head++] = dst;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* No extra space to save. Skip entry in extra space ring. */
+	vq->ins_ring_head = (vq->ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
+static inline void
+odm_dmadev_fill_sg(uint64_t *cmd, const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
+		   uint16_t nb_src, uint16_t nb_dst, union odm_instr_hdr_s *hdr)
+{
+	int i = 0, j = 0;
+	uint64_t h = 0;
+
+	cmd[j++] = hdr->u;
+	/* When nb_src is even */
+	if (!(nb_src & 0x1)) {
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Fill the iring with dst pointers */
+		for (i = 1; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is odd */
+		if (nb_dst & 0x1) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	} else {
+		/* When nb_src is odd */
+
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Handle the last src pointer */
+		h = ((uint64_t)dst[0].length << 32) | src[nb_src - 1].length;
+		cmd[j++] = h;
+		cmd[j++] = src[nb_src - 1].addr;
+		cmd[j++] = dst[0].addr;
+
+		/* Fill the iring with dst pointers */
+		for (i = 2; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is even */
+		if (!(nb_dst & 0x1)) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	}
+}
+
+static int
+odm_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src,
+		   const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_head, ins_ring_head;
+	uint16_t iring_sz_available, i, nb, num_words;
+	uint64_t cmd[ODM_IRING_ENTRY_SIZE_MAX];
+	struct odm_dev *odm = dev_private;
+	uint32_t s_sz = 0, d_sz = 0;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+	};
+
+	vq = &odm->vq[vchan];
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+	iring_sz_available = vq->iring_sz_available;
+	ins_ring_head = vq->ins_ring_head;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+
+	if (unlikely(nb_src > 4 || nb_dst > 4))
+		return -EINVAL;
+
+	for (i = 0; i < nb_src; i++)
+		s_sz += src[i].length;
+
+	for (i = 0; i < nb_dst; i++)
+		d_sz += dst[i].length;
+
+	if (s_sz != d_sz)
+		return -EINVAL;
+
+	nb = nb_src + nb_dst;
+	hdr.s.nfst = nb_src;
+	hdr.s.nlst = nb_dst;
+	num_words = 1 + 3 * (nb / 2 + (nb & 0x1));
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+		uint16_t words_avail = max_iring_words - iring_head;
+		uint16_t words_pend = num_words - words_avail;
+
+		if (unlikely(words_avail + words_pend > ODM_IRING_ENTRY_SIZE_MAX))
+			return -ENOSPC;
+
+		odm_dmadev_fill_sg(cmd, src, dst, nb_src, nb_dst, &hdr);
+		rte_memcpy((void *)&iring_head_ptr[iring_head], (void *)cmd, words_avail * 8);
+		rte_memcpy((void *)iring_head_ptr, (void *)&cmd[words_avail], words_pend * 8);
+		iring_head = words_pend;
+	} else {
+		odm_dmadev_fill_sg(&iring_head_ptr[iring_head], src, dst, nb_src, nb_dst, &hdr);
+		iring_head += num_words;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* Save extra space used for the instruction. */
+	vq->extra_ins_sz[ins_ring_head] = num_words - 4;
+
+	vq->ins_ring_head = (ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
 static int
 odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
 	      uint32_t size)

@@ -184,6 +417,9 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	dmadev->fp_obj->dev_private = odm;
 	dmadev->dev_ops = &odm_dmadev_ops;
 
+	dmadev->fp_obj->copy = odm_dmadev_copy;
+	dmadev->fp_obj->copy_sg = odm_dmadev_copy_sg;
+
 	odm->pci_dev = pci_dev;
 
 	rc = odm_dev_init(odm);

index e1373e0c7f..1d60d2d11a 100644
@@ -75,6 +75,10 @@ extern int odm_logtype;
 	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_debug(...)                                                                             \
+	rte_log(RTE_LOG_DEBUG, odm_logtype,                                                        \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
 
 #define ODM_MEMZONE_FLAGS                                                                          \
 	(RTE_MEMZONE_1GB | RTE_MEMZONE_16MB | RTE_MEMZONE_16GB | RTE_MEMZONE_256MB |               \

258 lines not on this page · raw message

new file mode 100644
index 0000000000..aeeb6f9e9a
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#ifndef _ODM_H_
+#define _ODM_H_
+
+#include <rte_log.h>
+
+extern int odm_logtype;
+
+#define odm_err(...)                                                                               \
+	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_info(...)                                                                              \
+	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+
+struct __rte_cache_aligned odm_dev {
+	struct rte_pci_device *pci_dev;
+	uint8_t *rbase;
+	uint16_t vfid;
+	uint8_t max_qs;
+	uint8_t num_qs;
+};
+
+#endif /* _ODM_H_ */

74 lines not on this page · raw message

index aeeb6f9e9a..7564ffbed4 100644
@@ -9,6 +9,47 @@
 
 extern int odm_logtype;
 
+/* ODM VF register offsets from VF_BAR0 */
+#define ODM_VDMA_EN(x)		(0x00 | (x << 3))
+#define ODM_VDMA_REQQ_CTL(x)	(0x80 | (x << 3))
+#define ODM_VDMA_DBELL(x)	(0x100 | (x << 3))
+#define ODM_VDMA_RING_CFG(x)	(0x180 | (x << 3))
+#define ODM_VDMA_IRING_BADDR(x) (0x200 | (x << 3))
+#define ODM_VDMA_CRING_BADDR(x) (0x280 | (x << 3))
+#define ODM_VDMA_COUNTS(x)	(0x300 | (x << 3))
+#define ODM_VDMA_IRING_NADDR(x) (0x380 | (x << 3))
+#define ODM_VDMA_CRING_NADDR(x) (0x400 | (x << 3))
+#define ODM_VDMA_IRING_DBG(x)	(0x480 | (x << 3))
+#define ODM_VDMA_CNT(x)		(0x580 | (x << 3))
+#define ODM_VF_INT		(0x1000)
+#define ODM_VF_INT_W1S		(0x1008)
+#define ODM_VF_INT_ENA_W1C	(0x1010)
+#define ODM_VF_INT_ENA_W1S	(0x1018)
+#define ODM_MBOX_VF_PF_DATA(i)	(0x2000 | (i << 3))
+
+#define ODM_MBOX_RETRY_CNT	(0xfffffff)
+#define ODM_MBOX_ERR_CODE_MAX	(0xf)
+#define ODM_IRING_IDLE_WAIT_CNT (0xfffffff)
+
+/**
+ * Enumeration odm_hdr_xtype_e
+ *
+ * ODM Transfer Type Enumeration
+ * Enumerates the pointer type in ODM_DMA_INSTR_HDR_S[XTYPE]
+ */
+#define ODM_XTYPE_INTERNAL 2
+#define ODM_XTYPE_FILL0	   4
+#define ODM_XTYPE_FILL1	   5
+
+/**
+ *  ODM Header completion type enumeration
+ *  Enumerates the completion type in ODM_DMA_INSTR_HDR_S[CT]
+ */
+#define ODM_HDR_CT_CW_CA 0x0
+#define ODM_HDR_CT_CW_NC 0x1
+
+#define ODM_MAX_QUEUES_PER_DEV 16
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -18,6 +59,81 @@ extern int odm_logtype;
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
 
+/**
+ * Structure odm_instr_hdr_s for ODM
+ *
+ * ODM DMA Instruction Header Format
+ */
+union odm_instr_hdr_s {
+	uint64_t u;
+	struct odm_instr_hdr {
+		uint64_t nfst : 3;
+		uint64_t reserved_3 : 1;
+		uint64_t nlst : 3;
+		uint64_t reserved_7_9 : 3;
+		uint64_t ct : 2;
+		uint64_t stse : 1;
+		uint64_t reserved_13_28 : 16;
+		uint64_t sts : 1;
+		uint64_t reserved_30_49 : 20;
+		uint64_t xtype : 3;
+		uint64_t reserved_53_63 : 11;
+	} s;
+};
+
+/**
+ *  ODM Completion Entry Structure
+ *
+ */
+union odm_cmpl_ent_s {
+	uint32_t u;
+	struct odm_cmpl_ent {
+		uint32_t cmp_code : 8;
+		uint32_t rsvd : 23;
+		uint32_t valid : 1;
+	} s;
+};
+
+/**
+ * ODM DMA Ring Configuration Register
+ */
+union odm_vdma_ring_cfg_s {
+	uint64_t u;
+	struct {
+		uint64_t isize : 8;
+		uint64_t rsvd_8_15 : 8;
+		uint64_t csize : 8;
+		uint64_t rsvd_24_63 : 40;
+	} s;
+};
+
+/**
+ * ODM DMA Instruction Ring DBG
+ */
+union odm_vdma_iring_dbg_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell_cnt : 32;
+		uint64_t offset : 16;
+		uint64_t rsvd_48_62 : 15;
+		uint64_t iwbusy : 1;
+	} s;
+};
+
+/**
+ * ODM DMA Counts
+ */
+union odm_vdma_counts_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell : 32;
+		uint64_t buf_used_cnt : 9;
+		uint64_t rsvd_41_43 : 3;
+		uint64_t rsvd_buf_used_cnt : 3;
+		uint64_t rsvd_47_63 : 17;
+	} s;
+};
+
 struct __rte_cache_aligned odm_dev {
 	struct rte_pci_device *pci_dev;
 	uint8_t *rbase;

49 lines not on this page · raw message

97 lines not on this page · raw message

index 7564ffbed4..9fd3e30ad8 100644
@@ -5,6 +5,10 @@
 #ifndef _ODM_H_
 #define _ODM_H_
 
+#include <stdint.h>
+
+#include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_log.h>
 
 extern int odm_logtype;

@@ -50,6 +54,9 @@ extern int odm_logtype;
 
 #define ODM_MAX_QUEUES_PER_DEV 16
 
+#define odm_read64(addr)       rte_read64_relaxed((volatile void *)(addr))
+#define odm_write64(val, addr) rte_write64_relaxed((val), (volatile void *)(addr))
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -142,4 +149,7 @@ struct __rte_cache_aligned odm_dev {
 	uint8_t num_qs;
 };
 
+int odm_dev_init(struct odm_dev *odm);
+int odm_dev_fini(struct odm_dev *odm);
+
 #endif /* _ODM_H_ */

index cc3342cf7b..bef335c10c 100644
@@ -23,6 +23,7 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	char name[RTE_DEV_NAME_MAX_LEN];
 	struct odm_dev *odm = NULL;
 	struct rte_dma_dev *dmadev;
+	int rc;
 
 	if (!pci_dev->mem_resource[0].addr)
 		return -ENODEV;

@@ -37,8 +38,20 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	}
 
 	odm_info("DMA device %s probed", name);
+	odm = dmadev->data->dev_private;
+
+	odm->pci_dev = pci_dev;
+
+	rc = odm_dev_init(odm);
+	if (rc < 0)
+		goto dma_pmd_release;
 
 	return 0;
+
+dma_pmd_release:
+	rte_dma_pmd_release(name);
+
+	return rc;
 }
 
 static int

index c0963da451..6094ace9fd 100644
@@ -7,6 +7,7 @@
 #include <bus_pci_driver.h>
 
 #include <rte_io.h>
+#include <rte_malloc.h>
 
 #include "odm.h"
 #include "odm_priv.h"

@@ -14,8 +15,15 @@
 static void
 odm_vchan_resc_free(struct odm_dev *odm, int qno)
 {
-	RTE_SET_USED(odm);
-	RTE_SET_USED(qno);
+	struct odm_queue *vq = &odm->vq[qno];
+
+	rte_memzone_free(vq->iring_mz);
+	rte_memzone_free(vq->cring_mz);
+	rte_free(vq->extra_ins_sz);
+
+	vq->iring_mz = NULL;
+	vq->cring_mz = NULL;
+	vq->extra_ins_sz = NULL;
 }
 
 static int

@@ -53,6 +61,138 @@ send_mbox_to_pf(struct odm_dev *odm, union odm_mbox_msg *msg, union odm_mbox_msg
 	return 0;
 }
 
+static int
+odm_queue_ring_config(struct odm_dev *odm, int vchan, int isize, int csize)
+{
+	union odm_vdma_ring_cfg_s ring_cfg = {0};
+	struct odm_queue *vq = &odm->vq[vchan];
+
+	if (vq->iring_mz == NULL || vq->cring_mz == NULL)
+		return -EINVAL;
+
+	ring_cfg.s.isize = (isize / 1024) - 1;
+	ring_cfg.s.csize = (csize / 1024) - 1;
+
+	odm_write64(ring_cfg.u, odm->rbase + ODM_VDMA_RING_CFG(vchan));
+	odm_write64(vq->iring_mz->iova, odm->rbase + ODM_VDMA_IRING_BADDR(vchan));
+	odm_write64(vq->cring_mz->iova, odm->rbase + ODM_VDMA_CRING_BADDR(vchan));
+
+	return 0;
+}
+
+int
+odm_enable(struct odm_dev *odm)
+{
+	struct odm_queue *vq;
+	int qno, rc = 0;
+
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		vq = &odm->vq[qno];
+
+		vq->desc_idx = vq->stats.completed_offset;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+		vq->iring_head = 0;
+		vq->cring_head = 0;
+		vq->ins_ring_head = 0;
+		vq->iring_sz_available = vq->iring_max_words;
+
+		rc = odm_queue_ring_config(odm, qno, vq->iring_max_words * 8,
+					   vq->cring_max_entry * 4);
+		if (rc < 0)
+			break;
+
+		odm_write64(0x1, odm->rbase + ODM_VDMA_EN(qno));
+	}
+
+	return rc;
+}
+
+int
+odm_disable(struct odm_dev *odm)
+{
+	int qno, wait_cnt = ODM_IRING_IDLE_WAIT_CNT;
+	uint64_t val;
+
+	/* Disable the queue and wait for the queue to became idle */
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		odm_write64(0x0, odm->rbase + ODM_VDMA_EN(qno));
+		do {
+			val = odm_read64(odm->rbase + ODM_VDMA_IRING_BADDR(qno));
+		} while ((!(val & 1ULL << 63)) && (--wait_cnt > 0));
+	}
+
+	return 0;
+}
+
+int
+odm_vchan_setup(struct odm_dev *odm, int vchan, int nb_desc)
+{
+	struct odm_queue *vq = &odm->vq[vchan];
+	int isize, csize, max_nb_desc, rc = 0;
+	union odm_mbox_msg mbox_msg;
+	const struct rte_memzone *mz;
+	char name[32];
+
+	if (vq->iring_mz != NULL)
+		odm_vchan_resc_free(odm, vchan);
+
+	mbox_msg.u[0] = 0;
+	mbox_msg.u[1] = 0;
+
+	/* ODM PF driver expects vfid starts from index 0 */
+	mbox_msg.q.vfid = odm->vfid;
+	mbox_msg.q.cmd = ODM_QUEUE_OPEN;
+	mbox_msg.q.qidx = vchan;
+	rc = send_mbox_to_pf(odm, &mbox_msg, &mbox_msg);
+	if (rc < 0)
+		return rc;
+
+	/* Determine instruction & completion ring sizes. */
+
+	/* Create iring that can support nb_desc. Round up to a multiple of 1024. */
+	isize = RTE_ALIGN_CEIL(nb_desc * ODM_IRING_ENTRY_SIZE_MAX * 8, 1024);
+	isize = RTE_MIN(isize, ODM_IRING_MAX_SIZE);
+	snprintf(name, sizeof(name), "vq%d_iring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, isize, 0, ODM_MEMZONE_FLAGS, 1024);
+	if (mz == NULL)
+		return -ENOMEM;
+	vq->iring_mz = mz;
+	vq->iring_max_words = isize / 8;
+
+	/* Create cring that can support max instructions that can be inflight in hw. */
+	max_nb_desc = (isize / (ODM_IRING_ENTRY_SIZE_MIN * 8));
+	csize = RTE_ALIGN_CEIL(max_nb_desc * sizeof(union odm_cmpl_ent_s), 1024);
+	snprintf(name, sizeof(name), "vq%d_cring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, csize, 0, ODM_MEMZONE_FLAGS, 1024);
+	if (mz == NULL) {
+		rc = -ENOMEM;
+		goto iring_free;
+	}
+	vq->cring_mz = mz;
+	vq->cring_max_entry = csize / 4;
+
+	/* Allocate memory to track the size of each instruction. */
+	snprintf(name, sizeof(name), "vq%d_extra%d", odm->vfid, vchan);
+	vq->extra_ins_sz = rte_zmalloc(name, vq->cring_max_entry, 0);
+	if (vq->extra_ins_sz == NULL) {
+		rc = -ENOMEM;
+		goto cring_free;
+	}
+
+	vq->stats = (struct vq_stats){0};
+	return rc;
+
+cring_free:
+	rte_memzone_free(odm->vq[vchan].cring_mz);
+	vq->cring_mz = NULL;
+iring_free:
+	rte_memzone_free(odm->vq[vchan].iring_mz);
+	vq->iring_mz = NULL;
+
+	return rc;
+}
+
 int
 odm_dev_init(struct odm_dev *odm)
 {

90 lines not on this page · raw message

97 lines not on this page · raw message

index 8c705978fe..13b2588246 100644
@@ -87,14 +87,73 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
+	      uint32_t size)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+
+	if (size < sizeof(rte_stats))
+		return -EINVAL;
+	if (rte_stats == NULL)
+		return -EINVAL;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[vchan].stats;
+
+		*rte_stats = *stats;
+	} else {
+		int i;
+
+		for (i = 0; i < odm->num_qs; i++) {
+			struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[i].stats;
+
+			rte_stats->submitted += stats->submitted;
+			rte_stats->completed += stats->completed;
+			rte_stats->errors += stats->errors;
+		}
+	}
+
+	return 0;
+}
+
+static void
+odm_vq_stats_reset(struct vq_stats *vq_stats)
+{
+	vq_stats->completed_offset += vq_stats->completed;
+	vq_stats->completed = 0;
+	vq_stats->errors = 0;
+	vq_stats->submitted = 0;
+}
+
+static int
+odm_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+	struct vq_stats *vq_stats;
+	int i;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		vq_stats = &odm->vq[vchan].stats;
+		odm_vq_stats_reset(vq_stats);
+	} else {
+		for (i = 0; i < odm->num_qs; i++) {
+			vq_stats = &odm->vq[i].stats;
+			odm_vq_stats_reset(vq_stats);
+		}
+	}
+
+	return 0;
+}
+
 static const struct rte_dma_dev_ops odm_dmadev_ops = {
 	.dev_close = odm_dmadev_close,
 	.dev_configure = odm_dmadev_configure,
 	.dev_info_get = odm_dmadev_info_get,
 	.dev_start = odm_dmadev_start,
 	.dev_stop = odm_dmadev_stop,
-	.stats_get = NULL,
-	.stats_reset = NULL,
+	.stats_get = odm_stats_get,
+	.stats_reset = odm_stats_reset,
 	.vchan_setup = odm_dmadev_vchan_setup,
 };
 

index 13b2588246..b21be83a89 100644
@@ -9,6 +9,7 @@
 #include <rte_common.h>
 #include <rte_dmadev.h>
 #include <rte_dmadev_pmd.h>
+#include <rte_memcpy.h>
 #include <rte_pci.h>
 
 #include "odm.h"

@@ -87,6 +88,238 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, uint32_t length,
+		uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_sz_available, iring_head;
+	const int num_words = ODM_IRING_ENTRY_SIZE_MIN;
+	struct odm_dev *odm = dev_private;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	uint64_t h;
+
+	const union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+		.s.nfst = 1,
+		.s.nlst = 1,
+	};
+
+	vq = &odm->vq[vchan];
+
+	h = length;
+	h |= ((uint64_t)length << 32);
+
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_sz_available = vq->iring_sz_available;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+
+		iring_head_ptr[iring_head] = hdr.u;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = h;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = src;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = dst;
+		iring_head = (iring_head + 1) % max_iring_words;
+	} else {
+		iring_head_ptr[iring_head++] = hdr.u;
+		iring_head_ptr[iring_head++] = h;
+		iring_head_ptr[iring_head++] = src;
+		iring_head_ptr[iring_head++] = dst;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* No extra space to save. Skip entry in extra space ring. */
+	vq->ins_ring_head = (vq->ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
+static inline void
+odm_dmadev_fill_sg(uint64_t *cmd, const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
+		   uint16_t nb_src, uint16_t nb_dst, union odm_instr_hdr_s *hdr)
+{
+	int i = 0, j = 0;
+	uint64_t h = 0;
+
+	cmd[j++] = hdr->u;
+	/* When nb_src is even */
+	if (!(nb_src & 0x1)) {
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Fill the iring with dst pointers */
+		for (i = 1; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is odd */
+		if (nb_dst & 0x1) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	} else {
+		/* When nb_src is odd */
+
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Handle the last src pointer */
+		h = ((uint64_t)dst[0].length << 32) | src[nb_src - 1].length;
+		cmd[j++] = h;
+		cmd[j++] = src[nb_src - 1].addr;
+		cmd[j++] = dst[0].addr;
+
+		/* Fill the iring with dst pointers */
+		for (i = 2; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is even */
+		if (!(nb_dst & 0x1)) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	}
+}
+
+static int
+odm_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src,
+		   const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_head, ins_ring_head;
+	uint16_t iring_sz_available, i, nb, num_words;
+	uint64_t cmd[ODM_IRING_ENTRY_SIZE_MAX];
+	struct odm_dev *odm = dev_private;
+	uint32_t s_sz = 0, d_sz = 0;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+	};
+
+	vq = &odm->vq[vchan];
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+	iring_sz_available = vq->iring_sz_available;
+	ins_ring_head = vq->ins_ring_head;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+
+	if (unlikely(nb_src > 4 || nb_dst > 4))
+		return -EINVAL;
+
+	for (i = 0; i < nb_src; i++)
+		s_sz += src[i].length;
+
+	for (i = 0; i < nb_dst; i++)
+		d_sz += dst[i].length;
+
+	if (s_sz != d_sz)
+		return -EINVAL;
+
+	nb = nb_src + nb_dst;
+	hdr.s.nfst = nb_src;
+	hdr.s.nlst = nb_dst;
+	num_words = 1 + 3 * (nb / 2 + (nb & 0x1));
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+		uint16_t words_avail = max_iring_words - iring_head;
+		uint16_t words_pend = num_words - words_avail;
+
+		if (unlikely(words_avail + words_pend > ODM_IRING_ENTRY_SIZE_MAX))
+			return -ENOSPC;
+
+		odm_dmadev_fill_sg(cmd, src, dst, nb_src, nb_dst, &hdr);
+		rte_memcpy((void *)&iring_head_ptr[iring_head], (void *)cmd, words_avail * 8);
+		rte_memcpy((void *)iring_head_ptr, (void *)&cmd[words_avail], words_pend * 8);
+		iring_head = words_pend;
+	} else {
+		odm_dmadev_fill_sg(&iring_head_ptr[iring_head], src, dst, nb_src, nb_dst, &hdr);
+		iring_head += num_words;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* Save extra space used for the instruction. */
+	vq->extra_ins_sz[ins_ring_head] = num_words - 4;
+
+	vq->ins_ring_head = (ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
 static int
 odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
 	      uint32_t size)

@@ -184,6 +417,9 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	dmadev->fp_obj->dev_private = odm;
 	dmadev->dev_ops = &odm_dmadev_ops;
 
+	dmadev->fp_obj->copy = odm_dmadev_copy;
+	dmadev->fp_obj->copy_sg = odm_dmadev_copy_sg;
+
 	odm->pci_dev = pci_dev;
 
 	rc = odm_dev_init(odm);

index e1373e0c7f..1d60d2d11a 100644
@@ -75,6 +75,10 @@ extern int odm_logtype;
 	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_debug(...)                                                                             \
+	rte_log(RTE_LOG_DEBUG, odm_logtype,                                                        \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
 
 #define ODM_MEMZONE_FLAGS                                                                          \
 	(RTE_MEMZONE_1GB | RTE_MEMZONE_16MB | RTE_MEMZONE_16GB | RTE_MEMZONE_256MB |               \

262 lines not on this page · raw message

new file mode 100644
index 0000000000..aeeb6f9e9a
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#ifndef _ODM_H_
+#define _ODM_H_
+
+#include <rte_log.h>
+
+extern int odm_logtype;
+
+#define odm_err(...)                                                                               \
+	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+#define odm_info(...)                                                                              \
+	rte_log(RTE_LOG_INFO, odm_logtype,                                                         \
+		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
+			RTE_FMT_TAIL(__VA_ARGS__, )))
+
+struct __rte_cache_aligned odm_dev {
+	struct rte_pci_device *pci_dev;
+	uint8_t *rbase;
+	uint16_t vfid;
+	uint8_t max_qs;
+	uint8_t num_qs;
+};
+
+#endif /* _ODM_H_ */

74 lines not on this page · raw message

index aeeb6f9e9a..8cc3e0de44 100644
@@ -9,6 +9,46 @@
 
 extern int odm_logtype;
 
+/* ODM VF register offsets from VF_BAR0 */
+#define ODM_VDMA_EN(x)		(0x00 | (x << 3))
+#define ODM_VDMA_REQQ_CTL(x)	(0x80 | (x << 3))
+#define ODM_VDMA_DBELL(x)	(0x100 | (x << 3))
+#define ODM_VDMA_RING_CFG(x)	(0x180 | (x << 3))
+#define ODM_VDMA_IRING_BADDR(x) (0x200 | (x << 3))
+#define ODM_VDMA_CRING_BADDR(x) (0x280 | (x << 3))
+#define ODM_VDMA_COUNTS(x)	(0x300 | (x << 3))
+#define ODM_VDMA_IRING_NADDR(x) (0x380 | (x << 3))
+#define ODM_VDMA_CRING_NADDR(x) (0x400 | (x << 3))
+#define ODM_VDMA_IRING_DBG(x)	(0x480 | (x << 3))
+#define ODM_VDMA_CNT(x)		(0x580 | (x << 3))
+#define ODM_VF_INT		(0x1000)
+#define ODM_VF_INT_W1S		(0x1008)
+#define ODM_VF_INT_ENA_W1C	(0x1010)
+#define ODM_VF_INT_ENA_W1S	(0x1018)
+#define ODM_MBOX_VF_PF_DATA(i)	(0x2000 | (i << 3))
+#define ODM_MBOX_RETRY_CNT	(0xfffffff)
+#define ODM_MBOX_ERR_CODE_MAX	(0xf)
+#define ODM_IRING_IDLE_WAIT_CNT (0xfffffff)
+
+/*
+ * Enumeration odm_hdr_xtype_e
+ *
+ * ODM Transfer Type Enumeration
+ * Enumerates the pointer type in ODM_DMA_INSTR_HDR_S[XTYPE]
+ */
+#define ODM_XTYPE_INTERNAL 2
+#define ODM_XTYPE_FILL0	   4
+#define ODM_XTYPE_FILL1	   5
+
+/*
+ *  ODM Header completion type enumeration
+ *  Enumerates the completion type in ODM_DMA_INSTR_HDR_S[CT]
+ */
+#define ODM_HDR_CT_CW_CA 0x0
+#define ODM_HDR_CT_CW_NC 0x1
+
+#define ODM_MAX_QUEUES_PER_DEV 16
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -18,6 +58,72 @@ extern int odm_logtype;
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \
 			RTE_FMT_TAIL(__VA_ARGS__, )))
 
+/*
+ * Structure odm_instr_hdr_s for ODM
+ *
+ * ODM DMA Instruction Header Format
+ */
+union odm_instr_hdr_s {
+	uint64_t u;
+	struct odm_instr_hdr {
+		uint64_t nfst : 3;
+		uint64_t reserved_3 : 1;
+		uint64_t nlst : 3;
+		uint64_t reserved_7_9 : 3;
+		uint64_t ct : 2;
+		uint64_t stse : 1;
+		uint64_t reserved_13_28 : 16;
+		uint64_t sts : 1;
+		uint64_t reserved_30_49 : 20;
+		uint64_t xtype : 3;
+		uint64_t reserved_53_63 : 11;
+	} s;
+};
+
+/* ODM Completion Entry Structure */
+union odm_cmpl_ent_s {
+	uint32_t u;
+	struct odm_cmpl_ent {
+		uint32_t cmp_code : 8;
+		uint32_t rsvd : 23;
+		uint32_t valid : 1;
+	} s;
+};
+
+/* ODM DMA Ring Configuration Register */
+union odm_vdma_ring_cfg_s {
+	uint64_t u;
+	struct {
+		uint64_t isize : 8;
+		uint64_t rsvd_8_15 : 8;
+		uint64_t csize : 8;
+		uint64_t rsvd_24_63 : 40;
+	} s;
+};
+
+/* ODM DMA Instruction Ring DBG */
+union odm_vdma_iring_dbg_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell_cnt : 32;
+		uint64_t offset : 16;
+		uint64_t rsvd_48_62 : 15;
+		uint64_t iwbusy : 1;
+	} s;
+};
+
+/* ODM DMA Counts */
+union odm_vdma_counts_s {
+	uint64_t u;
+	struct {
+		uint64_t dbell : 32;
+		uint64_t buf_used_cnt : 9;
+		uint64_t rsvd_41_43 : 3;
+		uint64_t rsvd_buf_used_cnt : 3;
+		uint64_t rsvd_47_63 : 17;
+	} s;
+};
+
 struct __rte_cache_aligned odm_dev {
 	struct rte_pci_device *pci_dev;
 	uint8_t *rbase;

49 lines not on this page · raw message

97 lines not on this page · raw message

index 8cc3e0de44..0bf0c6345b 100644
@@ -5,6 +5,10 @@
 #ifndef _ODM_H_
 #define _ODM_H_
 
+#include <stdint.h>
+
+#include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_log.h>
 
 extern int odm_logtype;

@@ -49,6 +53,9 @@ extern int odm_logtype;
 
 #define ODM_MAX_QUEUES_PER_DEV 16
 
+#define odm_read64(addr)       rte_read64_relaxed((volatile void *)(addr))
+#define odm_write64(val, addr) rte_write64_relaxed((val), (volatile void *)(addr))
+
 #define odm_err(...)                                                                               \
 	rte_log(RTE_LOG_ERR, odm_logtype,                                                          \
 		RTE_FMT("%s(): %u" RTE_FMT_HEAD(__VA_ARGS__, ), __func__, __LINE__,                \

@@ -132,4 +139,7 @@ struct __rte_cache_aligned odm_dev {
 	uint8_t num_qs;
 };
 
+int odm_dev_init(struct odm_dev *odm);
+int odm_dev_fini(struct odm_dev *odm);
+
 #endif /* _ODM_H_ */

index cc3342cf7b..bef335c10c 100644
@@ -23,6 +23,7 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	char name[RTE_DEV_NAME_MAX_LEN];
 	struct odm_dev *odm = NULL;
 	struct rte_dma_dev *dmadev;
+	int rc;
 
 	if (!pci_dev->mem_resource[0].addr)
 		return -ENODEV;

@@ -37,8 +38,20 @@ odm_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused, struct rte_pci_dev
 	}
 
 	odm_info("DMA device %s probed", name);
+	odm = dmadev->data->dev_private;
+
+	odm->pci_dev = pci_dev;
+
+	rc = odm_dev_init(odm);
+	if (rc < 0)
+		goto dma_pmd_release;
 
 	return 0;
+
+dma_pmd_release:
+	rte_dma_pmd_release(name);
+
+	return rc;
 }
 
 static int

index c0963da451..270808f4df 100644
@@ -7,6 +7,7 @@
 #include <bus_pci_driver.h>
 
 #include <rte_io.h>
+#include <rte_malloc.h>
 
 #include "odm.h"
 #include "odm_priv.h"

@@ -14,8 +15,15 @@
 static void
 odm_vchan_resc_free(struct odm_dev *odm, int qno)
 {
-	RTE_SET_USED(odm);
-	RTE_SET_USED(qno);
+	struct odm_queue *vq = &odm->vq[qno];
+
+	rte_memzone_free(vq->iring_mz);
+	rte_memzone_free(vq->cring_mz);
+	rte_free(vq->extra_ins_sz);
+
+	vq->iring_mz = NULL;
+	vq->cring_mz = NULL;
+	vq->extra_ins_sz = NULL;
 }
 
 static int

@@ -53,6 +61,138 @@ send_mbox_to_pf(struct odm_dev *odm, union odm_mbox_msg *msg, union odm_mbox_msg
 	return 0;
 }
 
+static int
+odm_queue_ring_config(struct odm_dev *odm, int vchan, int isize, int csize)
+{
+	union odm_vdma_ring_cfg_s ring_cfg = {0};
+	struct odm_queue *vq = &odm->vq[vchan];
+
+	if (vq->iring_mz == NULL || vq->cring_mz == NULL)
+		return -EINVAL;
+
+	ring_cfg.s.isize = (isize / 1024) - 1;
+	ring_cfg.s.csize = (csize / 1024) - 1;
+
+	odm_write64(ring_cfg.u, odm->rbase + ODM_VDMA_RING_CFG(vchan));
+	odm_write64(vq->iring_mz->iova, odm->rbase + ODM_VDMA_IRING_BADDR(vchan));
+	odm_write64(vq->cring_mz->iova, odm->rbase + ODM_VDMA_CRING_BADDR(vchan));
+
+	return 0;
+}
+
+int
+odm_enable(struct odm_dev *odm)
+{
+	struct odm_queue *vq;
+	int qno, rc = 0;
+
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		vq = &odm->vq[qno];
+
+		vq->desc_idx = vq->stats.completed_offset;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+		vq->iring_head = 0;
+		vq->cring_head = 0;
+		vq->ins_ring_head = 0;
+		vq->iring_sz_available = vq->iring_max_words;
+
+		rc = odm_queue_ring_config(odm, qno, vq->iring_max_words * 8,
+					   vq->cring_max_entry * 4);
+		if (rc < 0)
+			break;
+
+		odm_write64(0x1, odm->rbase + ODM_VDMA_EN(qno));
+	}
+
+	return rc;
+}
+
+int
+odm_disable(struct odm_dev *odm)
+{
+	int qno, wait_cnt = ODM_IRING_IDLE_WAIT_CNT;
+	uint64_t val;
+
+	/* Disable the queue and wait for the queue to became idle */
+	for (qno = 0; qno < odm->num_qs; qno++) {
+		odm_write64(0x0, odm->rbase + ODM_VDMA_EN(qno));
+		do {
+			val = odm_read64(odm->rbase + ODM_VDMA_IRING_BADDR(qno));
+		} while ((!(val & 1ULL << 63)) && (--wait_cnt > 0));
+	}
+
+	return 0;
+}
+
+int
+odm_vchan_setup(struct odm_dev *odm, int vchan, int nb_desc)
+{
+	struct odm_queue *vq = &odm->vq[vchan];
+	int isize, csize, max_nb_desc, rc = 0;
+	union odm_mbox_msg mbox_msg;
+	const struct rte_memzone *mz;
+	char name[32];
+
+	if (vq->iring_mz != NULL)
+		odm_vchan_resc_free(odm, vchan);
+
+	mbox_msg.u[0] = 0;
+	mbox_msg.u[1] = 0;
+
+	/* ODM PF driver expects vfid starts from index 0 */
+	mbox_msg.q.vfid = odm->vfid;
+	mbox_msg.q.cmd = ODM_QUEUE_OPEN;
+	mbox_msg.q.qidx = vchan;
+	rc = send_mbox_to_pf(odm, &mbox_msg, &mbox_msg);
+	if (rc < 0)
+		return rc;
+
+	/* Determine instruction & completion ring sizes. */
+
+	/* Create iring that can support nb_desc. Round up to a multiple of 1024. */
+	isize = RTE_ALIGN_CEIL(nb_desc * ODM_IRING_ENTRY_SIZE_MAX * 8, 1024);
+	isize = RTE_MIN(isize, ODM_IRING_MAX_SIZE);
+	snprintf(name, sizeof(name), "vq%d_iring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, isize, SOCKET_ID_ANY, 0, 1024);
+	if (mz == NULL)
+		return -ENOMEM;
+	vq->iring_mz = mz;
+	vq->iring_max_words = isize / 8;
+
+	/* Create cring that can support max instructions that can be inflight in hw. */
+	max_nb_desc = (isize / (ODM_IRING_ENTRY_SIZE_MIN * 8));
+	csize = RTE_ALIGN_CEIL(max_nb_desc * sizeof(union odm_cmpl_ent_s), 1024);
+	snprintf(name, sizeof(name), "vq%d_cring%d", odm->vfid, vchan);
+	mz = rte_memzone_reserve_aligned(name, csize, SOCKET_ID_ANY, 0, 1024);
+	if (mz == NULL) {
+		rc = -ENOMEM;
+		goto iring_free;
+	}
+	vq->cring_mz = mz;
+	vq->cring_max_entry = csize / 4;
+
+	/* Allocate memory to track the size of each instruction. */
+	snprintf(name, sizeof(name), "vq%d_extra%d", odm->vfid, vchan);
+	vq->extra_ins_sz = rte_zmalloc(name, vq->cring_max_entry, 0);
+	if (vq->extra_ins_sz == NULL) {
+		rc = -ENOMEM;
+		goto cring_free;
+	}
+
+	vq->stats = (struct vq_stats){0};
+	return rc;
+
+cring_free:
+	rte_memzone_free(odm->vq[vchan].cring_mz);
+	vq->cring_mz = NULL;
+iring_free:
+	rte_memzone_free(odm->vq[vchan].iring_mz);
+	vq->iring_mz = NULL;
+
+	return rc;
+}
+
 int
 odm_dev_init(struct odm_dev *odm)
 {

80 lines not on this page · raw message

97 lines not on this page · raw message

index 8c705978fe..13b2588246 100644
@@ -87,14 +87,73 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
+	      uint32_t size)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+
+	if (size < sizeof(rte_stats))
+		return -EINVAL;
+	if (rte_stats == NULL)
+		return -EINVAL;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[vchan].stats;
+
+		*rte_stats = *stats;
+	} else {
+		int i;
+
+		for (i = 0; i < odm->num_qs; i++) {
+			struct rte_dma_stats *stats = (struct rte_dma_stats *)&odm->vq[i].stats;
+
+			rte_stats->submitted += stats->submitted;
+			rte_stats->completed += stats->completed;
+			rte_stats->errors += stats->errors;
+		}
+	}
+
+	return 0;
+}
+
+static void
+odm_vq_stats_reset(struct vq_stats *vq_stats)
+{
+	vq_stats->completed_offset += vq_stats->completed;
+	vq_stats->completed = 0;
+	vq_stats->errors = 0;
+	vq_stats->submitted = 0;
+}
+
+static int
+odm_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
+{
+	struct odm_dev *odm = dev->fp_obj->dev_private;
+	struct vq_stats *vq_stats;
+	int i;
+
+	if (vchan != RTE_DMA_ALL_VCHAN) {
+		vq_stats = &odm->vq[vchan].stats;
+		odm_vq_stats_reset(vq_stats);
+	} else {
+		for (i = 0; i < odm->num_qs; i++) {
+			vq_stats = &odm->vq[i].stats;
+			odm_vq_stats_reset(vq_stats);
+		}
+	}
+
+	return 0;
+}
+
 static const struct rte_dma_dev_ops odm_dmadev_ops = {
 	.dev_close = odm_dmadev_close,
 	.dev_configure = odm_dmadev_configure,
 	.dev_info_get = odm_dmadev_info_get,
 	.dev_start = odm_dmadev_start,
 	.dev_stop = odm_dmadev_stop,
-	.stats_get = NULL,
-	.stats_reset = NULL,
+	.stats_get = odm_stats_get,
+	.stats_reset = odm_stats_reset,
 	.vchan_setup = odm_dmadev_vchan_setup,
 };
 

index 13b2588246..b21be83a89 100644
@@ -9,6 +9,7 @@
 #include <rte_common.h>
 #include <rte_dmadev.h>
 #include <rte_dmadev_pmd.h>
+#include <rte_memcpy.h>
 #include <rte_pci.h>
 
 #include "odm.h"

@@ -87,6 +88,238 @@ odm_dmadev_close(struct rte_dma_dev *dev)
 	return 0;
 }
 
+static int
+odm_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src, rte_iova_t dst, uint32_t length,
+		uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_sz_available, iring_head;
+	const int num_words = ODM_IRING_ENTRY_SIZE_MIN;
+	struct odm_dev *odm = dev_private;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	uint64_t h;
+
+	const union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+		.s.nfst = 1,
+		.s.nlst = 1,
+	};
+
+	vq = &odm->vq[vchan];
+
+	h = length;
+	h |= ((uint64_t)length << 32);
+
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_sz_available = vq->iring_sz_available;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+
+		iring_head_ptr[iring_head] = hdr.u;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = h;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = src;
+		iring_head = (iring_head + 1) % max_iring_words;
+
+		iring_head_ptr[iring_head] = dst;
+		iring_head = (iring_head + 1) % max_iring_words;
+	} else {
+		iring_head_ptr[iring_head++] = hdr.u;
+		iring_head_ptr[iring_head++] = h;
+		iring_head_ptr[iring_head++] = src;
+		iring_head_ptr[iring_head++] = dst;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* No extra space to save. Skip entry in extra space ring. */
+	vq->ins_ring_head = (vq->ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
+static inline void
+odm_dmadev_fill_sg(uint64_t *cmd, const struct rte_dma_sge *src, const struct rte_dma_sge *dst,
+		   uint16_t nb_src, uint16_t nb_dst, union odm_instr_hdr_s *hdr)
+{
+	int i = 0, j = 0;
+	uint64_t h = 0;
+
+	cmd[j++] = hdr->u;
+	/* When nb_src is even */
+	if (!(nb_src & 0x1)) {
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Fill the iring with dst pointers */
+		for (i = 1; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is odd */
+		if (nb_dst & 0x1) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	} else {
+		/* When nb_src is odd */
+
+		/* Fill the iring with src pointers */
+		for (i = 1; i < nb_src; i += 2) {
+			h = ((uint64_t)src[i].length << 32) | src[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = src[i - 1].addr;
+			cmd[j++] = src[i].addr;
+		}
+
+		/* Handle the last src pointer */
+		h = ((uint64_t)dst[0].length << 32) | src[nb_src - 1].length;
+		cmd[j++] = h;
+		cmd[j++] = src[nb_src - 1].addr;
+		cmd[j++] = dst[0].addr;
+
+		/* Fill the iring with dst pointers */
+		for (i = 2; i < nb_dst; i += 2) {
+			h = ((uint64_t)dst[i].length << 32) | dst[i - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[i - 1].addr;
+			cmd[j++] = dst[i].addr;
+		}
+
+		/* Handle the last dst pointer when nb_dst is even */
+		if (!(nb_dst & 0x1)) {
+			h = dst[nb_dst - 1].length;
+			cmd[j++] = h;
+			cmd[j++] = dst[nb_dst - 1].addr;
+			cmd[j++] = 0;
+		}
+	}
+}
+
+static int
+odm_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge *src,
+		   const struct rte_dma_sge *dst, uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
+{
+	uint16_t pending_submit_len, pending_submit_cnt, iring_head, ins_ring_head;
+	uint16_t iring_sz_available, i, nb, num_words;
+	uint64_t cmd[ODM_IRING_ENTRY_SIZE_MAX];
+	struct odm_dev *odm = dev_private;
+	uint32_t s_sz = 0, d_sz = 0;
+	uint64_t *iring_head_ptr;
+	struct odm_queue *vq;
+	union odm_instr_hdr_s hdr = {
+		.s.ct = ODM_HDR_CT_CW_NC,
+		.s.xtype = ODM_XTYPE_INTERNAL,
+	};
+
+	vq = &odm->vq[vchan];
+	const uint16_t max_iring_words = vq->iring_max_words;
+
+	iring_head_ptr = vq->iring_mz->addr;
+	iring_head = vq->iring_head;
+	iring_sz_available = vq->iring_sz_available;
+	ins_ring_head = vq->ins_ring_head;
+	pending_submit_len = vq->pending_submit_len;
+	pending_submit_cnt = vq->pending_submit_cnt;
+
+	if (unlikely(nb_src > 4 || nb_dst > 4))
+		return -EINVAL;
+
+	for (i = 0; i < nb_src; i++)
+		s_sz += src[i].length;
+
+	for (i = 0; i < nb_dst; i++)
+		d_sz += dst[i].length;
+
+	if (s_sz != d_sz)
+		return -EINVAL;
+
+	nb = nb_src + nb_dst;
+	hdr.s.nfst = nb_src;
+	hdr.s.nlst = nb_dst;
+	num_words = 1 + 3 * (nb / 2 + (nb & 0x1));
+
+	if (iring_sz_available < num_words)
+		return -ENOSPC;
+
+	if ((iring_head + num_words) >= max_iring_words) {
+		uint16_t words_avail = max_iring_words - iring_head;
+		uint16_t words_pend = num_words - words_avail;
+
+		if (unlikely(words_avail + words_pend > ODM_IRING_ENTRY_SIZE_MAX))
+			return -ENOSPC;
+
+		odm_dmadev_fill_sg(cmd, src, dst, nb_src, nb_dst, &hdr);
+		rte_memcpy((void *)&iring_head_ptr[iring_head], (void *)cmd, words_avail * 8);
+		rte_memcpy((void *)iring_head_ptr, (void *)&cmd[words_avail], words_pend * 8);
+		iring_head = words_pend;
+	} else {
+		odm_dmadev_fill_sg(&iring_head_ptr[iring_head], src, dst, nb_src, nb_dst, &hdr);
+		iring_head += num_words;
+	}
+
+	pending_submit_len += num_words;
+
+	if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
+		rte_wmb();
+		odm_write64(pending_submit_len, odm->rbase + ODM_VDMA_DBELL(vchan));
+		vq->stats.submitted += pending_submit_cnt + 1;
+		vq->pending_submit_len = 0;
+		vq->pending_submit_cnt = 0;
+	} else {
+		vq->pending_submit_len = pending_submit_len;
+		vq->pending_submit_cnt++;
+	}
+
+	vq->iring_head = iring_head;
+
+	vq->iring_sz_available = iring_sz_available - num_words;
+
+	/* Save extra space used for the instruction. */
+	vq->extra_ins_sz[ins_ring_head] = num_words - 4;
+
+	vq->ins_ring_head = (ins_ring_head + 1) % vq->cring_max_entry;
+
+	return vq->desc_idx++;
+}
+
 static int
 odm_stats_get(const struct rte_dma_dev *dev, uint16_t vchan, struct rte_dma_stats *rte_stats,
 	      uint32_t size)