vdpa/sfc: introduce Xilinx vDPA driver

drivers/vdpa/sfc/meson.build line 23

Quoted 11 lines · On Tue, 6 Jul 2021 22:14:09 +0530 Vijay Srivastava…

drivers/vdpa/sfc/meson.build new file +11 -0

+# Enable more warnings
+extra_flags += [
+	'-Wdisabled-optimization'
+]
+
+# Compiler and version dependent flags
+extra_flags += [
+	'-Waggregate-return',
+	'-Wbad-function-cast'
+]
+

Having each driver decide on different compiler flags seems like a bad policy in DPDK.

Stephen Hemminger

2021-08-13 15:36 UTC #8 lore raw

Quoted 21 lines · On Tue, 6 Jul 2021 22:14:09 +0530 Vijay Srivastava…

drivers/vdpa/sfc/sfc_vdpa.c new file +21 -0

+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str, uint32_t ll_default)
+{
+	size_t lt_prefix_str_size = strlen(lt_prefix_str);
+	size_t lt_str_size_max;
+	char *lt_str = NULL;
+	int ret;
+
+	if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) {
+		++lt_prefix_str_size; /* Reserve space for prefix separator */
+		lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1;
+	} else {
+		return RTE_LOGTYPE_PMD;
+	}
+
+	lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0);
+	if (lt_str == NULL)
+		return RTE_LOGTYPE_PMD;
+
+	strncpy(lt_str, lt_prefix_str, lt_prefix_str_size);

drivers/vdpa/sfc/sfc_vdpa.c line 183

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+	lt_str[lt_prefix_str_size - 1] = '.';
+	rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size,
+			    lt_str_size_max - lt_prefix_str_size);
+	lt_str[lt_str_size_max - 1] = '\0';
+
+	ret = rte_log_register_type_and_pick_level(lt_str, ll_default);
+	rte_free(lt_str);
+
+	return (ret < 0) ? RTE_LOGTYPE_PMD : ret;
+}

This seems like overkill doing per-device log level. Other drivers aren't doing this.

Once again my advice to DPDK driver developers is "don't be a snowflake" your driver is not special.

Vijay Kumar Srivastava

2021-10-29 11:32 UTC #9 lore raw

Hi Stephen,

Quoted 35 lines

-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Friday, August 13, 2021 9:06 PM
To: Vijay Kumar Srivastava <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [dpdk-dev] [PATCH 01/10] vdpa/sfc: introduce Xilinx vDPA driver
Quoted 24 lines · On Tue, 6 Jul 2021 22:14:09 +0530 Vijay Srivastava…
drivers/vdpa/sfc/sfc_vdpa.c new file +24 -0
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str, uint32_t ll_default) {
+	size_t lt_prefix_str_size = strlen(lt_prefix_str);
+	size_t lt_str_size_max;
+	char *lt_str = NULL;
+	int ret;
+
+	if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) {
+		++lt_prefix_str_size; /* Reserve space for prefix separator */
+		lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1;
+	} else {
+		return RTE_LOGTYPE_PMD;
+	}
+
+	lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0);
+	if (lt_str == NULL)
+		return RTE_LOGTYPE_PMD;
+
+	strncpy(lt_str, lt_prefix_str, lt_prefix_str_size);
+	lt_str[lt_prefix_str_size - 1] = '.';
+	rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size,
+			    lt_str_size_max - lt_prefix_str_size);
+	lt_str[lt_str_size_max - 1] = '\0';

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa.c
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa.c new file +5 -0
+
+	ret = rte_log_register_type_and_pick_level(lt_str, ll_default);
+	rte_free(lt_str);
+
+	return (ret < 0) ? RTE_LOGTYPE_PMD : ret; }
This seems like overkill doing per-device log level. Other drivers aren't doing this.

We use it for the debugging. This feature is useful for us so it's good to have it.

Stephen Hemminger

2021-08-13 15:36 UTC #10 lore raw

drivers/vdpa/sfc/sfc_vdpa.c line 25

Quoted 2 lines · On Tue, 6 Jul 2021 22:14:09 +0530 Vijay Srivastava…

drivers/vdpa/sfc/sfc_vdpa.c new file +2 -0

+static pthread_mutex_t sfc_vdpa_adapter_list_lock = PTHREAD_MUTEX_INITIALIZER;
+

Why do you need a pthread_mutex when simple DPDK spin lock will do?

Vijay Kumar Srivastava

2021-10-28 18:14 UTC #11 lore raw

Hi Stephen,

Quoted 8 lines

-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Friday, August 13, 2021 9:07 PM
To: Vijay Kumar Srivastava <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [dpdk-dev] [PATCH 01/10] vdpa/sfc: introduce Xilinx vDPA driver

Quoted 8 lines

Quoted 1 line · On Tue, 6 Jul 2021 22:14:09 +0530 Vijay Srivastava…
+static pthread_mutex_t sfc_vdpa_adapter_list_lock =
PTHREAD_MUTEX_INITIALIZER;
Quoted 1 line
+
Why do you need a pthread_mutex when simple DPDK spin lock will do?

rte_spinlock_lock is being used in the subsequent patches.
Only for the adapter list (sfc_vdpa_adapter_list) pthread_mutex is being used.

Regards,
Vijay

Vijay Srivastava

author

2021-07-07 08:25 UTC #12 lore raw

PATCH 2/10 vdpa/sfc: add support for device initialization

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst       |   6 +
 drivers/vdpa/sfc/meson.build      |   3 +
 drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
 drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
 drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
 drivers/vdpa/sfc/sfc_vdpa_hw.c    | 322 ++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
 10 files changed, 665 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

doc/guides/vdpadevs/sfc.rst +6 -0

index 59f990b..abb5900 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types available for control:
   Matches a subset of per-port log types registered during runtime.
   A full name for a particular type may be obtained by appending a
   dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.
+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +3 -0

index d916389..aac7c51 100644
@@ -30,4 +30,7 @@ endforeach
 deps += ['common_sfc_efx', 'bus_pci']
 sources = files(
 	'sfc_vdpa.c',
+	'sfc_vdpa_hw.c',
+	'sfc_vdpa_mcdi.c',
+	'sfc_vdpa_ops.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +23 -0

index d8faaca..12e8d6e 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
 		goto fail_vfio_setup;
 	}
 
+	sfc_vdpa_log_init(sva, "hw init");
+	if (sfc_vdpa_hw_init(sva) != 0) {
+		sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+		goto fail_hw_init;
+	}
+
+	sfc_vdpa_log_init(sva, "dev init");
+	sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+	if (sva->ops_data == NULL) {
+		sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+		goto fail_dev_init;
+	}
+
 	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
 	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
 
 	return 0;
 
+fail_dev_init:
+	sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+	sfc_vdpa_vfio_teardown(sva);
+
 fail_vfio_setup:
 fail_set_log_prefix:
 	rte_free(sva);

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
 	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
 
+	sfc_vdpa_device_fini(sva->ops_data);
+
+	sfc_vdpa_hw_fini(sva);
+
 	sfc_vdpa_vfio_teardown(sva);
 
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +48 -1

73 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +21 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug check
+ * in the driver only.
+ */
+#define SFC_VDPA_ASSERT(exp)			RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)			RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +322 -0

322 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_log.h +3 -0

index 0a3d6ad..59af790 100644
@@ -21,6 +21,9 @@
 /** Name prefix for the per-device log type used to report basic information */
 #define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
 
+/** Device MCDI log type name prefix */
+#define SFC_VDPA_LOGTYPE_MCDI_STR	SFC_VDPA_LOGTYPE_PREFIX "mcdi"
+
 #define SFC_VDPA_LOG_PREFIX_MAX	32
 
 /* Log PMD message, automatically add prefix and \n */

drivers/vdpa/sfc/sfc_vdpa_mcdi.c new file +74 -0

74 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +129 -0

129 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +36 -0

new file mode 100644
index 0000000..817b302
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_OPS_H
+#define _SFC_VDPA_OPS_H
+
+#include <rte_vdpa.h>
+
+#define SFC_VDPA_MAX_QUEUE_PAIRS		1
+
+enum sfc_vdpa_context {
+	SFC_VDPA_AS_PF = 0,
+	SFC_VDPA_AS_VF
+};
+
+enum sfc_vdpa_state {
+	SFC_VDPA_STATE_UNINITIALIZED = 0,
+	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_NSTATES
+};
+
+struct sfc_vdpa_ops_data {
+	void				*dev_handle;
+	struct rte_vdpa_device		*vdpa_dev;
+	enum sfc_vdpa_context		vdpa_context;
+	enum sfc_vdpa_state		state;
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *adapter, enum sfc_vdpa_context context);
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data);
+
+#endif /* _SFC_VDPA_OPS_H */

Signature · 1 line

1.8.3.1

Maxime Coquelin

2021-08-30 09:16 UTC #13 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Quoted 23 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst       |   6 +
 drivers/vdpa/sfc/meson.build      |   3 +
 drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
 drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
 drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
 drivers/vdpa/sfc/sfc_vdpa_hw.c    | 322 ++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
 10 files changed, 665 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

...

Quoted 31 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +25 -0

new file mode 100644
index 0000000..83f3696
@@ -0,0 +1,322 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_vfio.h>
+
+#include "efx.h"
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_ops.h"
+
+extern uint32_t sfc_logtype_driver;
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE   (sysconf(_SC_PAGESIZE))
+#endif
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp)
+{

drivers/vdpa/sfc/sfc_vdpa_hw.c line 35

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	void *mcdi_buf;
+	uint64_t mcdi_iova;
+	size_t mcdi_buff_size;
+	int ret;
+
+	mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+
+	sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+	mcdi_buf = rte_zmalloc(name, mcdi_buff_size, PAGE_SIZE);

You might want to allocate on the same NUMA node the device is on?

Quoted 22 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +22 -0

+	if (mcdi_buf == NULL) {
+		sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x: %s",
+			     name, (unsigned int)len, rte_strerror(rte_errno));
+		return -ENOMEM;
+	}
+
+	/* IOVA address for MCDI would be re-calculated if mapping
+	 * using default IOVA would fail.
+	 * TODO: Earlier there was no way to get valid IOVA range.
+	 * Recently a patch has been submitted to get the IOVA range
+	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+	 * in the kernel version >= 5.4. Support to get the default
+	 * IOVA address for MCDI buffer using available IOVA range
+	 * would be added later. Meanwhile default IOVA for MCDI buffer
+	 * is kept at high mem at 2TB. In case of overlap new available
+	 * addresses would be searched and same would be used.
+	 */
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mcdi_buf, mcdi_iova,

drivers/vdpa/sfc/sfc_vdpa_hw.c line 67

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			return ret;

You leak mcdi_buf here if DMA map fails.

Quoted 5 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0
+		}
+
+	} while (ret < 0);
+
+	esmp->esm_addr = mcdi_iova;

drivers/vdpa/sfc/sfc_vdpa_hw.c line 82

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	esmp->esm_base = mcdi_buf;
+	sva->mcdi_buff_size = mcdi_buff_size;
+
+	sfc_vdpa_info(sva,
+		      "DMA name=%s len=%zu => virt=%p iova=%" PRIx64,
+		      name, len, esmp->esm_base, esmp->esm_addr);
+
+	return 0;
+}
+

Thanks,
Maxime

Xia, Chenbo

2021-08-30 10:52 UTC #14 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Vijay,

Quoted 159 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Wednesday, July 7, 2021 12:44 AM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH 02/10] vdpa/sfc: add support for device initialization

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst       |   6 +
 drivers/vdpa/sfc/meson.build      |   3 +
 drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
 drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
 drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
 drivers/vdpa/sfc/sfc_vdpa_hw.c    | 322
++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
 10 files changed, 665 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

doc/guides/vdpadevs/sfc.rst +0 -0

index 59f990b..abb5900 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types available for

control:
Matches a subset of per-port log types registered during runtime. A full name for a particular type may be obtained by appending a dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.

doc/guides/vdpadevs/sfc.rst +6 -0

+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +3 -0

index d916389..aac7c51 100644
@@ -30,4 +30,7 @@ endforeach
 deps += ['common_sfc_efx', 'bus_pci']
 sources = files(
 	'sfc_vdpa.c',
+	'sfc_vdpa_hw.c',
+	'sfc_vdpa_mcdi.c',
+	'sfc_vdpa_ops.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +23 -0

index d8faaca..12e8d6e 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
 		goto fail_vfio_setup;
 	}
 
+	sfc_vdpa_log_init(sva, "hw init");
+	if (sfc_vdpa_hw_init(sva) != 0) {
+		sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+		goto fail_hw_init;
+	}
+
+	sfc_vdpa_log_init(sva, "dev init");
+	sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+	if (sva->ops_data == NULL) {
+		sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+		goto fail_dev_init;
+	}
+
 	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
 	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
 
 	return 0;
 
+fail_dev_init:
+	sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+	sfc_vdpa_vfio_teardown(sva);
+
 fail_vfio_setup:
 fail_set_log_prefix:
 	rte_free(sva);

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
 	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
 
+	sfc_vdpa_device_fini(sva->ops_data);
+
+	sfc_vdpa_hw_fini(sva);
+
 	sfc_vdpa_vfio_teardown(sva);
 
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +25 -0

index 3b77900..fb97258 100644
@@ -11,14 +11,38 @@
 
 #include <rte_bus_pci.h>
 
+#include "sfc_efx.h"
+#include "sfc_efx_mcdi.h"
+#include "sfc_vdpa_debug.h"
 #include "sfc_vdpa_log.h"
+#include "sfc_vdpa_ops.h"
+
+#define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
 /* Adapter private data */
 struct sfc_vdpa_adapter {
 	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	/*
+	 * PMD setup and configuration is not thread safe. Since it is not
+	 * performance sensitive, it is better to guarantee thread-safety
+	 * and add device level lock. vDPA control operations which
+	 * change its state should acquire the lock.
+	 */
+	rte_spinlock_t			lock;
 	struct rte_pci_device		*pdev;
 	struct rte_pci_addr		pci_addr;
 
+	efx_family_t			family;
+	efx_nic_t			*nic;
+	rte_spinlock_t			nic_lock;
+
+	efsys_bar_t			mem_bar;
+
+	struct sfc_efx_mcdi		mcdi;
+	size_t				mcdi_buff_size;
+
+	uint32_t			max_queue_count;
+
 	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
 	uint32_t			logtype_main;

@@ -26,6 +50,7 @@ struct sfc_vdpa_adapter {
 	int				vfio_dev_fd;
 	int				vfio_container_fd;
 	int				iommu_group_num;
+	struct sfc_vdpa_ops_data	*ops_data;
 };
 
 uint32_t

drivers/vdpa/sfc/sfc_vdpa.h struct sfc_vdpa_adapter { · line 67

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa.h +4 -1

@@ -36,5 +61,27 @@ struct sfc_vdpa_adapter {
 struct sfc_vdpa_adapter *
 sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
 
-#endif  /* _SFC_VDPA_H */
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sa);

Better to align the name here: sa -> sva

Quoted 86 lines

drivers/vdpa/sfc/sfc_vdpa.h +19 -0

+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva);
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp);
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
+
+static inline struct sfc_vdpa_adapter *
+sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
+{
+	return (struct sfc_vdpa_adapter *)dev_handle;
+}
+
+#endif  /* _SFC_VDPA_H */

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +21 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug check
+ * in the driver only.
+ */
+#define SFC_VDPA_ASSERT(exp)			RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)			RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +32 -0

new file mode 100644
index 0000000..83f3696
@@ -0,0 +1,322 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_vfio.h>
+
+#include "efx.h"
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_ops.h"
+
+extern uint32_t sfc_logtype_driver;
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE   (sysconf(_SC_PAGESIZE))
+#endif
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp)
+{
+	void *mcdi_buf;
+	uint64_t mcdi_iova;
+	size_t mcdi_buff_size;
+	int ret;
+
+	mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+

drivers/vdpa/sfc/sfc_vdpa_hw.c line 42

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+	mcdi_buf = rte_zmalloc(name, mcdi_buff_size, PAGE_SIZE);
+	if (mcdi_buf == NULL) {
+		sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x: %s",
+			     name, (unsigned int)len, rte_strerror(rte_errno));
+		return -ENOMEM;
+	}
+
+	/* IOVA address for MCDI would be re-calculated if mapping

What is MCDI?

Quoted 18 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +18 -0

+	 * using default IOVA would fail.
+	 * TODO: Earlier there was no way to get valid IOVA range.
+	 * Recently a patch has been submitted to get the IOVA range
+	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+	 * in the kernel version >= 5.4. Support to get the default
+	 * IOVA address for MCDI buffer using available IOVA range
+	 * would be added later. Meanwhile default IOVA for MCDI buffer
+	 * is kept at high mem at 2TB. In case of overlap new available
+	 * addresses would be searched and same would be used.
+	 */
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mcdi_buf, mcdi_iova,
+						 mcdi_buff_size);
+		if (ret == 0)
+			break;

drivers/vdpa/sfc/sfc_vdpa_hw.c line 70

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			return ret;
+		}
+
+	} while (ret < 0);

Is this DMA region for some hardware-specific control msg?

And how do you make sure this IOVA space you defined in this driver will not conflict with the IOVA space that vdpa device consumer (Most likely QEMU) defines (If QEMU, IOVA = guest physical address)

Thanks,
Chenbo

Quoted 12 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +12 -0

+
+	esmp->esm_addr = mcdi_iova;
+	esmp->esm_base = mcdi_buf;
+	sva->mcdi_buff_size = mcdi_buff_size;
+
+	sfc_vdpa_info(sva,
+		      "DMA name=%s len=%zu => virt=%p iova=%" PRIx64,
+		      name, len, esmp->esm_base, esmp->esm_addr);
+
+	return 0;
+}
+

Vijay Kumar Srivastava

2021-09-03 13:19 UTC #15 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Chenbo,

Quoted 25 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, August 30, 2021 4:22 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 15 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Wednesday, July 7, 2021 12:44 AM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---

[snip]

Quoted 16 lines

Quoted 5 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0

+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp)
+{
+	void *mcdi_buf;
+	uint64_t mcdi_iova;

drivers/vdpa/sfc/sfc_vdpa_hw.c

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	size_t mcdi_buff_size;
+	int ret;
+
+	mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+
+	sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+	mcdi_buf = rte_zmalloc(name, mcdi_buff_size, PAGE_SIZE);
+	if (mcdi_buf == NULL) {
+		sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x:

%s",

Quoted 7 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c line 42
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0
+			     name, (unsigned int)len, rte_strerror(rte_errno));
+		return -ENOMEM;
+	}
+
+	/* IOVA address for MCDI would be re-calculated if mapping
What is MCDI?

MCDI is a control interface between driver and firmware.
It is used by the host drivers to configure the adapter and retrieve status.

Quoted 32 lines

Quoted 5 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0

+	 * using default IOVA would fail.
+	 * TODO: Earlier there was no way to get valid IOVA range.
+	 * Recently a patch has been submitted to get the IOVA range
+	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+	 * in the kernel version >= 5.4. Support to get the default

drivers/vdpa/sfc/sfc_vdpa_hw.c

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	 * IOVA address for MCDI buffer using available IOVA range
+	 * would be added later. Meanwhile default IOVA for MCDI buffer
+	 * is kept at high mem at 2TB. In case of overlap new available
+	 * addresses would be searched and same would be used.
+	 */
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mcdi_buf,

mcdi_iova,

drivers/vdpa/sfc/sfc_vdpa_hw.c line 70

Quoted 13 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +13 -0

+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			return ret;
+		}
+
+	} while (ret < 0);

Is this DMA region for some hardware-specific control msg?

Quoted 3 lines

And how do you make sure this IOVA space you defined in this driver will not conflict with the IOVA space that vdpa device consumer (Most likely QEMU) defines (If QEMU, IOVA = guest physical address)

Currently IOVA for MCDI buffer is kept at very high mem at 2TB.

To handle IOVA overlap detection scenario a patch is in progress which will be submitted soon.
In that patch, upon IOVA overlap detection new available IOVA would be calculated and MCDI buffer would be remapped to new IOVA.

[snip]

Thanks,
Vijay

Xia, Chenbo

2021-09-06 03:02 UTC #16 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi,

Quoted 63 lines

-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Friday, September 3, 2021 9:20 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 25 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, August 30, 2021 4:22 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 15 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Wednesday, July 7, 2021 12:44 AM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---
[snip]
Quoted 16 lines
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp)
+{
+	void *mcdi_buf;
+	uint64_t mcdi_iova;
drivers/vdpa/sfc/sfc_vdpa_hw.c
Quoted 10 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0
+	size_t mcdi_buff_size;
+	int ret;
+
+	mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+
+	sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+	mcdi_buf = rte_zmalloc(name, mcdi_buff_size, PAGE_SIZE);
+	if (mcdi_buf == NULL) {
+		sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x:
%s",
Quoted 7 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c line 42
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0
+			     name, (unsigned int)len, rte_strerror(rte_errno));
+		return -ENOMEM;
+	}
+
+	/* IOVA address for MCDI would be re-calculated if mapping
What is MCDI?

Quoted 2 lines

MCDI is a control interface between driver and firmware.
It is used by the host drivers to configure the adapter and retrieve status.

Cool, thanks for explanation.

Quoted 33 lines

Quoted 6 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0

+	 * using default IOVA would fail.
+	 * TODO: Earlier there was no way to get valid IOVA range.
+	 * Recently a patch has been submitted to get the IOVA range
+	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+	 * in the kernel version >= 5.4. Support to get the default

drivers/vdpa/sfc/sfc_vdpa_hw.c

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	 * IOVA address for MCDI buffer using available IOVA range
+	 * would be added later. Meanwhile default IOVA for MCDI buffer
+	 * is kept at high mem at 2TB. In case of overlap new available
+	 * addresses would be searched and same would be used.
+	 */
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mcdi_buf,

mcdi_iova,

drivers/vdpa/sfc/sfc_vdpa_hw.c line 70

Quoted 13 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +13 -0

+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			return ret;
+		}
+
+	} while (ret < 0);

Is this DMA region for some hardware-specific control msg?

Quoted 5 lines

Quoted 3 lines
And how do you make sure this IOVA space you defined in this driver will not conflict with the IOVA space that vdpa device consumer (Most likely QEMU) defines (If QEMU, IOVA = guest physical address)
Currently IOVA for MCDI buffer is kept at very high mem at 2TB.

OK. That sounds a work-around to me but we can't make assumption of consumer not using that address range. And there is a security issue here, please see below comment.

Quoted 5 lines

To handle IOVA overlap detection scenario a patch is in progress which will be submitted soon.
In that patch, upon IOVA overlap detection new available IOVA would be calculated and MCDI buffer would be remapped to new IOVA.

Let's say there is a malicious guest who knows your initial IOVA range that is set up by your driver (even if it does not know, it can use tests to know. So use static IOVA range in host is more dangerous). It can use that address in any DMA-able queue and make DMA into the vdpa app. I think it could cause some security issue as you let guest easily writing host memory.

For now I don't see a perfect solution except PASID(Process Address Space ID). IIRC, We could let QEMU have a primary PASID and vdpa app have a secondary PASID so that VM can't perform DMA to vdpa app. But since it needs HW support and related support in vfio is not mature, I don't think we are able to use that solution now.

Any solution you can think of for your HW?

Thanks,
Chenbo

Quoted 5 lines

[snip]
Thanks,
Vijay

Vijay Kumar Srivastava

2021-10-01 17:31 UTC #17 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Chenbo,

Quoted 20 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, September 6, 2021 8:32 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi,
Quoted 10 lines
-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Friday, September 3, 2021 9:20 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization

[snip]

Quoted 7 lines

Quoted 4 lines
To handle IOVA overlap detection scenario a patch is in progress which will be submitted soon.
In that patch, upon IOVA overlap detection new available IOVA would be calculated and MCDI buffer would be remapped to new IOVA.
Let's say there is a malicious guest who knows your initial IOVA range that is set up by your driver (even if it does not know, it can use tests to know. So use static IOVA range in host is more dangerous).

Upcoming patch will handle IOVA conflict scenario. With that patch hardcoded IOVA would not be needed.
If malicious guest will try to use MCDI IOVA address then vDPA driver would detect IOVA overlap and would remap MCDI buffer to another available IOVA address. This IOVA address is for MCDI buffer which is used for the control path.
Just by only writing to MCDI buffer does not imply that malicious guest can send any control message to NIC to modify HW configuration.

Quoted 2 lines

It can use that address in any DMA-able queue and make DMA into the vdpa app. I think it could cause some security issue as you let guest easily writing host memory.

Can you please elaborate on this ?
In what scenarios host physical address can be accessed by malicious guest ?

Quoted 6 lines

For now I don't see a perfect solution except PASID(Process Address Space ID). IIRC, We could let QEMU have a primary PASID and vdpa app have a secondary PASID so that VM can't perform DMA to vdpa app. But since it needs HW support and related support in vfio is not mature, I don't think we are able to use that solution now.
Any solution you can think of for your HW?

Yes, It can be used. Our next version of HW will have the PASID support.

Regards,
Vijay

Xia, Chenbo

2021-10-09 03:06 UTC #18 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Vijay,

Quoted 40 lines

-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Saturday, October 2, 2021 1:32 AM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 7 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, September 6, 2021 8:32 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device
initialization
Quoted 13 lines
Hi,
Quoted 10 lines
-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Friday, September 3, 2021 9:20 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
[snip]
Quoted 5 lines
Quoted 4 lines
To handle IOVA overlap detection scenario a patch is in progress which will be submitted soon.
In that patch, upon IOVA overlap detection new available IOVA would be calculated and MCDI buffer would be remapped to new IOVA.
Let's say there is a malicious guest who knows your initial IOVA range
that is set

Quoted 8 lines

Quoted 1 line
up by your driver (even if it does not know, it can use tests to know. So
use static
Quoted 1 line
IOVA range in host is more dangerous).
Upcoming patch will handle IOVA conflict scenario. With that patch hardcoded IOVA would not be needed.
If malicious guest will try to use MCDI IOVA address then vDPA driver would detect IOVA overlap and would remap MCDI buffer to another available IOVA address.

Yes, I think I know your solution of driver solving the overlap problem.

Quoted 10 lines

This IOVA address is for MCDI buffer which is used for the control path. Just by only writing to MCDI buffer does not imply that malicious guest can send any control message to NIC to modify HW configuration.
Quoted 1 line
It can use that address in any DMA-able queue and make DMA into the vdpa
app. I think it could cause some security issue
Quoted 1 line
as you let guest easily writing host memory.
Can you please elaborate on this ?
In what scenarios host physical address can be accessed by malicious guest ?

As I have not reviewed the full series and not familiar with your HW. You can correct my understanding below:

I think your vdpa HW (let's say a VF) have two DMA regions: one in guest (w/o vIOMMU) and the other in vdpa app. Both share the same IOVA address space, and we don't want them overlap. Let's say we can make sure no overlap will happen and take an example here: guest DMA region's IOVA (GPA) range is 0x0000 to 0x1000 and vdpa app's is 0x1000 to 0x2000. A malicious guest could use a malicious driver to write 0x1500 in its virtio RX ring, so that HW will DMA to that address when packets come. Then the malicious guest performed an DMA to host memory. Although the guest does not know IOVA range of vdpa app, he can randomly guess to do the attack.

Any solution your HW/driver can prevent this from happening without PASID? Or do I miss something here?

Thanks,
Chenbo

Quoted 16 lines

Quoted 2 lines
For now I don't see a perfect solution except PASID(Process Address Space
ID).
Quoted 1 line
IIRC, We could let QEMU have a primary PASID and vdpa app have a
secondary
Quoted 1 line
PASID so that VM can't perform DMA to vdpa app. But since it needs HW
support
Quoted 1 line
and related support in vfio is not mature, I don't think we are able to
use that
Quoted 2 lines
solution now.
Any solution you can think of for your HW?
Yes, It can be used. Our next version of HW will have the PASID support.
Regards,
Vijay

Vijay Kumar Srivastava

2021-10-18 10:06 UTC #19 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Chenbo,

Quoted 31 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Saturday, October 9, 2021 8:36 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 21 lines
-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Saturday, October 2, 2021 1:32 AM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 8 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, September 6, 2021 8:32 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device
initialization

[Snip]

Quoted 10 lines

I think your vdpa HW (let's say a VF) have two DMA regions: one in guest (w/o vIOMMU) and the other in vdpa app. Both share the same IOVA address space, and we don't want them overlap. Let's say we can make sure no overlap will happen and take an example here: guest DMA region's IOVA (GPA) range is 0x0000 to 0x1000 and vdpa app's is 0x1000 to 0x2000. A malicious guest could use a malicious driver to write 0x1500 in its virtio RX ring, so that HW will DMA to that address when packets come. Then the malicious guest performed an DMA to host memory. Although the guest does not know IOVA range of vdpa app, he can randomly guess to do the attack.

Quoted 2 lines

Any solution your HW/driver can prevent this from happening without PASID? Or do I miss something here ?

Rx packet will carry headers making highly unlikely any proper MCDI data can be written to the IOVA address (for MCDI buffer) to work with by the FW. Writing to the buffer does not imply to issue the MCDI message. Even if MCDI is sent then FW is resilient enough to identify the incorrect MCDI and will reject the message.

This is going to affect only to VF on which malicious guest is present, as this MCDI buffer is specific to the corresponding VF. So it won't affect any control path operation on the any other VF or host.

For SW assisted Live migration implemented in the ifcvf vDPA driver it uses hard coded IOVA addresses for mediated vring. Could it have similar issue ?

Xia, Chenbo

2021-10-19 02:16 UTC #20 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Vijay,

Quoted 64 lines

-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Monday, October 18, 2021 6:06 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 31 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Saturday, October 9, 2021 8:36 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 21 lines
-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Saturday, October 2, 2021 1:32 AM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 8 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, September 6, 2021 8:32 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device
initialization
[Snip]
Quoted 6 lines
I think your vdpa HW (let's say a VF) have two DMA regions: one in guest (w/o vIOMMU) and the other in vdpa app. Both share the same IOVA address space, and we don't want them overlap. Let's say we can make sure no overlap will happen and take an example here: guest DMA region's IOVA (GPA) range is 0x0000 to 0x1000 and vdpa app's is 0x1000 to 0x2000. A malicious guest could use a malicious driver to write 0x1500 in its virtio RX ring, so that HW will
DMA
Quoted 6 lines
to that address when packets come. Then the malicious guest performed an DMA to host memory. Although the guest does not know IOVA range of vdpa app, he can randomly guess to do the attack.
Any solution your HW/driver can prevent this from happening without PASID? Or do I miss something here ?
Rx packet will carry headers making highly unlikely any proper MCDI data can be written to the IOVA address (for MCDI buffer) to work with by the FW. Writing to the buffer does not imply to issue the MCDI message. Even if MCDI is sent then FW is resilient enough to identify the incorrect MCDI and will reject the message.

Quoted 3 lines

This is going to affect only to VF on which malicious guest is present, as this MCDI buffer is specific to the corresponding VF.
So it won't affect any control path operation on the any other VF or host.

OK. So it's very hard to do attack with the FW detection. But about 'won't affect host', I think it depends on how you handle the DMA-ed control messages. Take a bad
example: if one DMA address saves a pointer and the malicious DMA makes the pointer be NULL, it will segfaults the program (But I don't think this will happen in your driver, just help you understand my point). So please check the control messages handling is robust.

And in the future, I would like to see this problem solved by PASID when your HW has the support.

Quoted 3 lines

For SW assisted Live migration implemented in the ifcvf vDPA driver it uses hard coded IOVA addresses for mediated vring. Could it have similar issue ?

Good point. It will and I think we may also need to check if will affect the host program, or deprecated the feature later.

Thanks,
Chenbo

Vijay Kumar Srivastava

2021-10-25 06:11 UTC #21 lore raw

Re: [dpdk-dev] [PATCH 02/10] vdpa/sfc: add support for device initialization

Hi Chenbo,

Quoted 7 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Tuesday, October 19, 2021 7:47 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Harpreet Singh Anand <[email protected]>; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH 02/10] vdpa/sfc: add support for device initialization

[Snip]

Quoted 11 lines

Quoted 2 lines
Rx packet will carry headers making highly unlikely any proper MCDI data can be written to the IOVA address (for MCDI buffer) to work with by the
FW.
Quoted 7 lines
Writing to the buffer does not imply to issue the MCDI message. Even if MCDI is sent then FW is resilient enough to identify the incorrect MCDI and will reject the message.
This is going to affect only to VF on which malicious guest is present, as this MCDI buffer is specific to the corresponding VF. So it won't affect any control path operation on the any other VF or host.

Quoted 7 lines

OK. So it's very hard to do attack with the FW detection. But about 'won't affect host', I think it depends on how you handle the DMA-ed control messages. Take a bad
example: if one DMA address saves a pointer and the malicious DMA makes the pointer be NULL, it will segfaults the program (But I don't think this will happen in your driver, just help you understand my point). So please check the control messages handling is robust.

Yes, It is highly unlikely that it can affect the host.

Quoted 2 lines

And in the future, I would like to see this problem solved by PASID when your HW has the support.

Yes. Sure.

Regards,
Vijay

Vijay Srivastava

author

2021-07-07 08:25 UTC #22 lore raw

PATCH 3/10 vdpa/sfc: add support to get device and protocol features

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_feature and get_protocol_features. This patch retrieves device supported features and enables protocol features.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/features/sfc.ini | 10 ++++
 drivers/common/sfc_efx/efsys.h       |  2 +-
 drivers/common/sfc_efx/version.map   | 10 ++++
 drivers/vdpa/sfc/sfc_vdpa.c          | 20 ++++++++
 drivers/vdpa/sfc/sfc_vdpa.h          |  2 +
 drivers/vdpa/sfc/sfc_vdpa_hw.c       | 13 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c      | 91 ++++++++++++++++++++++++++++++++----
 drivers/vdpa/sfc/sfc_vdpa_ops.h      |  3 ++
 8 files changed, 142 insertions(+), 9 deletions(-)

doc/guides/vdpadevs/features/sfc.ini +10 -0

index 71b6158..700d061 100644
@@ -4,6 +4,16 @@
 ; Refer to default.ini for the full list of available driver features.
 ;
 [Features]
+csum			= Y
+guest csum		= Y
+host tso4		= Y
+host tso6		= Y
+version 1		= Y
+mrg rxbuf		= Y
+any layout		= Y
+in_order		= Y
+proto host notifier	= Y
+IOMMU platform		= Y
 Linux			= Y
 x86-64			= Y
 Usage doc		= Y

drivers/common/sfc_efx/efsys.h +1 -1

index d133d61..37ec6b9 100644
@@ -187,7 +187,7 @@
 
 #define EFSYS_OPT_MAE 1
 
-#define EFSYS_OPT_VIRTIO 0
+#define EFSYS_OPT_VIRTIO 1
 
 /* ID */

drivers/common/sfc_efx/version.map +10 -0

index 5e724fd..03670c8 100644
@@ -226,6 +226,16 @@ INTERNAL {
 	efx_txq_nbufs;
 	efx_txq_size;
 
+	efx_virtio_fini;
+	efx_virtio_get_doorbell_offset;
+	efx_virtio_get_features;
+	efx_virtio_init;
+	efx_virtio_qcreate;
+	efx_virtio_qdestroy;
+	efx_virtio_qstart;
+	efx_virtio_qstop;
+	efx_virtio_verify_features;
+
 	sfc_efx_dev_class_get;
 	sfc_efx_family;

drivers/vdpa/sfc/sfc_vdpa.c +20 -0

index 12e8d6e..9c12dcb 100644
@@ -43,6 +43,26 @@ struct sfc_vdpa_adapter *
 	return found ? sva : NULL;
 }
 
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev)
+{
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (vdpa_dev == sva->ops_data->vdpa_dev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva->ops_data : NULL;
+}
+
 static int
 sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva)
 {

drivers/vdpa/sfc/sfc_vdpa.h +2 -0

index fb97258..08075e5 100644
@@ -60,6 +60,8 @@ struct sfc_vdpa_adapter {
 
 struct sfc_vdpa_adapter *
 sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev);
 
 int
 sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);

drivers/vdpa/sfc/sfc_vdpa_hw.c +13 -0

index 83f3696..84e680f 100644
@@ -273,10 +273,20 @@
 	if (rc != 0)
 		goto fail_estimate_rsrc_limits;
 
+	sfc_vdpa_log_init(sva, "init virtio");
+	rc = efx_virtio_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "virtio init failed: %s", rte_strerror(rc));
+		goto fail_virtio_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_virtio_init:
+	efx_nic_fini(enp);
+
 fail_estimate_rsrc_limits:
 fail_nic_reset:
 	efx_nic_unprobe(enp);

@@ -305,6 +315,9 @@
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_log_init(sva, "virtio fini");
+	efx_virtio_fini(enp);
+
 	sfc_vdpa_log_init(sva, "unprobe nic");
 	efx_nic_unprobe(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +83 -8

138 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h +3 -0

index 817b302..21cbb73 100644
@@ -26,6 +26,9 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+
+	uint64_t			dev_features;
+	uint64_t			drv_features;
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Maxime Coquelin

Reviewed-by on v1 3/10

2021-08-30 09:34 UTC #23 lore raw

Re: [dpdk-dev] [PATCH 03/10] vdpa/sfc: add support to get device and protocol features

Quoted 7 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Implement vDPA ops get_feature and get_protocol_features. This patch retrieves device supported features and enables protocol features.
Signed-off-by Vijay Kumar Srivastava <[email protected]>

Quoted 11 lines

---
 doc/guides/vdpadevs/features/sfc.ini | 10 ++++
 drivers/common/sfc_efx/efsys.h       |  2 +-
 drivers/common/sfc_efx/version.map   | 10 ++++
 drivers/vdpa/sfc/sfc_vdpa.c          | 20 ++++++++
 drivers/vdpa/sfc/sfc_vdpa.h          |  2 +
 drivers/vdpa/sfc/sfc_vdpa_hw.c       | 13 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c      | 91 ++++++++++++++++++++++++++++++++----
 drivers/vdpa/sfc/sfc_vdpa_ops.h      |  3 ++
 8 files changed, 142 insertions(+), 9 deletions(-)

Thanks,
Maxime

Vijay Srivastava

author

2021-07-07 08:26 UTC #24 lore raw

PATCH 4/10 vdpa/sfc: get device supported max queue count

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_queue_num to get the maximum number of queues supported by the device.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +13 -3

index 5750944..6c702e1 100644
@@ -31,10 +31,20 @@
 static int
 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
 {
-	RTE_SET_USED(vdpa_dev);
-	RTE_SET_USED(queue_num);
+	struct sfc_vdpa_ops_data *ops_data;
+	void *dev;
 
-	return -1;
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	*queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
+
+	sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %d",
+		      *queue_num);
+
+	return 0;
 }
 
 static int

Signature · 1 line

1.8.3.1

Maxime Coquelin

Reviewed-by on v1 4/10

2021-08-30 09:35 UTC #25 lore raw

Re: [dpdk-dev] [PATCH 04/10] vdpa/sfc: get device supported max queue count

Quoted 10 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Implement vDPA ops get_queue_num to get the maximum number of queues supported by the device.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

Thanks,
Maxime

Vijay Srivastava

author

2021-07-07 08:26 UTC #26 lore raw

PATCH 5/10 vdpa/sfc: add support to get VFIO device fd

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_vfio_device_fd to get the VFIO device fd.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

drivers/vdpa/sfc/sfc_vdpa_ops.c +24 -0

index 6c702e1..5253adb 100644
@@ -145,6 +145,29 @@
 	return -1;
 }
 
+static int
+sfc_vdpa_get_vfio_device_fd(int vid)
+{
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	int vfio_dev_fd;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+
+	sfc_vdpa_info(dev, "vDPA ops get_vfio_device_fd :: vfio fd : %d",
+		      vfio_dev_fd);
+
+	return vfio_dev_fd;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -153,6 +176,7 @@
 	.dev_close = sfc_vdpa_dev_close,
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
+	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Maxime Coquelin

Reviewed-by on v1 5/10

2021-08-30 09:39 UTC #27 lore raw

Re: [dpdk-dev] [PATCH 05/10] vdpa/sfc: add support to get VFIO device fd

Quoted 9 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Implement vDPA ops get_vfio_device_fd to get the VFIO device fd.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

Thanks,
Maxime

Vijay Srivastava

author

2021-07-07 08:26 UTC #28 lore raw

PATCH 6/10 vdpa/sfc: add support for dev conf and dev close ops

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops dev_conf and dev_close for DMA mapping, interrupt and virtqueue configurations.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa.c     |   6 +
 drivers/vdpa/sfc/sfc_vdpa.h     |  43 ++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c  |  70 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 527 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |  28 +++
 5 files changed, 654 insertions(+), 20 deletions(-)

drivers/vdpa/sfc/sfc_vdpa.c +6 -0

index 9c12dcb..ca13483 100644
@@ -246,6 +246,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_adapter_lock_init(sva);
+
 	sfc_vdpa_log_init(sva, "vfio init");
 	if (sfc_vdpa_vfio_setup(sva) < 0) {
 		sfc_vdpa_err(sva, "failed to setup device %s", pci_dev->name);

@@ -280,6 +282,8 @@ struct sfc_vdpa_ops_data *
 	sfc_vdpa_vfio_teardown(sva);
 
 fail_vfio_setup:
+	sfc_vdpa_adapter_lock_fini(sva);
+
 fail_set_log_prefix:
 	rte_free(sva);

@@ -311,6 +315,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_vfio_teardown(sva);
 
+	sfc_vdpa_adapter_lock_fini(sva);
+
 	rte_free(sva);
 
 	return 0;

drivers/vdpa/sfc/sfc_vdpa.h +43 -0

53 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_hw.c +70 -0

82 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.c +507 ~~-20~~

563 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h +28 -0

index 21cbb73..8d553c5 100644
@@ -18,17 +18,45 @@ enum sfc_vdpa_context {
 enum sfc_vdpa_state {
 	SFC_VDPA_STATE_UNINITIALIZED = 0,
 	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_CONFIGURING,
+	SFC_VDPA_STATE_CONFIGURED,
+	SFC_VDPA_STATE_CLOSING,
+	SFC_VDPA_STATE_CLOSED,
+	SFC_VDPA_STATE_STARTING,
+	SFC_VDPA_STATE_STARTED,
+	SFC_VDPA_STATE_STOPPING,
 	SFC_VDPA_STATE_NSTATES
 };
 
+struct sfc_vdpa_vring_info {
+	uint64_t	desc;
+	uint64_t	avail;
+	uint64_t	used;
+	uint64_t	size;
+	uint16_t	last_avail_idx;
+	uint16_t	last_used_idx;
+};
+
+typedef struct sfc_vdpa_vq_context_s {
+	uint8_t				enable;
+	uint32_t			pidx;
+	uint32_t			cidx;
+	efx_virtio_vq_t			*vq;
+} sfc_vdpa_vq_context_t;
+
 struct sfc_vdpa_ops_data {
 	void				*dev_handle;
+	int				vid;
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;
+	uint64_t			req_features;
+
+	uint16_t			vq_count;
+	struct sfc_vdpa_vq_context_s	vq_cxt[SFC_VDPA_MAX_QUEUE_PAIRS * 2];
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Maxime Coquelin

2021-08-30 11:35 UTC #29 lore raw

Re: [dpdk-dev] [PATCH 06/10] vdpa/sfc: add support for dev conf and dev close ops

Quoted 14 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Implement vDPA ops dev_conf and dev_close for DMA mapping, interrupt and virtqueue configurations.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa.c     |   6 +
 drivers/vdpa/sfc/sfc_vdpa.h     |  43 ++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c  |  70 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 527 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |  28 +++
 5 files changed, 654 insertions(+), 20 deletions(-)

...

Quoted 70 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +55 -0

index 84e680f..047bcc4 100644
@@ -8,6 +8,7 @@
 #include <rte_common.h>
 #include <rte_errno.h>
 #include <rte_vfio.h>
+#include <rte_vhost.h>
 
 #include "efx.h"
 #include "sfc_vdpa.h"

@@ -104,6 +105,75 @@
 	memset(esmp, 0, sizeof(*esmp));
 }
 
+int
+sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *ops_data, bool do_map)
+{
+	uint32_t i, j;
+	int rc;
+	struct rte_vhost_memory *vhost_mem = NULL;
+	struct rte_vhost_mem_region *mem_reg = NULL;
+	int vfio_container_fd;
+	void *dev;
+
+	dev = ops_data->dev_handle;
+	vfio_container_fd =
+		sfc_vdpa_adapter_by_dev_handle(dev)->vfio_container_fd;
+
+	rc = rte_vhost_get_mem_table(ops_data->vid, &vhost_mem);
+	if (rc < 0) {
+		sfc_vdpa_err(dev,
+			     "failed to get VM memory layout");
+		goto error;
+	}
+
+	for (i = 0; i < vhost_mem->nregions; i++) {
+		mem_reg = &vhost_mem->regions[i];
+
+		if (do_map) {
+			rc = rte_vfio_container_dma_map(vfio_container_fd,
+						mem_reg->host_user_addr,
+						mem_reg->guest_phys_addr,
+						mem_reg->size);
+			if (rc < 0) {
+				sfc_vdpa_err(dev,
+					     "DMA map failed : %s",
+					     rte_strerror(rte_errno));
+				goto failed_vfio_dma_map;
+			}
+		} else {
+			rc = rte_vfio_container_dma_unmap(vfio_container_fd,
+						mem_reg->host_user_addr,
+						mem_reg->guest_phys_addr,
+						mem_reg->size);
+			if (rc < 0) {
+				sfc_vdpa_err(dev,
+					     "DMA unmap failed : %s",
+					     rte_strerror(rte_errno));
+				goto error;
+			}
+		}
+	}
+
+	free(vhost_mem);
+
+	return 0;
+
+failed_vfio_dma_map:

drivers/vdpa/sfc/sfc_vdpa_hw.c line 171

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +10 -0

+	for (j = 0; j < i; j++) {
+		mem_reg = &vhost_mem->regions[j];
+		rc = rte_vfio_container_dma_unmap(vfio_container_fd,
+						  mem_reg->host_user_addr,
+						  mem_reg->guest_phys_addr,
+						  mem_reg->size);
+	}
+
+error:
+	if (vhost_mem)

The NULL check is not necessary, free() takes care of doing it.

Quoted 9 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +5 -0

+		free(vhost_mem);
+
+	return rc;
+}
+
 static int
 sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
 		      const efx_bar_region_t *mem_ebrp)

drivers/vdpa/sfc/sfc_vdpa_ops.c +0 -0

...

Quoted 15 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +14 -0

+static uint64_t
+hva_to_gpa(int vid, uint64_t hva)
+{
+	struct rte_vhost_memory *vhost_mem = NULL;
+	struct rte_vhost_mem_region *mem_reg = NULL;
+	uint32_t i;
+	uint64_t gpa = 0;
+
+	if (rte_vhost_get_mem_table(vid, &vhost_mem) < 0)
+		goto error;
+
+	for (i = 0; i < vhost_mem->nregions; i++) {
+		mem_reg = &vhost_mem->regions[i];
+

drivers/vdpa/sfc/sfc_vdpa_ops.c line 91

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0

+		if (hva >= mem_reg->host_user_addr &&
+				hva < mem_reg->host_user_addr + mem_reg->size) {
+			gpa = (hva - mem_reg->host_user_addr) +
+				mem_reg->guest_phys_addr;
+			break;
+		}
+	}
+
+error:
+	if (vhost_mem)

Ditto.

Quoted 24 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +24 -0

+		free(vhost_mem);
+	return gpa;
+}
+
+static int
+sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
+{
+	int rc;
+	int *irq_fd_ptr;
+	int vfio_dev_fd;
+	uint32_t i, num_vring;
+	struct rte_vhost_vring vring;
+	struct vfio_irq_set *irq_set;
+	struct rte_pci_device *pci_dev;
+	char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
+	void *dev;
+
+	num_vring = rte_vhost_get_vring_num(ops_data->vid);
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = sizeof(irq_set_buf);

drivers/vdpa/sfc/sfc_vdpa_ops.c line 125

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0

+	irq_set->count = num_vring + 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+			 VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = 0;
+	irq_fd_ptr = (int *)&irq_set->data;
+	irq_fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = pci_dev->intr_handle.fd;
+
+	for (i = 0; i < num_vring; i++) {
+		rte_vhost_get_vhost_vring(ops_data->vid, i, &vring);

This function may fail (even if really unlikely that it happens), maybe better to avoid using non-initialized callfd value if it happened.

Quoted 14 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +14 -0

+		irq_fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd;
+	}
+
+	rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (rc) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "error enabling MSI-X interrupts: %s",
+			     strerror(errno));
+		return -1;
+	}
+
+	return 0;
+}
+

Vijay Kumar Srivastava

2021-09-03 13:22 UTC #30 lore raw

Re: [dpdk-dev] [PATCH 06/10] vdpa/sfc: add support for dev conf and dev close ops

Hi Maxime,

Quoted 13 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, August 30, 2021 5:06 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [PATCH 06/10] vdpa/sfc: add support for dev conf and dev close ops
Quoted 1 line · Vijay Srivastava
From: Vijay Kumar Srivastava <[email protected]>

[snip]

Quoted 22 lines

rte_vfio_container_dma_unmap(vfio_container_fd,

Quoted 21 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +21 -0

+						mem_reg->host_user_addr,
+						mem_reg->guest_phys_addr,
+						mem_reg->size);
+			if (rc < 0) {
+				sfc_vdpa_err(dev,
+					     "DMA unmap failed : %s",
+					     rte_strerror(rte_errno));
+				goto error;
+			}
+		}
+	}
+
+	free(vhost_mem);
+
+	return 0;
+
+failed_vfio_dma_map:
+	for (j = 0; j < i; j++) {
+		mem_reg = &vhost_mem->regions[j];
+		rc = rte_vfio_container_dma_unmap(vfio_container_fd,
+						  mem_reg->host_user_addr,

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c line 171
Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c +6 -0
+						  mem_reg->guest_phys_addr,
+						  mem_reg->size);
+	}
+
+error:
+	if (vhost_mem)
The NULL check is not necessary, free() takes care of doing it.

OK.

Quoted 69 lines

Quoted 9 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +5 -0
+		free(vhost_mem);
+
+	return rc;
+}
+
 static int
 sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
 		      const efx_bar_region_t *mem_ebrp) diff --git
a/drivers/vdpa/sfc/sfc_vdpa_ops.c b/drivers/vdpa/sfc/sfc_vdpa_ops.c

...

Quoted 18 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +16 -0

+static uint64_t
+hva_to_gpa(int vid, uint64_t hva)
+{
+	struct rte_vhost_memory *vhost_mem = NULL;
+	struct rte_vhost_mem_region *mem_reg = NULL;
+	uint32_t i;
+	uint64_t gpa = 0;
+
+	if (rte_vhost_get_mem_table(vid, &vhost_mem) < 0)
+		goto error;
+
+	for (i = 0; i < vhost_mem->nregions; i++) {
+		mem_reg = &vhost_mem->regions[i];
+
+		if (hva >= mem_reg->host_user_addr &&
+				hva < mem_reg->host_user_addr + mem_reg-

size) {

drivers/vdpa/sfc/sfc_vdpa_ops.c line 91

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +8 -0

+			gpa = (hva - mem_reg->host_user_addr) +
+				mem_reg->guest_phys_addr;
+			break;
+		}
+	}
+
+error:
+	if (vhost_mem)

Ditto.

Quoted 28 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +28 -0

+		free(vhost_mem);
+	return gpa;
+}
+
+static int
+sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data *ops_data) {
+	int rc;
+	int *irq_fd_ptr;
+	int vfio_dev_fd;
+	uint32_t i, num_vring;
+	struct rte_vhost_vring vring;
+	struct vfio_irq_set *irq_set;
+	struct rte_pci_device *pci_dev;
+	char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
+	void *dev;
+
+	num_vring = rte_vhost_get_vring_num(ops_data->vid);
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = sizeof(irq_set_buf);
+	irq_set->count = num_vring + 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+			 VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = 0;

Quoted 9 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c line 125
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa_ops.c +5 -0
+	irq_fd_ptr = (int *)&irq_set->data;
+	irq_fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = pci_dev->intr_handle.fd;
+
+	for (i = 0; i < num_vring; i++) {
+		rte_vhost_get_vhost_vring(ops_data->vid, i, &vring);
This function may fail (even if really unlikely that it happens), maybe better to avoid using non-initialized callfd value if it happened.

OK. I will do it in the v2.

[snip]

Vijay Srivastava

author

2021-07-07 08:26 UTC #31 lore raw

PATCH 7/10 vdpa/sfc: add support to get queue notify area info

From: Vijay Kumar Srivastava <[email protected]>

Implement the vDPA ops get_notify_area to get the notify area info of the queue.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 166 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |   2 +
 2 files changed, 162 insertions(+), 6 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +160 -6

index 4228044..a7b9085 100644
@@ -3,6 +3,8 @@
  * Copyright(c) 2020-2021 Xilinx, Inc.
  */
 
+#include <pthread.h>
+#include <unistd.h>
 #include <sys/ioctl.h>
 
 #include <rte_errno.h>

@@ -534,6 +536,67 @@
 	return 0;
 }
 
+static void *
+sfc_vdpa_notify_ctrl(void *arg)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	int vid;
+
+	ops_data = arg;
+	if (ops_data == NULL)
+		return NULL;
+
+	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+
+	vid = ops_data->vid;
+
+	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "vDPA (%s): Notifier could not get configured",
+			      ops_data->vdpa_dev->device->name);
+
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+
+	return NULL;
+}
+
+static int
+sfc_vdpa_setup_notify_ctrl(int vid)
+{
+	int ret;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}
+
+	ops_data->is_notify_thread_started = false;
+
+	/*
+	 * Use rte_vhost_host_notifier_ctrl in a thread to avoid
+	 * dead lock scenario when multiple VFs are used in single vdpa
+	 * application and multiple VFs are passed to a single VM.
+	 */
+	ret = pthread_create(&ops_data->notify_tid, NULL,
+			     sfc_vdpa_notify_ctrl, ops_data);
+	if (ret != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "failed to create notify_ctrl thread: %s",
+			     rte_strerror(ret));
+		return -1;
+	}
+	ops_data->is_notify_thread_started = true;
+
+	return 0;
+}
+
 static int
 sfc_vdpa_dev_config(int vid)
 {

@@ -567,18 +630,19 @@
 	if (rc != 0)
 		goto fail_vdpa_start;
 
-	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+	rc = sfc_vdpa_setup_notify_ctrl(vid);
+	if (rc != 0)
+		goto fail_vdpa_notify;
 
-	sfc_vdpa_log_init(ops_data->dev_handle, "vhost notifier ctrl");
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		sfc_vdpa_info(ops_data->dev_handle,
-			      "vDPA (%s): software relay for notify is used.",
-			      vdpa_dev->device->name);
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_vdpa_notify:
+	sfc_vdpa_stop(ops_data);
+
 fail_vdpa_start:
 	sfc_vdpa_close(ops_data);

@@ -591,6 +655,7 @@
 static int
 sfc_vdpa_dev_close(int vid)
 {
+	int ret;
 	struct rte_vdpa_device *vdpa_dev;
 	struct sfc_vdpa_ops_data *ops_data;

@@ -605,6 +670,23 @@
 	}
 
 	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+	if (ops_data->is_notify_thread_started == true) {
+		void *status;
+		ret = pthread_cancel(ops_data->notify_tid);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to cancel notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+
+		ret = pthread_join(ops_data->notify_tid, &status);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to join terminated notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+	}
+	ops_data->is_notify_thread_started = false;
 
 	sfc_vdpa_stop(ops_data);
 	sfc_vdpa_close(ops_data);

@@ -655,6 +737,77 @@
 	return vfio_dev_fd;
 }
 
+static int
+sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
+{
+	int ret;
+	efx_nic_t *nic;
+	int vfio_dev_fd;
+	efx_rc_t rc;
+	unsigned int bar_offset;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	struct vfio_region_info reg = { .argsz = sizeof(reg) };
+	const efx_nic_cfg_t *encp;
+	int max_vring_cnt;
+	int64_t len;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	max_vring_cnt =
+		(sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
+	encp = efx_nic_cfg_get(nic);
+
+	if (qid >= max_vring_cnt) {
+		sfc_vdpa_err(dev, "invalid qid : %d", qid);
+		return -1;
+	}
+
+	if (ops_data->vq_cxt[qid].enable != B_TRUE) {
+		sfc_vdpa_err(dev, "vq is not enabled");
+		return -1;
+	}
+
+	rc = efx_virtio_get_doorbell_offset(ops_data->vq_cxt[qid].vq,
+					    &bar_offset);
+	if (rc != 0) {
+		sfc_vdpa_err(dev, "failed to get doorbell offset: %s",
+			     rte_strerror(rc));
+		return rc;
+	}
+
+	reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
+	if (ret != 0) {
+		sfc_vdpa_err(dev, "could not get device region info: %s",
+			     strerror(errno));
+		return ret;
+	}
+
+	*offset = reg.offset + bar_offset;
+
+	len = (1U << encp->enc_vi_window_shift) / 2;
+	if (len >= sysconf(_SC_PAGESIZE))
+		*size = sysconf(_SC_PAGESIZE);
+	else
+		return -1;
+
+	sfc_vdpa_info(dev, "vDPA ops get_notify_area :: offset : 0x%" PRIx64,
+		      *offset);
+
+	return 0;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -664,6 +817,7 @@
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
 	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
+	.get_notify_area = sfc_vdpa_get_notify_area,
 };
 
 struct sfc_vdpa_ops_data *

drivers/vdpa/sfc/sfc_vdpa_ops.h +2 -0

index 8d553c5..f7523ef 100644
@@ -50,6 +50,8 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+	pthread_t			notify_tid;
+	bool				is_notify_thread_started;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;

Signature · 1 line

1.8.3.1

Maxime Coquelin

2021-08-30 13:22 UTC #32 lore raw

Re: [dpdk-dev] [PATCH 07/10] vdpa/sfc: add support to get queue notify area info

Quoted 12 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Implement the vDPA ops get_notify_area to get the notify area info of the queue.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 166 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |   2 +
 2 files changed, 162 insertions(+), 6 deletions(-)
drivers/vdpa/sfc/sfc_vdpa_ops.c +0 -0

...

Quoted 54 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +54 -0

+static int
+sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
+{
+	int ret;
+	efx_nic_t *nic;
+	int vfio_dev_fd;
+	efx_rc_t rc;
+	unsigned int bar_offset;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	struct vfio_region_info reg = { .argsz = sizeof(reg) };
+	const efx_nic_cfg_t *encp;
+	int max_vring_cnt;
+	int64_t len;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	max_vring_cnt =
+		(sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
+	encp = efx_nic_cfg_get(nic);
+
+	if (qid >= max_vring_cnt) {
+		sfc_vdpa_err(dev, "invalid qid : %d", qid);
+		return -1;
+	}
+
+	if (ops_data->vq_cxt[qid].enable != B_TRUE) {
+		sfc_vdpa_err(dev, "vq is not enabled");
+		return -1;
+	}
+
+	rc = efx_virtio_get_doorbell_offset(ops_data->vq_cxt[qid].vq,
+					    &bar_offset);
+	if (rc != 0) {
+		sfc_vdpa_err(dev, "failed to get doorbell offset: %s",
+			     rte_strerror(rc));
+		return rc;
+	}
+
+	reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
+	if (ret != 0) {
+		sfc_vdpa_err(dev, "could not get device region info: %s",
+			     strerror(errno));

drivers/vdpa/sfc/sfc_vdpa_ops.c line 803

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0

+		return ret;
+	}
+
+	*offset = reg.offset + bar_offset;
+
+	len = (1U << encp->enc_vi_window_shift) / 2;
+	if (len >= sysconf(_SC_PAGESIZE))
+		*size = sysconf(_SC_PAGESIZE);
+	else
+		return -1;

This error case may deserve an error messageto be logged.

Quoted 7 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +7 -0

+
+	sfc_vdpa_info(dev, "vDPA ops get_notify_area :: offset : 0x%" PRIx64,
+		      *offset);
+
+	return 0;
+}
+

Vijay Srivastava

author

2021-07-07 08:26 UTC #33 lore raw

PATCH 8/10 vdpa/sfc: add support for MAC filter config

From: Vijay Kumar Srivastava <[email protected]>

Add support for unicast and broadcast MAC filter configuration.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst        |   4 ++
 drivers/vdpa/sfc/meson.build       |   1 +
 drivers/vdpa/sfc/sfc_vdpa.c        |  32 +++++++++
 drivers/vdpa/sfc/sfc_vdpa.h        |  30 ++++++++
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 144 +++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c     |  10 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c    |  17 +++++
 7 files changed, 238 insertions(+)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c

doc/guides/vdpadevs/sfc.rst +4 -0

index abb5900..ae5ef42 100644
@@ -71,6 +71,10 @@ boolean parameters value.
   **vdpa** device will work as vdpa device and will be probed by vdpa/sfc driver.
   If this parameter is not specified then ef100 device will operate as network device.
 
+- ``mac`` [mac address]
+
+  Configures MAC address which would be used to setup MAC filters.
+
 
 Dynamic Logging Parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

drivers/vdpa/sfc/meson.build +1 -0

index aac7c51..f69cba9 100644
@@ -33,4 +33,5 @@ sources = files(
 	'sfc_vdpa_hw.c',
 	'sfc_vdpa_mcdi.c',
 	'sfc_vdpa_ops.c',
+	'sfc_vdpa_filter.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +32 -0

index ca13483..703aa9e 100644
@@ -8,7 +8,9 @@
 #include <sys/queue.h>
 
 #include <rte_common.h>
+#include <rte_devargs.h>
 #include <rte_errno.h>
+#include <rte_kvargs.h>
 #include <rte_string_fns.h>
 #include <rte_vfio.h>
 #include <rte_vhost.h>

@@ -202,6 +204,31 @@ struct sfc_vdpa_ops_data *
 	return (ret < 0) ? RTE_LOGTYPE_PMD : ret;
 }
 
+static int
+sfc_vdpa_kvargs_parse(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	struct rte_devargs *devargs = pci_dev->device.devargs;
+	/*
+	 * To get the device class a mandatory param 'class' is being
+	 * used so included SFC_EFX_KVARG_DEV_CLASS in the param list.
+	 */
+	const char **params = (const char *[]){
+		SFC_EFX_KVARG_DEV_CLASS,
+		SFC_VDPA_MAC_ADDR,
+		NULL,
+	};
+
+	if (devargs == NULL)
+		return 0;
+
+	sva->kvargs = rte_kvargs_parse(devargs->args, params);
+	if (sva->kvargs == NULL)
+		return -EINVAL;
+
+	return 0;
+}
+
 static struct rte_pci_id pci_id_sfc_vdpa_efx_map[] = {
 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) },
 	{ .vendor_id = 0, /* sentinel */ },

@@ -244,6 +271,10 @@ struct sfc_vdpa_ops_data *
 	if (ret != 0)
 		goto fail_set_log_prefix;
 
+	ret = sfc_vdpa_kvargs_parse(sva);
+	if (ret != 0)
+		goto fail_kvargs_parse;
+
 	sfc_vdpa_log_init(sva, "entry");
 
 	sfc_vdpa_adapter_lock_init(sva);

@@ -284,6 +315,7 @@ struct sfc_vdpa_ops_data *
 fail_vfio_setup:
 	sfc_vdpa_adapter_lock_fini(sva);
 
+fail_kvargs_parse:
 fail_set_log_prefix:
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +30 -0

index b103b0a..fd480ca 100644
@@ -17,8 +17,29 @@
 #include "sfc_vdpa_log.h"
 #include "sfc_vdpa_ops.h"
 
+#define SFC_VDPA_MAC_ADDR			"mac"
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
+/* Broadcast & Unicast MAC filters are supported */
+#define SFC_MAX_SUPPORTED_FILTERS		2
+
+/*
+ * Get function-local index of the associated VI from the
+ * virtqueue number. Queue 0 is reserved for MCDI
+ */
+#define SFC_VDPA_GET_VI_INDEX(vq_num) (((vq_num) / 2) + 1)
+
+enum sfc_vdpa_filter_type {
+	SFC_VDPA_BCAST_MAC_FILTER = 0,
+	SFC_VDPA_UCAST_MAC_FILTER = 1,
+	SFC_VDPA_FILTER_NTYPE
+};
+
+typedef struct sfc_vdpa_filter_s {
+	int				filter_cnt;
+	efx_filter_spec_t		spec[SFC_MAX_SUPPORTED_FILTERS];
+} sfc_vdpa_filter_t;
+
 /* Adapter private data */
 struct sfc_vdpa_adapter {
 	TAILQ_ENTRY(sfc_vdpa_adapter)	next;

@@ -32,6 +53,8 @@ struct sfc_vdpa_adapter {
 	struct rte_pci_device		*pdev;
 	struct rte_pci_addr		pci_addr;
 
+	struct rte_kvargs		*kvargs;
+
 	efx_family_t			family;
 	efx_nic_t			*nic;
 	rte_spinlock_t			nic_lock;

@@ -46,6 +69,8 @@ struct sfc_vdpa_adapter {
 	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
 	uint32_t			logtype_main;
 
+	sfc_vdpa_filter_t		filters;
+
 	int				vfio_group_fd;
 	int				vfio_dev_fd;
 	int				vfio_container_fd;

@@ -83,6 +108,11 @@ struct sfc_vdpa_ops_data *
 int
 sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *vdpa_data, bool do_map);
 
+int
+sfc_vdpa_filter_remove(struct sfc_vdpa_ops_data *ops_data);
+int
+sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data);
+
 static inline struct sfc_vdpa_adapter *
 sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
 {

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +144 -0

144 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_hw.c +10 -0

index 047bcc4..3a98c8c 100644
@@ -350,10 +350,20 @@
 		goto fail_virtio_init;
 	}
 
+	sfc_vdpa_log_init(sva, "init filter");
+	rc = efx_filter_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "filter init failed: %s", rte_strerror(rc));
+		goto fail_filter_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_filter_init:
+	efx_virtio_fini(enp);
+
 fail_virtio_init:
 	efx_nic_fini(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +17 -0

index a7b9085..f14b385 100644
@@ -423,6 +423,8 @@
 
 	sfc_vdpa_disable_vfio_intr(ops_data);
 
+	sfc_vdpa_filter_remove(ops_data);
+
 	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
 }

@@ -462,12 +464,27 @@
 			goto fail_vq_start;
 	}
 
+	ops_data->vq_count = i;
+
+	sfc_vdpa_log_init(ops_data->dev_handle,
+			  "configure MAC filters");
+	rc = sfc_vdpa_filter_config(ops_data);
+	if (rc != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "MAC filter config failed: %s",
+			     rte_strerror(rc));
+		goto fail_filter_cfg;
+	}
+
 	ops_data->state = SFC_VDPA_STATE_STARTED;
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_filter_cfg:
+	/* remove already created filters */
+	sfc_vdpa_filter_remove(ops_data);
 fail_vq_start:
 	/* stop already started virtqueues */
 	for (j = 0; j < i; j++)

Signature · 1 line

1.8.3.1

Maxime Coquelin

2021-08-30 13:47 UTC #34 lore raw

Re: [dpdk-dev] [PATCH 08/10] vdpa/sfc: add support for MAC filter config

Quoted 3 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Add support for unicast and broadcast MAC filter configuration.

I wonder whether we should not add an API to the vDPA framework for this MAC management, as this is something generic.

What do you think?

Quoted 12 lines

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst        |   4 ++
 drivers/vdpa/sfc/meson.build       |   1 +
 drivers/vdpa/sfc/sfc_vdpa.c        |  32 +++++++++
 drivers/vdpa/sfc/sfc_vdpa.h        |  30 ++++++++
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 144 +++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c     |  10 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c    |  17 +++++
 7 files changed, 238 insertions(+)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c

Vijay Kumar Srivastava

2021-09-03 13:20 UTC #35 lore raw

Re: [dpdk-dev] [PATCH 08/10] vdpa/sfc: add support for MAC filter config

Hi Maxime,

Quoted 11 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, August 30, 2021 7:18 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [PATCH 08/10] vdpa/sfc: add support for MAC filter config
On 7/6/21 6:44 PM, Vijay Srivastava wrote:

Quoted 8 lines

Quoted 3 lines
From: Vijay Kumar Srivastava <[email protected]>
Add support for unicast and broadcast MAC filter configuration.
I wonder whether we should not add an API to the vDPA framework for this MAC management, as this is something generic.
What do you think?

I think it's good idea to add an API for MAC configuration in vDPA framework.

Quoted 13 lines

Quoted 7 lines

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst        |   4 ++
 drivers/vdpa/sfc/meson.build       |   1 +
 drivers/vdpa/sfc/sfc_vdpa.c        |  32 +++++++++
 drivers/vdpa/sfc/sfc_vdpa.h        |  30 ++++++++
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 144

+++++++++++++++++++++++++++++++++++++

Quoted 5 lines

 drivers/vdpa/sfc/sfc_vdpa_hw.c     |  10 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c    |  17 +++++
 7 files changed, 238 insertions(+)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c

Vijay Srivastava

author

2021-07-07 08:26 UTC #36 lore raw

PATCH 9/10 vdpa/sfc: add support to set vring state

From: Vijay Kumar Srivastava <[email protected]>

Implements vDPA ops set_vring_state to configure vring state.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 54 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 4 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +50 -4

index f14b385..5a3b766 100644
@@ -716,11 +716,57 @@
 static int
 sfc_vdpa_set_vring_state(int vid, int vring, int state)
 {
-	RTE_SET_USED(vid);
-	RTE_SET_USED(vring);
-	RTE_SET_USED(state);
+	struct sfc_vdpa_ops_data *ops_data;
+	struct rte_vdpa_device *vdpa_dev;
+	efx_rc_t rc;
+	int vring_max;
+	void *dev;
 
-	return -1;
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	sfc_vdpa_info(dev,
+		      "vDPA ops set_vring_state: vid: %d, vring: %d, state:%d",
+		      vid, vring, state);
+
+	vring_max = (sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	if (vring < 0 || vring > vring_max) {
+		sfc_vdpa_err(dev, "received invalid vring id : %d to set state",
+			     vring);
+		return -1;
+	}
+
+	/*
+	 * Skip if device is not yet started. virtqueues state can be
+	 * changed once it is created and other configurations are done.
+	 */
+	if (ops_data->state != SFC_VDPA_STATE_STARTED)
+		return 0;
+
+	if (ops_data->vq_cxt[vring].enable == state)
+		return 0;
+
+	if (state == 0) {
+		rc = sfc_vdpa_virtq_stop(ops_data, vring);
+		if (rc != 0) {
+			sfc_vdpa_err(dev, "virtqueue stop failed: %s",
+				     rte_strerror(rc));
+		}
+	} else {
+		rc = sfc_vdpa_virtq_start(ops_data, vring);
+		if (rc != 0) {
+			sfc_vdpa_err(dev, "virtqueue start failed: %s",
+				     rte_strerror(rc));
+		}
+	}
+
+	return rc;
 }
 
 static int

Signature · 1 line

1.8.3.1

Maxime Coquelin

Reviewed-by on v1 9/10

2021-08-30 13:58 UTC #37 lore raw

Re: [dpdk-dev] [PATCH 09/10] vdpa/sfc: add support to set vring state

Quoted 9 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>
Implements vDPA ops set_vring_state to configure vring state.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 54 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 4 deletions(-)

Thanks,
Maxime

Vijay Srivastava

author

2021-07-07 08:26 UTC #38 lore raw

PATCH 10/10 vdpa/sfc: set a multicast filter during vDPA init

From: Vijay Kumar Srivastava <[email protected]>

Insert unknown multicast filter to allow IPv6 neighbor discovery

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa.h        |  3 ++-
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 19 +++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

drivers/vdpa/sfc/sfc_vdpa.h +2 -1

index fd480ca..68bf79a 100644
@@ -21,7 +21,7 @@
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
 /* Broadcast & Unicast MAC filters are supported */
-#define SFC_MAX_SUPPORTED_FILTERS		2
+#define SFC_MAX_SUPPORTED_FILTERS		3
 
 /*
  * Get function-local index of the associated VI from the

@@ -32,6 +32,7 @@
 enum sfc_vdpa_filter_type {
 	SFC_VDPA_BCAST_MAC_FILTER = 0,
 	SFC_VDPA_UCAST_MAC_FILTER = 1,
+	SFC_VDPA_MCAST_DST_FILTER = 2,
 	SFC_VDPA_FILTER_NTYPE
 };

drivers/vdpa/sfc/sfc_vdpa_filter.c +17 -2

index 03b6a5d..74204d3 100644
@@ -39,8 +39,12 @@
 	spec->efs_flags = EFX_FILTER_FLAG_RX;
 	spec->efs_dmaq_id = qid;
 
-	rc = efx_filter_spec_set_eth_local(spec, EFX_FILTER_SPEC_VID_UNSPEC,
-					   eth_addr);
+	if (eth_addr == NULL)
+		rc = efx_filter_spec_set_mc_def(spec);
+	else
+		rc = efx_filter_spec_set_eth_local(spec,
+						   EFX_FILTER_SPEC_VID_UNSPEC,
+						   eth_addr);
 	if (rc != 0)
 		return rc;

@@ -114,6 +118,17 @@ int sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data)
 	else
 		sva->filters.filter_cnt++;
 
+	sfc_vdpa_log_init(sva, "insert unknown mcast filter");
+	spec = &sva->filters.spec[SFC_VDPA_MCAST_DST_FILTER];
+
+	rc = sfc_vdpa_set_mac_filter(nic, spec, qid, NULL);
+	if (rc != 0)
+		sfc_vdpa_err(sva,
+			     "mcast filter insertion failed: %s",
+			     rte_strerror(rc));
+	else
+		sva->filters.filter_cnt++;
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return rc;

Signature · 1 line

1.8.3.1

Xia, Chenbo

2021-07-07 08:30 UTC #39 lore raw

Hi Vijay,

Quoted 26 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Wednesday, July 7, 2021 12:44 AM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Srivastava <[email protected]>
Subject: [PATCH 00/10] vdpa/sfc: introduce Xilinx vDPA driver
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
This Patch series includes following patches:
Vijay Kumar Srivastava (10):
vdpa/sfc: introduce Xilinx vDPA driver
vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init

[...]

Quoted 1 line

1.8.3.1

Did you register to mail-list? Your patch was not in mail list as the log says you are not a member of it. And @Yigit, Ferruh helped you release the mails this time. Please register to mail-list first if you have not, or there will be problems when you reply/send new versions.

Thanks,
Chenbo

Andrew Rybchenko

2021-07-07 11:09 UTC #40 lore raw

Quoted 6 lines · Vijay Srivastava

This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
This Patch series includes following patches:

Quoted 11 lines

Vijay Kumar Srivastava (10):
vdpa/sfc: introduce Xilinx vDPA driver
vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init

Series-acked-by Andrew Rybchenko <[email protected]>

Maxime Coquelin

2021-10-27 13:18 UTC #41 lore raw

In reply to Andrew Rybchenko

Hi Vijay,

-rc2 deadline is approaching, do you think v2 will be ready on time so that we can have your driver in v21.11?

Thanks,
Maxime

Quoted 51 lines · Vijay Srivastava

This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
This Patch series includes following patches:
Vijay Kumar Srivastava (10):
vdpa/sfc: introduce Xilinx vDPA driver vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init
  MAINTAINERS                            |   6 +
  doc/guides/rel_notes/release_21_08.rst |   5 +
  doc/guides/vdpadevs/features/sfc.ini   |  19 +
  doc/guides/vdpadevs/sfc.rst            | 107 ++++
  drivers/common/sfc_efx/efsys.h         |   2 +-
  drivers/common/sfc_efx/version.map     |  10 +
  drivers/vdpa/meson.build               |   1 +
  drivers/vdpa/sfc/meson.build           |  37 ++
  drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++
  drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++
  drivers/vdpa/sfc/sfc_vdpa_debug.h      |  21 +
  drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++
  drivers/vdpa/sfc/sfc_vdpa_hw.c         | 415 +++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa_log.h        |  80 +++
  drivers/vdpa/sfc/sfc_vdpa_mcdi.c       |  74 +++
  drivers/vdpa/sfc/sfc_vdpa_ops.c        | 942 +++++++++++++++++++++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa_ops.h        |  69 +++
  drivers/vdpa/sfc/version.map           |   3 +
  18 files changed, 2479 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/vdpadevs/features/sfc.ini create mode 100644 doc/guides/vdpadevs/sfc.rst create mode 100644 drivers/vdpa/sfc/meson.build create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h create mode 100644 drivers/vdpa/sfc/version.map

Andrew Rybchenko

2021-10-27 15:04 UTC #42 lore raw

Hi Maxime,

Quoted 4 lines · Maxime Coquelin

Hi Vijay,
-rc2 deadline is approaching, do you think v2 will be ready on time so that we can have your driver in v21.11?

Vijay is working on a it. We have v2, but it does not work after rebase on rc1. Hopefully Vijay will sort it out shortly.

Thanks,
Andrew.

Quoted 57 lines

Thanks,
Maxime
Quoted 52 lines · Vijay Srivastava
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
This Patch series includes following patches:
Vijay Kumar Srivastava (10):
   vdpa/sfc: introduce Xilinx vDPA driver vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init
MAINTAINERS                            |   6 + doc/guides/rel_notes/release_21_08.rst |   5 + doc/guides/vdpadevs/features/sfc.ini   | 19 + doc/guides/vdpadevs/sfc.rst            | 107 ++++ drivers/common/sfc_efx/efsys.h         |   2 +- drivers/common/sfc_efx/version.map     | 10 + drivers/vdpa/meson.build               |   1 + drivers/vdpa/sfc/meson.build           | 37 ++ drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++ drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++ drivers/vdpa/sfc/sfc_vdpa_debug.h      | 21 + drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++ drivers/vdpa/sfc/sfc_vdpa_hw.c         | 415 +++++++++++++++ drivers/vdpa/sfc/sfc_vdpa_log.h        | 80 +++ drivers/vdpa/sfc/sfc_vdpa_mcdi.c       | 74 +++ drivers/vdpa/sfc/sfc_vdpa_ops.c        | 942
+++++++++++++++++++++++++++++++++
drivers/vdpa/sfc/sfc_vdpa_ops.h        | 69 +++ drivers/vdpa/sfc/version.map           |   3 +
  18 files changed, 2479 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/vdpadevs/features/sfc.ini create mode 100644 doc/guides/vdpadevs/sfc.rst create mode 100644 drivers/vdpa/sfc/meson.build create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h create mode 100644 drivers/vdpa/sfc/version.map

Maxime Coquelin

2021-10-27 19:56 UTC #43 lore raw

Hi Andrew,

Quoted 10 lines · Andrew Rybchenko

Hi Maxime,
Quoted 4 lines · Maxime Coquelin
Hi Vijay,
-rc2 deadline is approaching, do you think v2 will be ready on time so that we can have your driver in v21.11?
Vijay is working on a it. We have v2, but it does not work after rebase on rc1. Hopefully Vijay will sort it out shortly.

Thanks for the update.

In case you are still stuck with this regression tomorrow end of day, feel free to send it anyway mentioning it is not working, so that we can have time for a v3 if necessary.

Maxime

Quoted 61 lines

Thanks,
Andrew.
Quoted 57 lines
Thanks,
Maxime
Quoted 52 lines · Vijay Srivastava
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
This Patch series includes following patches:
Vijay Kumar Srivastava (10):
   vdpa/sfc: introduce Xilinx vDPA driver vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init
MAINTAINERS                            |   6 + doc/guides/rel_notes/release_21_08.rst |   5 + doc/guides/vdpadevs/features/sfc.ini   | 19 + doc/guides/vdpadevs/sfc.rst            | 107 ++++ drivers/common/sfc_efx/efsys.h         |   2 +- drivers/common/sfc_efx/version.map     | 10 + drivers/vdpa/meson.build               |   1 + drivers/vdpa/sfc/meson.build           | 37 ++ drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++ drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++ drivers/vdpa/sfc/sfc_vdpa_debug.h      | 21 + drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++ drivers/vdpa/sfc/sfc_vdpa_hw.c         | 415 +++++++++++++++ drivers/vdpa/sfc/sfc_vdpa_log.h        | 80 +++ drivers/vdpa/sfc/sfc_vdpa_mcdi.c       | 74 +++ drivers/vdpa/sfc/sfc_vdpa_ops.c        | 942
+++++++++++++++++++++++++++++++++
drivers/vdpa/sfc/sfc_vdpa_ops.h        | 69 +++ drivers/vdpa/sfc/version.map           |   3 +
  18 files changed, 2479 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/vdpadevs/features/sfc.ini create mode 100644 doc/guides/vdpadevs/sfc.rst create mode 100644 drivers/vdpa/sfc/meson.build create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h create mode 100644 drivers/vdpa/sfc/version.map

Vijay Kumar Srivastava

2021-10-28 18:01 UTC #44 lore raw

HI Maxime,

With the rc1 code base, an issue https://bugs.dpdk.org/show_bug.cgi?id=862 was observed. Due to this vDPA app was not working properly. After fixing this issue (Fix needs to be submitted) vDPA app is working fine with the this patch series.

Regards,
Vijay

Quoted 79 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Wednesday, October 27, 2021 8:35 PM
To: Maxime Coquelin <[email protected]>; Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [PATCH 00/10] vdpa/sfc: introduce Xilinx vDPA driver
Hi Maxime,
Quoted 4 lines · Maxime Coquelin
Hi Vijay,
-rc2 deadline is approaching, do you think v2 will be ready on time so that we can have your driver in v21.11?
Vijay is working on a it. We have v2, but it does not work after rebase on rc1. Hopefully Vijay will sort it out shortly.
Thanks,
Andrew.
Quoted 57 lines
Thanks,
Maxime
Quoted 52 lines · Vijay Srivastava
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
This Patch series includes following patches:
Vijay Kumar Srivastava (10):
   vdpa/sfc: introduce Xilinx vDPA driver vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init
MAINTAINERS                            |   6 + doc/guides/rel_notes/release_21_08.rst |   5 + doc/guides/vdpadevs/features/sfc.ini   | 19 + doc/guides/vdpadevs/sfc.rst            | 107 ++++ drivers/common/sfc_efx/efsys.h         |   2 +- drivers/common/sfc_efx/version.map     | 10 + drivers/vdpa/meson.build               |   1 + drivers/vdpa/sfc/meson.build           | 37 ++ drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++ drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++ drivers/vdpa/sfc/sfc_vdpa_debug.h      | 21 + drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++ drivers/vdpa/sfc/sfc_vdpa_hw.c         | 415 +++++++++++++++ drivers/vdpa/sfc/sfc_vdpa_log.h        | 80 +++ drivers/vdpa/sfc/sfc_vdpa_mcdi.c       | 74 +++ drivers/vdpa/sfc/sfc_vdpa_ops.c        | 942
+++++++++++++++++++++++++++++++++
drivers/vdpa/sfc/sfc_vdpa_ops.h        | 69 +++ drivers/vdpa/sfc/version.map           |   3 +
  18 files changed, 2479 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/vdpadevs/features/sfc.ini create mode 100644 doc/guides/vdpadevs/sfc.rst create mode 100644 drivers/vdpa/sfc/meson.build create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h create mode 100644 drivers/vdpa/sfc/version.map

Vijay Srivastava

author

2021-10-28 07:56 UTC #45 lore raw

v2 cover vdpa/sfc: introduce Xilinx vDPA driver

This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.

 MAINTAINERS                            |   6 +
 doc/guides/rel_notes/release_21_11.rst |   5 +
 doc/guides/vdpadevs/features/sfc.ini   |  19 +
 doc/guides/vdpadevs/sfc.rst            | 107 ++++
 drivers/common/sfc_efx/efsys.h         |   2 +-
 drivers/common/sfc_efx/version.map     |  10 +
 drivers/vdpa/meson.build               |   1 +
 drivers/vdpa/sfc/meson.build           |  37 ++
 drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++
 drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++
 drivers/vdpa/sfc/sfc_vdpa_debug.h      |  21 +
 drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c         | 419 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h        |  59 ++
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c       |  74 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c        | 947 +++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h        |  69 +++
 drivers/vdpa/sfc/version.map           |   3 +
 18 files changed, 2467 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/vdpadevs/features/sfc.ini
 create mode 100644 doc/guides/vdpadevs/sfc.rst
 create mode 100644 drivers/vdpa/sfc/meson.build
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h
 create mode 100644 drivers/vdpa/sfc/version.map

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:56 UTC #46 lore raw

PATCH v2 1/10 vdpa/sfc: introduce Xilinx vDPA driver

From: Vijay Kumar Srivastava <[email protected]>

Add new vDPA PMD to support vDPA operation by Xilinx devices. This patch implements probe and remove functions.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Updated logging mcaros to remove redundant code.

 MAINTAINERS                            |   6 +
 doc/guides/rel_notes/release_21_11.rst |   5 +
 doc/guides/vdpadevs/features/sfc.ini   |   9 ++
 doc/guides/vdpadevs/sfc.rst            |  97 +++++++++++
 drivers/vdpa/meson.build               |   1 +
 drivers/vdpa/sfc/meson.build           |  33 ++++
 drivers/vdpa/sfc/sfc_vdpa.c            | 286 +++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa.h            |  40 +++++
 drivers/vdpa/sfc/sfc_vdpa_log.h        |  56 +++++++
 drivers/vdpa/sfc/version.map           |   3 +
 10 files changed, 536 insertions(+)
 create mode 100644 doc/guides/vdpadevs/features/sfc.ini
 create mode 100644 doc/guides/vdpadevs/sfc.rst
 create mode 100644 drivers/vdpa/sfc/meson.build
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h
 create mode 100644 drivers/vdpa/sfc/version.map

MAINTAINERS +6 -0

index be2c9b6..5d12c49 100644
@@ -1236,6 +1236,12 @@ F: drivers/vdpa/mlx5/
 F: doc/guides/vdpadevs/mlx5.rst
 F: doc/guides/vdpadevs/features/mlx5.ini
 
+Xilinx sfc vDPA
+M: Vijay Kumar Srivastava <[email protected]>
+F: drivers/vdpa/sfc/
+F: doc/guides/vdpadevs/sfc.rst
+F: doc/guides/vdpadevs/features/sfc.ini
+
 
 Eventdev Drivers
 ----------------

doc/guides/rel_notes/release_21_11.rst +5 -0

index 1ccac87..bd0a604 100644
@@ -305,6 +305,11 @@ New Features
     * Pcapng format with timestamps and meta-data.
     * Fixes packet capture with stripped VLAN tags.
 
+* **Add new vDPA PMD based on Xilinx devices.**
+
+  Added a new Xilinx vDPA  (``sfc_vdpa``) PMD.
+  See the :doc:`../vdpadevs/sfc` guide for more details on this driver.
+
 
 Removed Items
 -------------

doc/guides/vdpadevs/features/sfc.ini new file +9 -0

new file mode 100644
index 0000000..71b6158
@@ -0,0 +1,9 @@
+;
+; Supported features of the 'sfc' vDPA driver.
+;
+; Refer to default.ini for the full list of available driver features.
+;
+[Features]
+Linux			= Y
+x86-64			= Y
+Usage doc		= Y

doc/guides/vdpadevs/sfc.rst new file +97 -0

97 lines not on this page · raw message

drivers/vdpa/meson.build +1 -0

index f765fe3..77412c7 100644
@@ -8,6 +8,7 @@ endif
 drivers = [
         'ifc',
         'mlx5',
+        'sfc',
 ]
 std_deps = ['bus_pci', 'kvargs']
 std_deps += ['vhost']

drivers/vdpa/sfc/meson.build new file +33 -0

new file mode 100644
index 0000000..d916389
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright(c) 2020-2021 Xilinx, Inc.
+
+if (arch_subdir != 'x86' or not dpdk_conf.get('RTE_ARCH_64')) and (arch_subdir != 'arm' or not host_machine.cpu_family().startswith('aarch64'))
+	build = false
+	reason = 'only supported on x86_64 and aarch64'
+endif
+
+fmt_name = 'sfc_vdpa'
+extra_flags = []
+
+# Enable more warnings
+extra_flags += [
+	'-Wdisabled-optimization'
+]
+
+# Compiler and version dependent flags
+extra_flags += [
+	'-Waggregate-return',
+	'-Wbad-function-cast'
+]
+
+foreach flag: extra_flags
+	if cc.has_argument(flag)
+		cflags += flag
+	endif
+endforeach
+
+deps += ['common_sfc_efx', 'bus_pci']
+sources = files(
+	'sfc_vdpa.c',
+)

drivers/vdpa/sfc/sfc_vdpa.c new file +286 -0

286 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa.h new file +40 -0

new file mode 100644
index 0000000..3b77900
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_H
+#define _SFC_VDPA_H
+
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_bus_pci.h>
+
+#include "sfc_vdpa_log.h"
+
+/* Adapter private data */
+struct sfc_vdpa_adapter {
+	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	struct rte_pci_device		*pdev;
+	struct rte_pci_addr		pci_addr;
+
+	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
+	uint32_t			logtype_main;
+
+	int				vfio_group_fd;
+	int				vfio_dev_fd;
+	int				vfio_container_fd;
+	int				iommu_group_num;
+};
+
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str,
+			  uint32_t ll_default);
+
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+
+#endif  /* _SFC_VDPA_H */
+

drivers/vdpa/sfc/sfc_vdpa_log.h new file +56 -0

56 lines not on this page · raw message

drivers/vdpa/sfc/version.map new file +3 -0

new file mode 100644
index 0000000..4a76d1d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

Signature · 1 line

1.8.3.1

Xia, Chenbo

2021-10-28 08:21 UTC #47 lore raw

Hi Vijay,

Quoted 8 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Thursday, October 28, 2021 3:55 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v2 01/10] vdpa/sfc: introduce Xilinx vDPA driver

Quoted 7 lines

From: Vijay Kumar Srivastava <[email protected]>
Add new vDPA PMD to support vDPA operation by Xilinx devices. This patch implements probe and remove functions.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
---

I believe my comments (some grammar problems and 'use allowlist instead of whiltelist') and Stephen's comments for v1 are not addressed.

Please address them in v1 first.

Thanks,
Chenbo

Vijay Srivastava

author

2021-10-28 07:56 UTC #48 lore raw

PATCH v2 2/10 vdpa/sfc: add support for device initialization

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Used rte_memzone_reserve_aligned for mcdi buffer allocation.
* Freeing mcdi buff when DMA map fails.
* Fixed one typo.

 doc/guides/vdpadevs/sfc.rst       |   6 +
 drivers/vdpa/sfc/meson.build      |   3 +
 drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
 drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
 drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
 drivers/vdpa/sfc/sfc_vdpa_hw.c    | 327 ++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
 10 files changed, 670 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

doc/guides/vdpadevs/sfc.rst +6 -0

index 59f990b..abb5900 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types available for control:
   Matches a subset of per-port log types registered during runtime.
   A full name for a particular type may be obtained by appending a
   dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.
+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +3 -0

index d916389..aac7c51 100644
@@ -30,4 +30,7 @@ endforeach
 deps += ['common_sfc_efx', 'bus_pci']
 sources = files(
 	'sfc_vdpa.c',
+	'sfc_vdpa_hw.c',
+	'sfc_vdpa_mcdi.c',
+	'sfc_vdpa_ops.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +23 -0

index a6e1a9e..00fa94a 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
 		goto fail_vfio_setup;
 	}
 
+	sfc_vdpa_log_init(sva, "hw init");
+	if (sfc_vdpa_hw_init(sva) != 0) {
+		sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+		goto fail_hw_init;
+	}
+
+	sfc_vdpa_log_init(sva, "dev init");
+	sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+	if (sva->ops_data == NULL) {
+		sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+		goto fail_dev_init;
+	}
+
 	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
 	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
 
 	return 0;
 
+fail_dev_init:
+	sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+	sfc_vdpa_vfio_teardown(sva);
+
 fail_vfio_setup:
 fail_set_log_prefix:
 	rte_free(sva);

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
 	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
 
+	sfc_vdpa_device_fini(sva->ops_data);
+
+	sfc_vdpa_hw_fini(sva);
+
 	sfc_vdpa_vfio_teardown(sva);
 
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +48 -1

73 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +21 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug check
+ * in the driver only.
+ */
+#define SFC_VDPA_ASSERT(exp)			RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)			RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +327 -0

327 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_log.h +3 -0

index 858e5ee..4e7a84f 100644
@@ -21,6 +21,9 @@
 /** Name prefix for the per-device log type used to report basic information */
 #define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
 
+/** Device MCDI log type name prefix */
+#define SFC_VDPA_LOGTYPE_MCDI_STR	SFC_VDPA_LOGTYPE_PREFIX "mcdi"
+
 #define SFC_VDPA_LOG_PREFIX_MAX	32
 
 /* Log PMD message, automatically add prefix and \n */

drivers/vdpa/sfc/sfc_vdpa_mcdi.c new file +74 -0

74 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +129 -0

129 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +36 -0

new file mode 100644
index 0000000..817b302
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_OPS_H
+#define _SFC_VDPA_OPS_H
+
+#include <rte_vdpa.h>
+
+#define SFC_VDPA_MAX_QUEUE_PAIRS		1
+
+enum sfc_vdpa_context {
+	SFC_VDPA_AS_PF = 0,
+	SFC_VDPA_AS_VF
+};
+
+enum sfc_vdpa_state {
+	SFC_VDPA_STATE_UNINITIALIZED = 0,
+	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_NSTATES
+};
+
+struct sfc_vdpa_ops_data {
+	void				*dev_handle;
+	struct rte_vdpa_device		*vdpa_dev;
+	enum sfc_vdpa_context		vdpa_context;
+	enum sfc_vdpa_state		state;
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *adapter, enum sfc_vdpa_context context);
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data);
+
+#endif /* _SFC_VDPA_OPS_H */

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:56 UTC #49 lore raw

PATCH v2 3/10 vdpa/sfc: add support to get device and protocol features

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_feature and get_protocol_features. This patch retrieves device supported features and enables protocol features.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/features/sfc.ini | 10 ++++
 drivers/common/sfc_efx/efsys.h       |  2 +-
 drivers/common/sfc_efx/version.map   | 10 ++++
 drivers/vdpa/sfc/sfc_vdpa.c          | 20 ++++++++
 drivers/vdpa/sfc/sfc_vdpa.h          |  2 +
 drivers/vdpa/sfc/sfc_vdpa_hw.c       | 13 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c      | 91 ++++++++++++++++++++++++++++++++----
 drivers/vdpa/sfc/sfc_vdpa_ops.h      |  3 ++
 8 files changed, 142 insertions(+), 9 deletions(-)

doc/guides/vdpadevs/features/sfc.ini +10 -0

index 71b6158..700d061 100644
@@ -4,6 +4,16 @@
 ; Refer to default.ini for the full list of available driver features.
 ;
 [Features]
+csum			= Y
+guest csum		= Y
+host tso4		= Y
+host tso6		= Y
+version 1		= Y
+mrg rxbuf		= Y
+any layout		= Y
+in_order		= Y
+proto host notifier	= Y
+IOMMU platform		= Y
 Linux			= Y
 x86-64			= Y
 Usage doc		= Y

drivers/common/sfc_efx/efsys.h +1 -1

index d133d61..37ec6b9 100644
@@ -187,7 +187,7 @@
 
 #define EFSYS_OPT_MAE 1
 
-#define EFSYS_OPT_VIRTIO 0
+#define EFSYS_OPT_VIRTIO 1
 
 /* ID */

drivers/common/sfc_efx/version.map +10 -0

index 642a62e..ec86220 100644
@@ -247,6 +247,16 @@ INTERNAL {
 	efx_txq_nbufs;
 	efx_txq_size;
 
+	efx_virtio_fini;
+	efx_virtio_get_doorbell_offset;
+	efx_virtio_get_features;
+	efx_virtio_init;
+	efx_virtio_qcreate;
+	efx_virtio_qdestroy;
+	efx_virtio_qstart;
+	efx_virtio_qstop;
+	efx_virtio_verify_features;
+
 	sfc_efx_dev_class_get;
 	sfc_efx_family;

drivers/vdpa/sfc/sfc_vdpa.c +20 -0

index 00fa94a..4927698 100644
@@ -43,6 +43,26 @@ struct sfc_vdpa_adapter *
 	return found ? sva : NULL;
 }
 
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev)
+{
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (vdpa_dev == sva->ops_data->vdpa_dev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva->ops_data : NULL;
+}
+
 static int
 sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva)
 {

drivers/vdpa/sfc/sfc_vdpa.h +2 -0

index 046f25d..c10c3d3 100644
@@ -60,6 +60,8 @@ struct sfc_vdpa_adapter {
 
 struct sfc_vdpa_adapter *
 sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev);
 
 int
 sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);

drivers/vdpa/sfc/sfc_vdpa_hw.c +13 -0

index 7c256ff..7a67bd8 100644
@@ -278,10 +278,20 @@
 	if (rc != 0)
 		goto fail_estimate_rsrc_limits;
 
+	sfc_vdpa_log_init(sva, "init virtio");
+	rc = efx_virtio_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "virtio init failed: %s", rte_strerror(rc));
+		goto fail_virtio_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_virtio_init:
+	efx_nic_fini(enp);
+
 fail_estimate_rsrc_limits:
 fail_nic_reset:
 	efx_nic_unprobe(enp);

@@ -310,6 +320,9 @@
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_log_init(sva, "virtio fini");
+	efx_virtio_fini(enp);
+
 	sfc_vdpa_log_init(sva, "unprobe nic");
 	efx_nic_unprobe(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +83 -8

138 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h +3 -0

index 817b302..21cbb73 100644
@@ -26,6 +26,9 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+
+	uint64_t			dev_features;
+	uint64_t			drv_features;
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:56 UTC #50 lore raw

PATCH v2 4/10 vdpa/sfc: get device supported max queue count

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_queue_num to get the maximum number of queues supported by the device.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +13 -3

index 5750944..6c702e1 100644
@@ -31,10 +31,20 @@
 static int
 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
 {
-	RTE_SET_USED(vdpa_dev);
-	RTE_SET_USED(queue_num);
+	struct sfc_vdpa_ops_data *ops_data;
+	void *dev;
 
-	return -1;
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	*queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
+
+	sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %d",
+		      *queue_num);
+
+	return 0;
 }
 
 static int

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:57 UTC #51 lore raw

PATCH v2 5/10 vdpa/sfc: add support to get VFIO device fd

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_vfio_device_fd to get the VFIO device fd.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

drivers/vdpa/sfc/sfc_vdpa_ops.c +24 -0

index 6c702e1..5253adb 100644
@@ -145,6 +145,29 @@
 	return -1;
 }
 
+static int
+sfc_vdpa_get_vfio_device_fd(int vid)
+{
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	int vfio_dev_fd;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+
+	sfc_vdpa_info(dev, "vDPA ops get_vfio_device_fd :: vfio fd : %d",
+		      vfio_dev_fd);
+
+	return vfio_dev_fd;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -153,6 +176,7 @@
 	.dev_close = sfc_vdpa_dev_close,
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
+	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:57 UTC #52 lore raw

PATCH v2 6/10 vdpa/sfc: add support for dev conf and dev close ops

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops dev_conf and dev_close for DMA mapping, interrupt and virtqueue configurations.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Removed redundant null check while calling free().
* Added error handling for rte_vhost_get_vhost_vring().

 drivers/vdpa/sfc/sfc_vdpa.c     |   6 +
 drivers/vdpa/sfc/sfc_vdpa.h     |  43 ++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c  |  69 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 530 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |  28 +++
 5 files changed, 656 insertions(+), 20 deletions(-)

drivers/vdpa/sfc/sfc_vdpa.c +6 -0

index 4927698..9ffea59 100644
@@ -246,6 +246,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_adapter_lock_init(sva);
+
 	sfc_vdpa_log_init(sva, "vfio init");
 	if (sfc_vdpa_vfio_setup(sva) < 0) {
 		sfc_vdpa_err(sva, "failed to setup device %s", pci_dev->name);

@@ -280,6 +282,8 @@ struct sfc_vdpa_ops_data *
 	sfc_vdpa_vfio_teardown(sva);
 
 fail_vfio_setup:
+	sfc_vdpa_adapter_lock_fini(sva);
+
 fail_set_log_prefix:
 	rte_free(sva);

@@ -311,6 +315,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_vfio_teardown(sva);
 
+	sfc_vdpa_adapter_lock_fini(sva);
+
 	rte_free(sva);
 
 	return 0;

drivers/vdpa/sfc/sfc_vdpa.h +43 -0

53 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_hw.c +69 -0

81 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.c +510 ~~-20~~

566 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h +28 -0

index 21cbb73..8d553c5 100644
@@ -18,17 +18,45 @@ enum sfc_vdpa_context {
 enum sfc_vdpa_state {
 	SFC_VDPA_STATE_UNINITIALIZED = 0,
 	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_CONFIGURING,
+	SFC_VDPA_STATE_CONFIGURED,
+	SFC_VDPA_STATE_CLOSING,
+	SFC_VDPA_STATE_CLOSED,
+	SFC_VDPA_STATE_STARTING,
+	SFC_VDPA_STATE_STARTED,
+	SFC_VDPA_STATE_STOPPING,
 	SFC_VDPA_STATE_NSTATES
 };
 
+struct sfc_vdpa_vring_info {
+	uint64_t	desc;
+	uint64_t	avail;
+	uint64_t	used;
+	uint64_t	size;
+	uint16_t	last_avail_idx;
+	uint16_t	last_used_idx;
+};
+
+typedef struct sfc_vdpa_vq_context_s {
+	uint8_t				enable;
+	uint32_t			pidx;
+	uint32_t			cidx;
+	efx_virtio_vq_t			*vq;
+} sfc_vdpa_vq_context_t;
+
 struct sfc_vdpa_ops_data {
 	void				*dev_handle;
+	int				vid;
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;
+	uint64_t			req_features;
+
+	uint16_t			vq_count;
+	struct sfc_vdpa_vq_context_s	vq_cxt[SFC_VDPA_MAX_QUEUE_PAIRS * 2];
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:57 UTC #53 lore raw

PATCH v2 7/10 vdpa/sfc: add support to get queue notify area info

From: Vijay Kumar Srivastava <[email protected]>

Implement the vDPA ops get_notify_area to get the notify area info of the queue.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Added error log in sfc_vdpa_get_notify_area.

 drivers/vdpa/sfc/sfc_vdpa_ops.c | 168 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |   2 +
 2 files changed, 164 insertions(+), 6 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +162 -6

index de1c81a..774d73e 100644
@@ -3,6 +3,8 @@
  * Copyright(c) 2020-2021 Xilinx, Inc.
  */
 
+#include <pthread.h>
+#include <unistd.h>
 #include <sys/ioctl.h>
 
 #include <rte_errno.h>

@@ -537,6 +539,67 @@
 	return 0;
 }
 
+static void *
+sfc_vdpa_notify_ctrl(void *arg)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	int vid;
+
+	ops_data = arg;
+	if (ops_data == NULL)
+		return NULL;
+
+	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+
+	vid = ops_data->vid;
+
+	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "vDPA (%s): Notifier could not get configured",
+			      ops_data->vdpa_dev->device->name);
+
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+
+	return NULL;
+}
+
+static int
+sfc_vdpa_setup_notify_ctrl(int vid)
+{
+	int ret;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}
+
+	ops_data->is_notify_thread_started = false;
+
+	/*
+	 * Use rte_vhost_host_notifier_ctrl in a thread to avoid
+	 * dead lock scenario when multiple VFs are used in single vdpa
+	 * application and multiple VFs are passed to a single VM.
+	 */
+	ret = pthread_create(&ops_data->notify_tid, NULL,
+			     sfc_vdpa_notify_ctrl, ops_data);
+	if (ret != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "failed to create notify_ctrl thread: %s",
+			     rte_strerror(ret));
+		return -1;
+	}
+	ops_data->is_notify_thread_started = true;
+
+	return 0;
+}
+
 static int
 sfc_vdpa_dev_config(int vid)
 {

@@ -570,18 +633,19 @@
 	if (rc != 0)
 		goto fail_vdpa_start;
 
-	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+	rc = sfc_vdpa_setup_notify_ctrl(vid);
+	if (rc != 0)
+		goto fail_vdpa_notify;
 
-	sfc_vdpa_log_init(ops_data->dev_handle, "vhost notifier ctrl");
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		sfc_vdpa_info(ops_data->dev_handle,
-			      "vDPA (%s): software relay for notify is used.",
-			      vdpa_dev->device->name);
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_vdpa_notify:
+	sfc_vdpa_stop(ops_data);
+
 fail_vdpa_start:
 	sfc_vdpa_close(ops_data);

@@ -594,6 +658,7 @@
 static int
 sfc_vdpa_dev_close(int vid)
 {
+	int ret;
 	struct rte_vdpa_device *vdpa_dev;
 	struct sfc_vdpa_ops_data *ops_data;

@@ -608,6 +673,23 @@
 	}
 
 	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+	if (ops_data->is_notify_thread_started == true) {
+		void *status;
+		ret = pthread_cancel(ops_data->notify_tid);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to cancel notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+
+		ret = pthread_join(ops_data->notify_tid, &status);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to join terminated notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+	}
+	ops_data->is_notify_thread_started = false;
 
 	sfc_vdpa_stop(ops_data);
 	sfc_vdpa_close(ops_data);

@@ -658,6 +740,79 @@
 	return vfio_dev_fd;
 }
 
+static int
+sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
+{
+	int ret;
+	efx_nic_t *nic;
+	int vfio_dev_fd;
+	efx_rc_t rc;
+	unsigned int bar_offset;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	struct vfio_region_info reg = { .argsz = sizeof(reg) };
+	const efx_nic_cfg_t *encp;
+	int max_vring_cnt;
+	int64_t len;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	max_vring_cnt =
+		(sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
+	encp = efx_nic_cfg_get(nic);
+
+	if (qid >= max_vring_cnt) {
+		sfc_vdpa_err(dev, "invalid qid : %d", qid);
+		return -1;
+	}
+
+	if (ops_data->vq_cxt[qid].enable != B_TRUE) {
+		sfc_vdpa_err(dev, "vq is not enabled");
+		return -1;
+	}
+
+	rc = efx_virtio_get_doorbell_offset(ops_data->vq_cxt[qid].vq,
+					    &bar_offset);
+	if (rc != 0) {
+		sfc_vdpa_err(dev, "failed to get doorbell offset: %s",
+			     rte_strerror(rc));
+		return rc;
+	}
+
+	reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
+	if (ret != 0) {
+		sfc_vdpa_err(dev, "could not get device region info: %s",
+			     strerror(errno));
+		return ret;
+	}
+
+	*offset = reg.offset + bar_offset;
+
+	len = (1U << encp->enc_vi_window_shift) / 2;
+	if (len >= sysconf(_SC_PAGESIZE)) {
+		*size = sysconf(_SC_PAGESIZE);
+	} else {
+		sfc_vdpa_err(dev, "invalid VI window size : 0x%" PRIx64, len);
+		return -1;
+	}
+
+	sfc_vdpa_info(dev, "vDPA ops get_notify_area :: offset : 0x%" PRIx64,
+		      *offset);
+
+	return 0;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -667,6 +822,7 @@
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
 	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
+	.get_notify_area = sfc_vdpa_get_notify_area,
 };
 
 struct sfc_vdpa_ops_data *

drivers/vdpa/sfc/sfc_vdpa_ops.h +2 -0

index 8d553c5..f7523ef 100644
@@ -50,6 +50,8 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+	pthread_t			notify_tid;
+	bool				is_notify_thread_started;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:57 UTC #54 lore raw

PATCH v2 8/10 vdpa/sfc: add support for MAC filter config

From: Vijay Kumar Srivastava <[email protected]>

Add support for unicast and broadcast MAC filter configuration.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst        |   4 ++
 drivers/vdpa/sfc/meson.build       |   1 +
 drivers/vdpa/sfc/sfc_vdpa.c        |  32 +++++++++
 drivers/vdpa/sfc/sfc_vdpa.h        |  30 ++++++++
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 144 +++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c     |  10 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c    |  17 +++++
 7 files changed, 238 insertions(+)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c

doc/guides/vdpadevs/sfc.rst +4 -0

index abb5900..ae5ef42 100644
@@ -71,6 +71,10 @@ boolean parameters value.
   **vdpa** device will work as vdpa device and will be probed by vdpa/sfc driver.
   If this parameter is not specified then ef100 device will operate as network device.
 
+- ``mac`` [mac address]
+
+  Configures MAC address which would be used to setup MAC filters.
+
 
 Dynamic Logging Parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

drivers/vdpa/sfc/meson.build +1 -0

index aac7c51..f69cba9 100644
@@ -33,4 +33,5 @@ sources = files(
 	'sfc_vdpa_hw.c',
 	'sfc_vdpa_mcdi.c',
 	'sfc_vdpa_ops.c',
+	'sfc_vdpa_filter.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +32 -0

index 9ffea59..012616b 100644
@@ -8,7 +8,9 @@
 #include <sys/queue.h>
 
 #include <rte_common.h>
+#include <rte_devargs.h>
 #include <rte_errno.h>
+#include <rte_kvargs.h>
 #include <rte_string_fns.h>
 #include <rte_vfio.h>
 #include <rte_vhost.h>

@@ -202,6 +204,31 @@ struct sfc_vdpa_ops_data *
 	return (ret < 0) ? RTE_LOGTYPE_PMD : ret;
 }
 
+static int
+sfc_vdpa_kvargs_parse(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	struct rte_devargs *devargs = pci_dev->device.devargs;
+	/*
+	 * To get the device class a mandatory param 'class' is being
+	 * used so included SFC_EFX_KVARG_DEV_CLASS in the param list.
+	 */
+	const char **params = (const char *[]){
+		RTE_DEVARGS_KEY_CLASS,
+		SFC_VDPA_MAC_ADDR,
+		NULL,
+	};
+
+	if (devargs == NULL)
+		return 0;
+
+	sva->kvargs = rte_kvargs_parse(devargs->args, params);
+	if (sva->kvargs == NULL)
+		return -EINVAL;
+
+	return 0;
+}
+
 static struct rte_pci_id pci_id_sfc_vdpa_efx_map[] = {
 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) },
 	{ .vendor_id = 0, /* sentinel */ },

@@ -244,6 +271,10 @@ struct sfc_vdpa_ops_data *
 	if (ret != 0)
 		goto fail_set_log_prefix;
 
+	ret = sfc_vdpa_kvargs_parse(sva);
+	if (ret != 0)
+		goto fail_kvargs_parse;
+
 	sfc_vdpa_log_init(sva, "entry");
 
 	sfc_vdpa_adapter_lock_init(sva);

@@ -284,6 +315,7 @@ struct sfc_vdpa_ops_data *
 fail_vfio_setup:
 	sfc_vdpa_adapter_lock_fini(sva);
 
+fail_kvargs_parse:
 fail_set_log_prefix:
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +30 -0

index 1bf96e7..dbd099f 100644
@@ -17,8 +17,29 @@
 #include "sfc_vdpa_log.h"
 #include "sfc_vdpa_ops.h"
 
+#define SFC_VDPA_MAC_ADDR			"mac"
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
+/* Broadcast & Unicast MAC filters are supported */
+#define SFC_MAX_SUPPORTED_FILTERS		2
+
+/*
+ * Get function-local index of the associated VI from the
+ * virtqueue number. Queue 0 is reserved for MCDI
+ */
+#define SFC_VDPA_GET_VI_INDEX(vq_num) (((vq_num) / 2) + 1)
+
+enum sfc_vdpa_filter_type {
+	SFC_VDPA_BCAST_MAC_FILTER = 0,
+	SFC_VDPA_UCAST_MAC_FILTER = 1,
+	SFC_VDPA_FILTER_NTYPE
+};
+
+typedef struct sfc_vdpa_filter_s {
+	int				filter_cnt;
+	efx_filter_spec_t		spec[SFC_MAX_SUPPORTED_FILTERS];
+} sfc_vdpa_filter_t;
+
 /* Adapter private data */
 struct sfc_vdpa_adapter {
 	TAILQ_ENTRY(sfc_vdpa_adapter)	next;

@@ -32,6 +53,8 @@ struct sfc_vdpa_adapter {
 	struct rte_pci_device		*pdev;
 	struct rte_pci_addr		pci_addr;
 
+	struct rte_kvargs		*kvargs;
+
 	efx_family_t			family;
 	efx_nic_t			*nic;
 	rte_spinlock_t			nic_lock;

@@ -46,6 +69,8 @@ struct sfc_vdpa_adapter {
 	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
 	uint32_t			logtype_main;
 
+	sfc_vdpa_filter_t		filters;
+
 	int				vfio_group_fd;
 	int				vfio_dev_fd;
 	int				vfio_container_fd;

@@ -83,6 +108,11 @@ struct sfc_vdpa_ops_data *
 int
 sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *vdpa_data, bool do_map);
 
+int
+sfc_vdpa_filter_remove(struct sfc_vdpa_ops_data *ops_data);
+int
+sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data);
+
 static inline struct sfc_vdpa_adapter *
 sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
 {

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +144 -0

144 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_hw.c +10 -0

index b473708..5307b03 100644
@@ -354,10 +354,20 @@
 		goto fail_virtio_init;
 	}
 
+	sfc_vdpa_log_init(sva, "init filter");
+	rc = efx_filter_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "filter init failed: %s", rte_strerror(rc));
+		goto fail_filter_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_filter_init:
+	efx_virtio_fini(enp);
+
 fail_virtio_init:
 	efx_nic_fini(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +17 -0

index 774d73e..8551b65 100644
@@ -426,6 +426,8 @@
 
 	sfc_vdpa_disable_vfio_intr(ops_data);
 
+	sfc_vdpa_filter_remove(ops_data);
+
 	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
 }

@@ -465,12 +467,27 @@
 			goto fail_vq_start;
 	}
 
+	ops_data->vq_count = i;
+
+	sfc_vdpa_log_init(ops_data->dev_handle,
+			  "configure MAC filters");
+	rc = sfc_vdpa_filter_config(ops_data);
+	if (rc != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "MAC filter config failed: %s",
+			     rte_strerror(rc));
+		goto fail_filter_cfg;
+	}
+
 	ops_data->state = SFC_VDPA_STATE_STARTED;
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_filter_cfg:
+	/* remove already created filters */
+	sfc_vdpa_filter_remove(ops_data);
 fail_vq_start:
 	/* stop already started virtqueues */
 	for (j = 0; j < i; j++)

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:57 UTC #55 lore raw

PATCH v2 9/10 vdpa/sfc: add support to set vring state

From: Vijay Kumar Srivastava <[email protected]>

Implements vDPA ops set_vring_state to configure vring state.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 54 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 4 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +50 -4

index 8551b65..3430643 100644
@@ -719,11 +719,57 @@
 static int
 sfc_vdpa_set_vring_state(int vid, int vring, int state)
 {
-	RTE_SET_USED(vid);
-	RTE_SET_USED(vring);
-	RTE_SET_USED(state);
+	struct sfc_vdpa_ops_data *ops_data;
+	struct rte_vdpa_device *vdpa_dev;
+	efx_rc_t rc;
+	int vring_max;
+	void *dev;
 
-	return -1;
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	sfc_vdpa_info(dev,
+		      "vDPA ops set_vring_state: vid: %d, vring: %d, state:%d",
+		      vid, vring, state);
+
+	vring_max = (sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	if (vring < 0 || vring > vring_max) {
+		sfc_vdpa_err(dev, "received invalid vring id : %d to set state",
+			     vring);
+		return -1;
+	}
+
+	/*
+	 * Skip if device is not yet started. virtqueues state can be
+	 * changed once it is created and other configurations are done.
+	 */
+	if (ops_data->state != SFC_VDPA_STATE_STARTED)
+		return 0;
+
+	if (ops_data->vq_cxt[vring].enable == state)
+		return 0;
+
+	if (state == 0) {
+		rc = sfc_vdpa_virtq_stop(ops_data, vring);
+		if (rc != 0) {
+			sfc_vdpa_err(dev, "virtqueue stop failed: %s",
+				     rte_strerror(rc));
+		}
+	} else {
+		rc = sfc_vdpa_virtq_start(ops_data, vring);
+		if (rc != 0) {
+			sfc_vdpa_err(dev, "virtqueue start failed: %s",
+				     rte_strerror(rc));
+		}
+	}
+
+	return rc;
 }
 
 static int

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-28 07:57 UTC #56 lore raw

PATCH v2 10/10 vdpa/sfc: set a multicast filter during vDPA init

From: Vijay Kumar Srivastava <[email protected]>

Insert unknown multicast filter to allow IPv6 neighbor discovery

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa.h        |  3 ++-
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 19 +++++++++++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

drivers/vdpa/sfc/sfc_vdpa.h +2 -1

index dbd099f..bedc76c 100644
@@ -21,7 +21,7 @@
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
 /* Broadcast & Unicast MAC filters are supported */
-#define SFC_MAX_SUPPORTED_FILTERS		2
+#define SFC_MAX_SUPPORTED_FILTERS		3
 
 /*
  * Get function-local index of the associated VI from the

@@ -32,6 +32,7 @@
 enum sfc_vdpa_filter_type {
 	SFC_VDPA_BCAST_MAC_FILTER = 0,
 	SFC_VDPA_UCAST_MAC_FILTER = 1,
+	SFC_VDPA_MCAST_DST_FILTER = 2,
 	SFC_VDPA_FILTER_NTYPE
 };

drivers/vdpa/sfc/sfc_vdpa_filter.c +17 -2

index 03b6a5d..74204d3 100644
@@ -39,8 +39,12 @@
 	spec->efs_flags = EFX_FILTER_FLAG_RX;
 	spec->efs_dmaq_id = qid;
 
-	rc = efx_filter_spec_set_eth_local(spec, EFX_FILTER_SPEC_VID_UNSPEC,
-					   eth_addr);
+	if (eth_addr == NULL)
+		rc = efx_filter_spec_set_mc_def(spec);
+	else
+		rc = efx_filter_spec_set_eth_local(spec,
+						   EFX_FILTER_SPEC_VID_UNSPEC,
+						   eth_addr);
 	if (rc != 0)
 		return rc;

@@ -114,6 +118,17 @@ int sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data)
 	else
 		sva->filters.filter_cnt++;
 
+	sfc_vdpa_log_init(sva, "insert unknown mcast filter");
+	spec = &sva->filters.spec[SFC_VDPA_MCAST_DST_FILTER];
+
+	rc = sfc_vdpa_set_mac_filter(nic, spec, qid, NULL);
+	if (rc != 0)
+		sfc_vdpa_err(sva,
+			     "mcast filter insertion failed: %s",
+			     rte_strerror(rc));
+	else
+		sva->filters.filter_cnt++;
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return rc;

Signature · 1 line

1.8.3.1

Xia, Chenbo

2021-10-28 08:08 UTC #57 lore raw

Quoted 24 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Thursday, October 28, 2021 3:55 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Srivastava <[email protected]>
Subject: [PATCH v2 00/10] vdpa/sfc: introduce Xilinx vDPA driver
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
Vijay Kumar Srivastava (10):
vdpa/sfc: introduce Xilinx vDPA driver
vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init

Quoted 1 line

1.8.3.1

I remember Maxime added R-by in some patches? And Andrew also acked the whole series?

If yes, you should add those tags in new version.

Thanks,
Chenbo

Maxime Coquelin

2021-10-28 08:11 UTC #58 lore raw

Quoted 24 lines · Chenbo

Quoted 24 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Thursday, October 28, 2021 3:55 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Srivastava <[email protected]>
Subject: [PATCH v2 00/10] vdpa/sfc: introduce Xilinx vDPA driver
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
Vijay Kumar Srivastava (10):
vdpa/sfc: introduce Xilinx vDPA driver vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init

Quoted 6 lines

Quoted 1 line
1.8.3.1
I remember Maxime added R-by in some patches? And Andrew also acked the whole series?
If yes, you should add those tags in new version.

Agree with Chenbo, but I'd like to add that it should be done only on patches that were not significantly reworked.

Maxime

Quoted 3 lines

Thanks,
Chenbo

Maxime Coquelin

2021-10-28 14:35 UTC #59 lore raw

Quoted 35 lines · Vijay Srivastava

This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.
Vijay Kumar Srivastava (10):
vdpa/sfc: introduce Xilinx vDPA driver vdpa/sfc: add support for device initialization vdpa/sfc: add support to get device and protocol features vdpa/sfc: get device supported max queue count vdpa/sfc: add support to get VFIO device fd vdpa/sfc: add support for dev conf and dev close ops vdpa/sfc: add support to get queue notify area info vdpa/sfc: add support for MAC filter config vdpa/sfc: add support to set vring state vdpa/sfc: set a multicast filter during vDPA init
  MAINTAINERS                            |   6 +
  doc/guides/rel_notes/release_21_11.rst |   5 +
  doc/guides/vdpadevs/features/sfc.ini   |  19 +
  doc/guides/vdpadevs/sfc.rst            | 107 ++++
  drivers/common/sfc_efx/efsys.h         |   2 +-
  drivers/common/sfc_efx/version.map     |  10 +
  drivers/vdpa/meson.build               |   1 +
  drivers/vdpa/sfc/meson.build           |  37 ++
  drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++
  drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++
  drivers/vdpa/sfc/sfc_vdpa_debug.h      |  21 +
  drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++
  drivers/vdpa/sfc/sfc_vdpa_hw.c         | 419 +++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa_log.h        |  59 ++
  drivers/vdpa/sfc/sfc_vdpa_mcdi.c       |  74 +++
  drivers/vdpa/sfc/sfc_vdpa_ops.c        | 947 +++++++++++++++++++++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa_ops.h        |  69 +++
  drivers/vdpa/sfc/version.map           |   3 +
  18 files changed, 2467 insertions(+), 1 deletion(-)

Quoted 14 lines

create mode 100644 doc/guides/vdpadevs/features/sfc.ini create mode 100644 doc/guides/vdpadevs/sfc.rst create mode 100644 drivers/vdpa/sfc/meson.build create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h create mode 100644 drivers/vdpa/sfc/version.map

Please also note that doc build is failing:
http://mails.dpdk.org/archives/test-report/2021-October/235652.html

"
Warning, treated as error:
/home/runner/work/dpdk/dpdk/doc/guides/vdpadevs/sfc.rst:document isn't included in any toctree "

Maxime

Vijay Kumar Srivastava

2021-10-28 18:03 UTC #60 lore raw

Hi Maxime,

Quoted 14 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Thursday, October 28, 2021 8:05 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [PATCH v2 00/10] vdpa/sfc: introduce Xilinx vDPA driver
Quoted 3 lines · Vijay Srivastava
This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.

[SNIP]

Quoted 11 lines

Quoted 3 lines
create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h create mode 100644 drivers/vdpa/sfc/version.map
Please also note that doc build is failing:
http://mails.dpdk.org/archives/test-report/2021-October/235652.html
"
Warning, treated as error:
/home/runner/work/dpdk/dpdk/doc/guides/vdpadevs/sfc.rst:document isn't included in any toctree "

Now including it in the toctree of vdpadevs/index.rst.

Regards,
Vijay

Vijay Srivastava

author

2021-10-29 14:48 UTC #61 lore raw

v3 cover vdpa/sfc: introduce Xilinx vDPA driver

This patch series introduces vDPA driver for Xilinx devices. The Xilinx vDPA (vhost data path acceleration) provides support for the Xilinx SN1022 SmartNICs.

 MAINTAINERS                            |   6 +
 doc/guides/rel_notes/release_21_11.rst |   5 +
 doc/guides/vdpadevs/features/sfc.ini   |  19 +
 doc/guides/vdpadevs/index.rst          |   2 +
 doc/guides/vdpadevs/sfc.rst            | 107 ++++
 drivers/common/sfc_efx/efsys.h         |   2 +-
 drivers/common/sfc_efx/version.map     |  10 +
 drivers/vdpa/meson.build               |   1 +
 drivers/vdpa/sfc/meson.build           |  26 +
 drivers/vdpa/sfc/sfc_vdpa.c            | 367 +++++++++++++
 drivers/vdpa/sfc/sfc_vdpa.h            | 163 ++++++
 drivers/vdpa/sfc/sfc_vdpa_debug.h      |  21 +
 drivers/vdpa/sfc/sfc_vdpa_filter.c     | 159 ++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c         | 419 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h        |  59 ++
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c       |  74 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c        | 947 +++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h        |  69 +++
 drivers/vdpa/sfc/version.map           |   3 +
 19 files changed, 2458 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/vdpadevs/features/sfc.ini
 create mode 100644 doc/guides/vdpadevs/sfc.rst
 create mode 100644 drivers/vdpa/sfc/meson.build
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h
 create mode 100644 drivers/vdpa/sfc/version.map

Signature · 1 line

1.8.3.1

Vijay Srivastava

author

2021-10-29 14:48 UTC #62 lore raw

PATCH v3 1/10 vdpa/sfc: introduce Xilinx vDPA driver

From: Vijay Kumar Srivastava <[email protected]>

Add new vDPA PMD to support vDPA operations of Xilinx devices. This patch implements probe and remove functions.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Updated logging macros to remove redundant code.

v3:
* Replaced depreciated whitelist with allowlist.
* Text corrections in the sfc.rst and commit msg.
* Added sfc in the toctree of doc/guides/vdpadevs/index.rst.
* Removed extra compiler flags.

 MAINTAINERS                            |   6 +
 doc/guides/rel_notes/release_21_11.rst |   5 +
 doc/guides/vdpadevs/features/sfc.ini   |   9 ++
 doc/guides/vdpadevs/index.rst          |   2 +
 doc/guides/vdpadevs/sfc.rst            |  97 +++++++++++
 drivers/vdpa/meson.build               |   1 +
 drivers/vdpa/sfc/meson.build           |  22 +++
 drivers/vdpa/sfc/sfc_vdpa.c            | 286 +++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa.h            |  40 +++++
 drivers/vdpa/sfc/sfc_vdpa_log.h        |  56 +++++++
 drivers/vdpa/sfc/version.map           |   3 +
 11 files changed, 527 insertions(+)
 create mode 100644 doc/guides/vdpadevs/features/sfc.ini
 create mode 100644 doc/guides/vdpadevs/sfc.rst
 create mode 100644 drivers/vdpa/sfc/meson.build
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h
 create mode 100644 drivers/vdpa/sfc/version.map

MAINTAINERS +6 -0

index be2c9b6..5d12c49 100644
@@ -1236,6 +1236,12 @@ F: drivers/vdpa/mlx5/
 F: doc/guides/vdpadevs/mlx5.rst
 F: doc/guides/vdpadevs/features/mlx5.ini
 
+Xilinx sfc vDPA
+M: Vijay Kumar Srivastava <[email protected]>
+F: drivers/vdpa/sfc/
+F: doc/guides/vdpadevs/sfc.rst
+F: doc/guides/vdpadevs/features/sfc.ini
+
 
 Eventdev Drivers
 ----------------

doc/guides/rel_notes/release_21_11.rst +5 -0

index 1ccac87..bd0a604 100644
@@ -305,6 +305,11 @@ New Features
     * Pcapng format with timestamps and meta-data.
     * Fixes packet capture with stripped VLAN tags.
 
+* **Add new vDPA PMD based on Xilinx devices.**
+
+  Added a new Xilinx vDPA  (``sfc_vdpa``) PMD.
+  See the :doc:`../vdpadevs/sfc` guide for more details on this driver.
+
 
 Removed Items
 -------------

doc/guides/vdpadevs/features/sfc.ini new file +9 -0

new file mode 100644
index 0000000..71b6158
@@ -0,0 +1,9 @@
+;
+; Supported features of the 'sfc' vDPA driver.
+;
+; Refer to default.ini for the full list of available driver features.
+;
+[Features]
+Linux			= Y
+x86-64			= Y
+Usage doc		= Y

doc/guides/vdpadevs/index.rst +2 -0

index 1a13efe..f1a946e 100644
@@ -14,3 +14,5 @@ which can be used from an application through vhost API.
     features_overview
     ifc
     mlx5
+    sfc
+

doc/guides/vdpadevs/sfc.rst new file +97 -0

97 lines not on this page · raw message

drivers/vdpa/meson.build +1 -0

index f765fe3..77412c7 100644
@@ -8,6 +8,7 @@ endif
 drivers = [
         'ifc',
         'mlx5',
+        'sfc',
 ]
 std_deps = ['bus_pci', 'kvargs']
 std_deps += ['vhost']

drivers/vdpa/sfc/meson.build new file +22 -0

new file mode 100644
index 0000000..4255d65
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright(c) 2020-2021 Xilinx, Inc.
+
+if (arch_subdir != 'x86' or not dpdk_conf.get('RTE_ARCH_64')) and (arch_subdir != 'arm' or not host_machine.cpu_family().startswith('aarch64'))
+	build = false
+	reason = 'only supported on x86_64 and aarch64'
+endif
+
+fmt_name = 'sfc_vdpa'
+extra_flags = []
+
+foreach flag: extra_flags
+	if cc.has_argument(flag)
+		cflags += flag
+	endif
+endforeach
+
+deps += ['common_sfc_efx', 'bus_pci']
+sources = files(
+	'sfc_vdpa.c',
+)

drivers/vdpa/sfc/sfc_vdpa.c new file +286 -0

286 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa.h new file +40 -0

new file mode 100644
index 0000000..3b77900
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_H
+#define _SFC_VDPA_H
+
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_bus_pci.h>
+
+#include "sfc_vdpa_log.h"
+
+/* Adapter private data */
+struct sfc_vdpa_adapter {
+	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	struct rte_pci_device		*pdev;
+	struct rte_pci_addr		pci_addr;
+
+	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
+	uint32_t			logtype_main;
+
+	int				vfio_group_fd;
+	int				vfio_dev_fd;
+	int				vfio_container_fd;
+	int				iommu_group_num;
+};
+
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str,
+			  uint32_t ll_default);
+
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+
+#endif  /* _SFC_VDPA_H */
+

drivers/vdpa/sfc/sfc_vdpa_log.h new file +56 -0

56 lines not on this page · raw message

drivers/vdpa/sfc/version.map new file +3 -0

new file mode 100644
index 0000000..4a76d1d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

Signature · 1 line

1.8.3.1

Mattias Rönnblom

2021-10-29 20:07 UTC #63 lore raw

Quoted 264 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>

Add new vDPA PMD to support vDPA operations of Xilinx devices. This patch implements probe and remove functions.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Updated logging macros to remove redundant code.

v3:
* Replaced depreciated whitelist with allowlist.
* Text corrections in the sfc.rst and commit msg.
* Added sfc in the toctree of doc/guides/vdpadevs/index.rst.
* Removed extra compiler flags.

  MAINTAINERS                            |   6 +
  doc/guides/rel_notes/release_21_11.rst |   5 +
  doc/guides/vdpadevs/features/sfc.ini   |   9 ++
  doc/guides/vdpadevs/index.rst          |   2 +
  doc/guides/vdpadevs/sfc.rst            |  97 +++++++++++
  drivers/vdpa/meson.build               |   1 +
  drivers/vdpa/sfc/meson.build           |  22 +++
  drivers/vdpa/sfc/sfc_vdpa.c            | 286 +++++++++++++++++++++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa.h            |  40 +++++
  drivers/vdpa/sfc/sfc_vdpa_log.h        |  56 +++++++
  drivers/vdpa/sfc/version.map           |   3 +
  11 files changed, 527 insertions(+)

create mode 100644 doc/guides/vdpadevs/features/sfc.ini create mode 100644 doc/guides/vdpadevs/sfc.rst create mode 100644 drivers/vdpa/sfc/meson.build create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h create mode 100644 drivers/vdpa/sfc/version.map

MAINTAINERS +6 -0

index be2c9b6..5d12c49 100644
@@ -1236,6 +1236,12 @@ F: drivers/vdpa/mlx5/
  F: doc/guides/vdpadevs/mlx5.rst
  F: doc/guides/vdpadevs/features/mlx5.ini
 
+Xilinx sfc vDPA
+M: Vijay Kumar Srivastava <[email protected]>
+F: drivers/vdpa/sfc/
+F: doc/guides/vdpadevs/sfc.rst
+F: doc/guides/vdpadevs/features/sfc.ini
+
 
  Eventdev Drivers
  ----------------

doc/guides/rel_notes/release_21_11.rst +5 -0

index 1ccac87..bd0a604 100644
@@ -305,6 +305,11 @@ New Features
      * Pcapng format with timestamps and meta-data.
      * Fixes packet capture with stripped VLAN tags.
 
+* **Add new vDPA PMD based on Xilinx devices.**
+
+  Added a new Xilinx vDPA  (``sfc_vdpa``) PMD.
+  See the :doc:`../vdpadevs/sfc` guide for more details on this driver.
+
 
  Removed Items
  -------------

doc/guides/vdpadevs/features/sfc.ini new file +9 -0

new file mode 100644
index 0000000..71b6158
@@ -0,0 +1,9 @@
+;
+; Supported features of the 'sfc' vDPA driver.
+;
+; Refer to default.ini for the full list of available driver features.
+;
+[Features]
+Linux			= Y
+x86-64			= Y
+Usage doc		= Y

doc/guides/vdpadevs/index.rst +2 -0

index 1a13efe..f1a946e 100644
@@ -14,3 +14,5 @@ which can be used from an application through vhost API.
      features_overview
      ifc
      mlx5
+    sfc
+

doc/guides/vdpadevs/sfc.rst new file +97 -0

new file mode 100644
index 0000000..44e694f
@@ -0,0 +1,97 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2021 Xilinx Corporation.
+
+Xilinx vDPA driver
+==================
+
+The Xilinx vDPA (vhost data path acceleration) driver (**librte_pmd_sfc_vdpa**)
+provides support for the Xilinx SN1022 SmartNICs family of 10/25/40/50/100 Gbps
+adapters that have support for latest Linux and FreeBSD operating systems.
+
+More information can be found at Xilinx website https://www.xilinx.com.
+
+
+Xilinx vDPA implementation
+--------------------------
+
+ef100 device can be configured in the net device or vDPA mode.
+Adding "class=vdpa" parameter helps to specify that this
+device is to be used in vDPA mode. If this parameter is not specified, device
+will be probed by net/sfc driver and will used as a net device.
+
+This PMD uses libefx (common/sfc_efx) code to access the device firmware.
+
+
+Supported NICs
+--------------
+
+- Xilinx SN1022 SmartNICs
+
+
+Features
+--------
+
+Features of the Xilinx vDPA driver are:
+
+- Compatibility with virtio 0.95 and 1.0
+
+
+Non-supported Features
+----------------------
+
+- Control Queue
+- Multi queue
+- Live Migration
+
+
+Prerequisites
+-------------
+
+Requires firmware version: v1.0.7.0 or higher
+
+Visit `Xilinx Support Downloads <https://www.xilinx.com/support.html>`_
+to get Xilinx Utilities with the latest firmware.
+Follow instructions from Alveo SN1000 SmartNICs User Guide to
+update firmware and configure the adapter.
+
+
+Per-Device Parameters
+~~~~~~~~~~~~~~~~~~~~~
+
+The following per-device parameters can be passed via EAL PCI device
+allowlist option like "-a 02:00.0,arg1=value1,...".
+
+Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify
+boolean parameters value.
+
+- ``class`` [net|vdpa] (default **net**)
+
+  Choose the mode of operation of ef100 device.
+  **net** device will work as network device and will be probed by net/sfc driver.
+  **vdpa** device will work as vdpa device and will be probed by vdpa/sfc driver.
+  If this parameter is not specified then ef100 device will operate as network device.
+
+
+Dynamic Logging Parameters
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One may leverage EAL option "--log-level" to change default levels
+for the log types supported by the driver. The option is used with
+an argument typically consisting of two parts separated by a colon.
+
+Level value is the last part which takes a symbolic name (or integer).
+Log type is the former part which may shell match syntax.
+Depending on the choice of the expression, the given log level may
+be used either for some specific log type or for a subset of types.
+
+SFC vDPA PMD provides the following log types available for control:
+
+- ``pmd.vdpa.sfc.driver`` (default level is **notice**)
+
+  Affects driver-wide messages unrelated to any particular devices.
+
+- ``pmd.vdpa.sfc.main`` (default level is **notice**)
+
+  Matches a subset of per-port log types registered during runtime.
+  A full name for a particular type may be obtained by appending a
+  dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.

drivers/vdpa/meson.build +1 -0

index f765fe3..77412c7 100644
@@ -8,6 +8,7 @@ endif
  drivers = [
          'ifc',
          'mlx5',
+        'sfc',
  ]
  std_deps = ['bus_pci', 'kvargs']
  std_deps += ['vhost']

drivers/vdpa/sfc/meson.build new file +22 -0

new file mode 100644
index 0000000..4255d65
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright(c) 2020-2021 Xilinx, Inc.
+
+if (arch_subdir != 'x86' or not dpdk_conf.get('RTE_ARCH_64')) and (arch_subdir != 'arm' or not host_machine.cpu_family().startswith('aarch64'))
+	build = false
+	reason = 'only supported on x86_64 and aarch64'
+endif
+
+fmt_name = 'sfc_vdpa'
+extra_flags = []
+
+foreach flag: extra_flags
+	if cc.has_argument(flag)
+		cflags += flag
+	endif
+endforeach
+
+deps += ['common_sfc_efx', 'bus_pci']
+sources = files(
+	'sfc_vdpa.c',
+)

drivers/vdpa/sfc/sfc_vdpa.c new file +20 -0

new file mode 100644
index 0000000..d85c52b
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_vfio.h>
+#include <rte_vhost.h>
+
+#include "efx.h"
+#include "sfc_efx.h"
+#include "sfc_vdpa.h"
+
+TAILQ_HEAD(sfc_vdpa_adapter_list_head, sfc_vdpa_adapter);

drivers/vdpa/sfc/sfc_vdpa.c line 30

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+static struct sfc_vdpa_adapter_list_head sfc_vdpa_adapter_list =
+	TAILQ_HEAD_INITIALIZER(sfc_vdpa_adapter_list);
+
+static pthread_mutex_t sfc_vdpa_adapter_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev)
+{
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;

Remove found flag and set sva to NULL here instead.

Quoted 26 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +26 -0

+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (pdev == sva->pdev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva : NULL;
+}
+
+static int
+sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *dev = sva->pdev;
+	char dev_name[RTE_DEV_NAME_MAX_LEN] = {0};
+	int rc;
+
+	if (dev == NULL)
+		goto fail_inval;
+
+	rte_pci_device_name(&dev->addr, dev_name, RTE_DEV_NAME_MAX_LEN);

drivers/vdpa/sfc/sfc_vdpa.c line 66

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+
+	sva->vfio_container_fd = rte_vfio_container_create();
+	if (sva->vfio_container_fd < 0)	{
+		sfc_vdpa_err(sva, "failed to create VFIO container");
+		goto fail_container_create;
+	}
+
+	rc = rte_vfio_get_group_num(rte_pci_get_sysfs_path(), dev_name,
+				    &sva->iommu_group_num);
+	if (rc <= 0) {

Only rc < 0 guarantees that rte_errno is set.

Quoted 26 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +26 -0

+		sfc_vdpa_err(sva, "failed to get IOMMU group for %s : %s",
+			     dev_name, rte_strerror(-rc));
+		goto fail_get_group_num;
+	}
+
+	sva->vfio_group_fd =
+		rte_vfio_container_group_bind(sva->vfio_container_fd,
+					      sva->iommu_group_num);
+	if (sva->vfio_group_fd < 0) {
+		sfc_vdpa_err(sva,
+			     "failed to bind IOMMU group %d to container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+		goto fail_group_bind;
+	}
+
+	if (rte_pci_map_device(dev) != 0) {
+		sfc_vdpa_err(sva, "failed to map PCI device %s : %s",
+			     dev_name, rte_strerror(rte_errno));
+		goto fail_pci_map_device;
+	}
+
+	sva->vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
+
+	return 0;
+
+fail_pci_map_device:

drivers/vdpa/sfc/sfc_vdpa.c line 102

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+	if (rte_vfio_container_group_unbind(sva->vfio_container_fd,
+					sva->iommu_group_num) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to unbind IOMMU group %d from container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+	}
+
+fail_group_bind:
+fail_get_group_num:
+	if (rte_vfio_container_destroy(sva->vfio_container_fd) != 0) {

Don't use braces for single statements, per DPDK coding style.

Quoted 302 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +184 -0

+		sfc_vdpa_err(sva, "failed to destroy container %d",
+			     sva->vfio_container_fd);
+	}
+
+fail_container_create:
+fail_inval:
+	return -1;
+}
+
+static void
+sfc_vdpa_vfio_teardown(struct sfc_vdpa_adapter *sva)
+{
+	rte_pci_unmap_device(sva->pdev);
+
+	if (rte_vfio_container_group_unbind(sva->vfio_container_fd,
+					    sva->iommu_group_num) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to unbind IOMMU group %d from container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+	}
+
+	if (rte_vfio_container_destroy(sva->vfio_container_fd) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to destroy container %d",
+			     sva->vfio_container_fd);
+	}
+}
+
+static int
+sfc_vdpa_set_log_prefix(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	int ret;
+
+	ret = snprintf(sva->log_prefix, sizeof(sva->log_prefix),
+		       "PMD: sfc_vdpa " PCI_PRI_FMT " : ",
+		       pci_dev->addr.domain, pci_dev->addr.bus,
+		       pci_dev->addr.devid, pci_dev->addr.function);
+
+	if (ret < 0 || ret >= (int)sizeof(sva->log_prefix)) {
+		SFC_VDPA_GENERIC_LOG(ERR,
+			"reserved log prefix is too short for " PCI_PRI_FMT,
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str, uint32_t ll_default)
+{
+	size_t lt_prefix_str_size = strlen(lt_prefix_str);
+	size_t lt_str_size_max;
+	char *lt_str = NULL;
+	int ret;
+
+	if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) {
+		++lt_prefix_str_size; /* Reserve space for prefix separator */
+		lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1;
+	} else {
+		return RTE_LOGTYPE_PMD;
+	}
+
+	lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0);
+	if (lt_str == NULL)
+		return RTE_LOGTYPE_PMD;
+
+	strncpy(lt_str, lt_prefix_str, lt_prefix_str_size);
+	lt_str[lt_prefix_str_size - 1] = '.';
+	rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size,
+			    lt_str_size_max - lt_prefix_str_size);
+	lt_str[lt_str_size_max - 1] = '\0';
+
+	ret = rte_log_register_type_and_pick_level(lt_str, ll_default);
+	rte_free(lt_str);
+
+	return ret < 0 ? RTE_LOGTYPE_PMD : ret;
+}
+
+static struct rte_pci_id pci_id_sfc_vdpa_efx_map[] = {
+	{ RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) },
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+static int
+sfc_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	struct rte_pci_device *pci_dev)
+{
+	struct sfc_vdpa_adapter *sva = NULL;
+	uint32_t logtype_main;
+	int ret = 0;
+
+	if (sfc_efx_dev_class_get(pci_dev->device.devargs) !=
+			SFC_EFX_DEV_CLASS_VDPA) {
+		SFC_VDPA_GENERIC_LOG(INFO,
+			"Incompatible device class: skip probing, should be probed by other sfc driver.");
+			return 1;
+	}
+
+	/*
+	 * It will not be probed in the secondary process. As device class
+	 * is vdpa so return 0 to avoid probe by other sfc driver
+	 */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	logtype_main = sfc_vdpa_register_logtype(&pci_dev->addr,
+						 SFC_VDPA_LOGTYPE_MAIN_STR,
+						 RTE_LOG_NOTICE);
+
+	sva = rte_zmalloc("sfc_vdpa", sizeof(struct sfc_vdpa_adapter), 0);
+	if (sva == NULL)
+		goto fail_zmalloc;
+
+	sva->pdev = pci_dev;
+	sva->logtype_main = logtype_main;
+
+	ret = sfc_vdpa_set_log_prefix(sva);
+	if (ret != 0)
+		goto fail_set_log_prefix;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "vfio init");
+	if (sfc_vdpa_vfio_setup(sva) < 0) {
+		sfc_vdpa_err(sva, "failed to setup device %s", pci_dev->name);
+		goto fail_vfio_setup;
+	}
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	sfc_vdpa_log_init(sva, "done");
+
+	return 0;
+
+fail_vfio_setup:
+fail_set_log_prefix:
+	rte_free(sva);
+
+fail_zmalloc:
+	return -1;
+}
+
+static int
+sfc_vdpa_pci_remove(struct rte_pci_device *pci_dev)
+{
+	struct sfc_vdpa_adapter *sva = NULL;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return -1;
+
+	sva = sfc_vdpa_get_adapter_by_dev(pci_dev);
+	if (sva == NULL) {
+		sfc_vdpa_info(sva, "invalid device: %s", pci_dev->name);
+		return -1;
+	}
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	sfc_vdpa_vfio_teardown(sva);
+
+	rte_free(sva);
+
+	return 0;
+}
+
+static struct rte_pci_driver rte_sfc_vdpa = {
+	.id_table = pci_id_sfc_vdpa_efx_map,
+	.drv_flags = 0,
+	.probe = sfc_vdpa_pci_probe,
+	.remove = sfc_vdpa_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(net_sfc_vdpa, rte_sfc_vdpa);
+RTE_PMD_REGISTER_PCI_TABLE(net_sfc_vdpa, pci_id_sfc_vdpa_efx_map);
+RTE_PMD_REGISTER_KMOD_DEP(net_sfc_vdpa, "* vfio-pci");
+RTE_LOG_REGISTER_SUFFIX(sfc_vdpa_logtype_driver, driver, NOTICE);

drivers/vdpa/sfc/sfc_vdpa.h new file +40 -0

new file mode 100644
index 0000000..3b77900
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_H
+#define _SFC_VDPA_H
+
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_bus_pci.h>
+
+#include "sfc_vdpa_log.h"
+
+/* Adapter private data */
+struct sfc_vdpa_adapter {
+	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	struct rte_pci_device		*pdev;
+	struct rte_pci_addr		pci_addr;
+
+	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
+	uint32_t			logtype_main;
+
+	int				vfio_group_fd;
+	int				vfio_dev_fd;
+	int				vfio_container_fd;
+	int				iommu_group_num;
+};
+
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str,
+			  uint32_t ll_default);
+
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+
+#endif  /* _SFC_VDPA_H */
+

drivers/vdpa/sfc/sfc_vdpa_log.h new file +56 -0

new file mode 100644
index 0000000..858e5ee
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_LOG_H_
+#define _SFC_VDPA_LOG_H_
+
+/** Generic driver log type */
+extern int sfc_vdpa_logtype_driver;
+
+/** Common log type name prefix */
+#define SFC_VDPA_LOGTYPE_PREFIX	"pmd.vdpa.sfc."
+
+/** Log PMD generic message, add a prefix and a line break */
+#define SFC_VDPA_GENERIC_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, sfc_vdpa_logtype_driver,		\
+		RTE_FMT("PMD: " RTE_FMT_HEAD(__VA_ARGS__ ,) "\n",	\
+			RTE_FMT_TAIL(__VA_ARGS__ ,)))
+
+/** Name prefix for the per-device log type used to report basic information */
+#define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
+
+#define SFC_VDPA_LOG_PREFIX_MAX	32
+
+/* Log PMD message, automatically add prefix and \n */
+#define SFC_VDPA_LOG(sva, level, ...) \
+	do {								\
+		const struct sfc_vdpa_adapter *_sva = (sva);		\
+									\
+		rte_log(RTE_LOG_ ## level, _sva->logtype_main,		\
+			RTE_FMT("%s" RTE_FMT_HEAD(__VA_ARGS__ ,) "\n",	\
+				_sva->log_prefix,			\
+				RTE_FMT_TAIL(__VA_ARGS__ ,)));		\
+	} while (0)
+
+#define sfc_vdpa_err(sva, ...) \
+	SFC_VDPA_LOG(sva, ERR, __VA_ARGS__)
+
+#define sfc_vdpa_warn(sva, ...) \
+	SFC_VDPA_LOG(sva, WARNING, __VA_ARGS__)
+
+#define sfc_vdpa_notice(sva, ...) \
+	SFC_VDPA_LOG(sva, NOTICE, __VA_ARGS__)
+
+#define sfc_vdpa_info(sva, ...) \
+	SFC_VDPA_LOG(sva, INFO, __VA_ARGS__)
+
+#define sfc_vdpa_log_init(sva, ...) \
+	SFC_VDPA_LOG(sva, INFO,				\
+		RTE_FMT("%s(): "			\
+			RTE_FMT_HEAD(__VA_ARGS__ ,),	\
+			__func__,			\
+			RTE_FMT_TAIL(__VA_ARGS__ ,)))
+
+#endif /* _SFC_VDPA_LOG_H_ */

drivers/vdpa/sfc/version.map new file +3 -0

new file mode 100644
index 0000000..4a76d1d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};

Vijay Kumar Srivastava

2021-11-01 08:13 UTC #64 lore raw

Hi Mattias,

Quoted 12 lines

-----Original Message-----
From: Mattias Rönnblom <[email protected]>
Sent: Saturday, October 30, 2021 1:38 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver
Quoted 1 line · Vijay Srivastava
From: Vijay Kumar Srivastava <[email protected]>

[SNIP]

Quoted 7 lines

drivers/vdpa/sfc/sfc_vdpa.c line 30
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa.c new file +4 -0
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev) {
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;
Remove found flag and set sva to NULL here instead.

This flag is needed for the scenario when pdev != sva->pdev. In this scenario, sva would be non-null and it should not be returned. So I think it's OK to keep this flag.

Quoted 26 lines

Quoted 15 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +15 -0

+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (pdev == sva->pdev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva : NULL;
+}
+

drivers/vdpa/sfc/sfc_vdpa.c

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+static int
+sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva) {
+	struct rte_pci_device *dev = sva->pdev;
+	char dev_name[RTE_DEV_NAME_MAX_LEN] = {0};
+	int rc;
+
+	if (dev == NULL)
+		goto fail_inval;
+
+	rte_pci_device_name(&dev->addr, dev_name,

RTE_DEV_NAME_MAX_LEN);

Quoted 12 lines

drivers/vdpa/sfc/sfc_vdpa.c line 66

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+
+	sva->vfio_container_fd = rte_vfio_container_create();
+	if (sva->vfio_container_fd < 0)	{
+		sfc_vdpa_err(sva, "failed to create VFIO container");
+		goto fail_container_create;
+	}
+
+	rc = rte_vfio_get_group_num(rte_pci_get_sysfs_path(), dev_name,
+				    &sva->iommu_group_num);
+	if (rc <= 0) {

Only rc < 0 guarantees that rte_errno is set.

rte_vfio_get_group_num retunrs >0 on success 0 for non-existent group or VFIO <0 for errors so used check '(rc <= 0)' looks OK.

Quoted 31 lines

Quoted 20 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +20 -0

+		sfc_vdpa_err(sva, "failed to get IOMMU group for %s : %s",
+			     dev_name, rte_strerror(-rc));
+		goto fail_get_group_num;
+	}
+
+	sva->vfio_group_fd =
+		rte_vfio_container_group_bind(sva->vfio_container_fd,
+					      sva->iommu_group_num);
+	if (sva->vfio_group_fd < 0) {
+		sfc_vdpa_err(sva,
+			     "failed to bind IOMMU group %d to container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+		goto fail_group_bind;
+	}
+
+	if (rte_pci_map_device(dev) != 0) {
+		sfc_vdpa_err(sva, "failed to map PCI device %s : %s",
+			     dev_name, rte_strerror(rte_errno));
+		goto fail_pci_map_device;
+	}

drivers/vdpa/sfc/sfc_vdpa.c

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+
+	sva->vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
+
+	return 0;
+
+fail_pci_map_device:
+	if (rte_vfio_container_group_unbind(sva->vfio_container_fd,
+					sva->iommu_group_num) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to unbind IOMMU group %d from container

%d",

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa.c line 102
Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa.c new file +6 -0
+			     sva->iommu_group_num, sva->vfio_container_fd);
+	}
+
+fail_group_bind:
+fail_get_group_num:
+	if (rte_vfio_container_destroy(sva->vfio_container_fd) != 0) {
Don't use braces for single statements, per DPDK coding style.

As per DPDK coding guidelines :
"Do not use braces ({ and }) for control statements with zero or just a single statement, unless that statement is more than a single line in which case the braces are permitted."

I believe it can be applied here as well as it has more than one line.

Regards,
Vijay

Quoted 3 lines

Quoted 3 lines
drivers/vdpa/sfc/sfc_vdpa.c new file +3 -0
+		sfc_vdpa_err(sva, "failed to destroy container %d",
+			     sva->vfio_container_fd);
+	}

[SNIP]

Xia, Chenbo

2021-11-01 08:30 UTC #65 lore raw

Hi Vijay,

Quoted 8 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver

Quoted 7 lines

From: Vijay Kumar Srivastava <[email protected]>
Add new vDPA PMD to support vDPA operations of Xilinx devices. This patch implements probe and remove functions.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>

Note that your patch is not rebased on the latest commit. Reviewers may have trouble to review your patch.

Quoted 54 lines

---
v2:
* Updated logging macros to remove redundant code.

v3:
* Replaced depreciated whitelist with allowlist.
* Text corrections in the sfc.rst and commit msg.
* Added sfc in the toctree of doc/guides/vdpadevs/index.rst.
* Removed extra compiler flags.

 MAINTAINERS                            |   6 +
 doc/guides/rel_notes/release_21_11.rst |   5 +
 doc/guides/vdpadevs/features/sfc.ini   |   9 ++
 doc/guides/vdpadevs/index.rst          |   2 +
 doc/guides/vdpadevs/sfc.rst            |  97 +++++++++++
 drivers/vdpa/meson.build               |   1 +
 drivers/vdpa/sfc/meson.build           |  22 +++
 drivers/vdpa/sfc/sfc_vdpa.c            | 286
+++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa.h            |  40 +++++
 drivers/vdpa/sfc/sfc_vdpa_log.h        |  56 +++++++
 drivers/vdpa/sfc/version.map           |   3 +
 11 files changed, 527 insertions(+)
 create mode 100644 doc/guides/vdpadevs/features/sfc.ini
 create mode 100644 doc/guides/vdpadevs/sfc.rst
 create mode 100644 drivers/vdpa/sfc/meson.build
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_log.h
 create mode 100644 drivers/vdpa/sfc/version.map

MAINTAINERS +6 -0

index be2c9b6..5d12c49 100644
@@ -1236,6 +1236,12 @@ F: drivers/vdpa/mlx5/
 F: doc/guides/vdpadevs/mlx5.rst
 F: doc/guides/vdpadevs/features/mlx5.ini
 
+Xilinx sfc vDPA
+M: Vijay Kumar Srivastava <[email protected]>
+F: drivers/vdpa/sfc/
+F: doc/guides/vdpadevs/sfc.rst
+F: doc/guides/vdpadevs/features/sfc.ini
+
 
 Eventdev Drivers
 ----------------

doc/guides/rel_notes/release_21_11.rst +0 -0

index 1ccac87..bd0a604 100644

doc/guides/rel_notes/release_21_11.rst New Features · line 308

Quoted 4 lines

doc/guides/rel_notes/release_21_11.rst +1 -0

@@ -305,6 +305,11 @@ New Features
     * Pcapng format with timestamps and meta-data.
     * Fixes packet capture with stripped VLAN tags.
 
+* **Add new vDPA PMD based on Xilinx devices.**

Add -> Added

Quoted 28 lines

doc/guides/rel_notes/release_21_11.rst +4 -0

+
+  Added a new Xilinx vDPA  (``sfc_vdpa``) PMD.
+  See the :doc:`../vdpadevs/sfc` guide for more details on this driver.
+
 
 Removed Items
 -------------

doc/guides/vdpadevs/features/sfc.ini new file +9 -0

new file mode 100644
index 0000000..71b6158
@@ -0,0 +1,9 @@
+;
+; Supported features of the 'sfc' vDPA driver.
+;
+; Refer to default.ini for the full list of available driver features.
+;
+[Features]
+Linux			= Y
+x86-64			= Y
+Usage doc		= Y

doc/guides/vdpadevs/index.rst +0 -0

index 1a13efe..f1a946e 100644

doc/guides/vdpadevs/index.rst which can be used from an application through vhost API. · line 18

Quoted 5 lines

doc/guides/vdpadevs/index.rst +2 -0

@@ -14,3 +14,5 @@ which can be used from an application through vhost API.
     features_overview
     ifc
     mlx5
+    sfc
+

The above blank line not needed.

Quoted 198 lines

doc/guides/vdpadevs/sfc.rst new file +7 -0

new file mode 100644
index 0000000..44e694f
@@ -0,0 +1,97 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2021 Xilinx Corporation.
+
+Xilinx vDPA driver
+==================
+
+The Xilinx vDPA (vhost data path acceleration) driver

(**librte_pmd_sfc_vdpa**)
+provides support for the Xilinx SN1022 SmartNICs family of 10/25/40/50/100 Gbps

doc/guides/vdpadevs/sfc.rst new file +62 -0

+adapters that have support for latest Linux and FreeBSD operating systems.
+
+More information can be found at Xilinx website https://www.xilinx.com.
+
+
+Xilinx vDPA implementation
+--------------------------
+
+ef100 device can be configured in the net device or vDPA mode.
+Adding "class=vdpa" parameter helps to specify that this
+device is to be used in vDPA mode. If this parameter is not specified, device
+will be probed by net/sfc driver and will used as a net device.
+
+This PMD uses libefx (common/sfc_efx) code to access the device firmware.
+
+
+Supported NICs
+--------------
+
+- Xilinx SN1022 SmartNICs
+
+
+Features
+--------
+
+Features of the Xilinx vDPA driver are:
+
+- Compatibility with virtio 0.95 and 1.0
+
+
+Non-supported Features
+----------------------
+
+- Control Queue
+- Multi queue
+- Live Migration
+
+
+Prerequisites
+-------------
+
+Requires firmware version: v1.0.7.0 or higher
+
+Visit `Xilinx Support Downloads <https://www.xilinx.com/support.html>`_
+to get Xilinx Utilities with the latest firmware.
+Follow instructions from Alveo SN1000 SmartNICs User Guide to
+update firmware and configure the adapter.
+
+
+Per-Device Parameters
+~~~~~~~~~~~~~~~~~~~~~
+
+The following per-device parameters can be passed via EAL PCI device
+allowlist option like "-a 02:00.0,arg1=value1,...".
+
+Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify
+boolean parameters value.
+
+- ``class`` [net|vdpa] (default **net**)
+
+  Choose the mode of operation of ef100 device.
+  **net** device will work as network device and will be probed by net/sfc

driver.
+ **vdpa** device will work as vdpa device and will be probed by vdpa/sfc driver.
+ If this parameter is not specified then ef100 device will operate as network device.

doc/guides/vdpadevs/sfc.rst new file +25 -0

+
+
+Dynamic Logging Parameters
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One may leverage EAL option "--log-level" to change default levels
+for the log types supported by the driver. The option is used with
+an argument typically consisting of two parts separated by a colon.
+
+Level value is the last part which takes a symbolic name (or integer).
+Log type is the former part which may shell match syntax.
+Depending on the choice of the expression, the given log level may
+be used either for some specific log type or for a subset of types.
+
+SFC vDPA PMD provides the following log types available for control:
+
+- ``pmd.vdpa.sfc.driver`` (default level is **notice**)
+
+  Affects driver-wide messages unrelated to any particular devices.
+
+- ``pmd.vdpa.sfc.main`` (default level is **notice**)
+
+  Matches a subset of per-port log types registered during runtime.
+  A full name for a particular type may be obtained by appending a
+  dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.

drivers/vdpa/meson.build +1 -0

index f765fe3..77412c7 100644
@@ -8,6 +8,7 @@ endif
 drivers = [
         'ifc',
         'mlx5',
+        'sfc',
 ]
 std_deps = ['bus_pci', 'kvargs']
 std_deps += ['vhost']

drivers/vdpa/sfc/meson.build new file +5 -0

new file mode 100644
index 0000000..4255d65
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# Copyright(c) 2020-2021 Xilinx, Inc.
+
+if (arch_subdir != 'x86' or not dpdk_conf.get('RTE_ARCH_64')) and

(arch_subdir != 'arm' or not host_machine.cpu_family().startswith('aarch64'))

drivers/vdpa/sfc/meson.build new file +17 -0

+	build = false
+	reason = 'only supported on x86_64 and aarch64'
+endif
+
+fmt_name = 'sfc_vdpa'
+extra_flags = []
+
+foreach flag: extra_flags
+	if cc.has_argument(flag)
+		cflags += flag
+	endif
+endforeach
+
+deps += ['common_sfc_efx', 'bus_pci']
+sources = files(
+	'sfc_vdpa.c',
+)

drivers/vdpa/sfc/sfc_vdpa.c new file +43 -0

new file mode 100644
index 0000000..d85c52b
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_string_fns.h>
+#include <rte_vfio.h>
+#include <rte_vhost.h>
+
+#include "efx.h"
+#include "sfc_efx.h"
+#include "sfc_vdpa.h"
+
+TAILQ_HEAD(sfc_vdpa_adapter_list_head, sfc_vdpa_adapter);
+static struct sfc_vdpa_adapter_list_head sfc_vdpa_adapter_list =
+	TAILQ_HEAD_INITIALIZER(sfc_vdpa_adapter_list);
+
+static pthread_mutex_t sfc_vdpa_adapter_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev)
+{
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (pdev == sva->pdev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva : NULL;

drivers/vdpa/sfc/sfc_vdpa.c line 53

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+}
+
+static int
+sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *dev = sva->pdev;
+	char dev_name[RTE_DEV_NAME_MAX_LEN] = {0};
+	int rc;
+
+	if (dev == NULL)

I think it will never happen

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+		goto fail_inval;
+
+	rte_pci_device_name(&dev->addr, dev_name, RTE_DEV_NAME_MAX_LEN);
+
+	sva->vfio_container_fd = rte_vfio_container_create();
+	if (sva->vfio_container_fd < 0)	{
+		sfc_vdpa_err(sva, "failed to create VFIO container");
+		goto fail_container_create;
+	}
+

drivers/vdpa/sfc/sfc_vdpa.c line 73

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +10 -0

+	rc = rte_vfio_get_group_num(rte_pci_get_sysfs_path(), dev_name,
+				    &sva->iommu_group_num);
+	if (rc <= 0) {
+		sfc_vdpa_err(sva, "failed to get IOMMU group for %s : %s",
+			     dev_name, rte_strerror(-rc));
+		goto fail_get_group_num;
+	}
+
+	sva->vfio_group_fd =
+		rte_vfio_container_group_bind(sva->vfio_container_fd,

DPDK allows up to 100 chars per line. Please check all code that do not need new lines. It will make the code look better.

Quoted 230 lines

drivers/vdpa/sfc/sfc_vdpa.c new file +128 -0

+					      sva->iommu_group_num);
+	if (sva->vfio_group_fd < 0) {
+		sfc_vdpa_err(sva,
+			     "failed to bind IOMMU group %d to container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+		goto fail_group_bind;
+	}
+
+	if (rte_pci_map_device(dev) != 0) {
+		sfc_vdpa_err(sva, "failed to map PCI device %s : %s",
+			     dev_name, rte_strerror(rte_errno));
+		goto fail_pci_map_device;
+	}
+
+	sva->vfio_dev_fd = rte_intr_dev_fd_get(dev->intr_handle);
+
+	return 0;
+
+fail_pci_map_device:
+	if (rte_vfio_container_group_unbind(sva->vfio_container_fd,
+					sva->iommu_group_num) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to unbind IOMMU group %d from container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+	}
+
+fail_group_bind:
+fail_get_group_num:
+	if (rte_vfio_container_destroy(sva->vfio_container_fd) != 0) {
+		sfc_vdpa_err(sva, "failed to destroy container %d",
+			     sva->vfio_container_fd);
+	}
+
+fail_container_create:
+fail_inval:
+	return -1;
+}
+
+static void
+sfc_vdpa_vfio_teardown(struct sfc_vdpa_adapter *sva)
+{
+	rte_pci_unmap_device(sva->pdev);
+
+	if (rte_vfio_container_group_unbind(sva->vfio_container_fd,
+					    sva->iommu_group_num) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to unbind IOMMU group %d from container %d",
+			     sva->iommu_group_num, sva->vfio_container_fd);
+	}
+
+	if (rte_vfio_container_destroy(sva->vfio_container_fd) != 0) {
+		sfc_vdpa_err(sva,
+			     "failed to destroy container %d",
+			     sva->vfio_container_fd);
+	}
+}
+
+static int
+sfc_vdpa_set_log_prefix(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	int ret;
+
+	ret = snprintf(sva->log_prefix, sizeof(sva->log_prefix),
+		       "PMD: sfc_vdpa " PCI_PRI_FMT " : ",
+		       pci_dev->addr.domain, pci_dev->addr.bus,
+		       pci_dev->addr.devid, pci_dev->addr.function);
+
+	if (ret < 0 || ret >= (int)sizeof(sva->log_prefix)) {
+		SFC_VDPA_GENERIC_LOG(ERR,
+			"reserved log prefix is too short for " PCI_PRI_FMT,
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str, uint32_t ll_default)
+{
+	size_t lt_prefix_str_size = strlen(lt_prefix_str);
+	size_t lt_str_size_max;
+	char *lt_str = NULL;
+	int ret;
+
+	if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) {
+		++lt_prefix_str_size; /* Reserve space for prefix separator */
+		lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1;
+	} else {
+		return RTE_LOGTYPE_PMD;
+	}
+
+	lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0);
+	if (lt_str == NULL)
+		return RTE_LOGTYPE_PMD;
+
+	strncpy(lt_str, lt_prefix_str, lt_prefix_str_size);
+	lt_str[lt_prefix_str_size - 1] = '.';
+	rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size,
+			    lt_str_size_max - lt_prefix_str_size);
+	lt_str[lt_str_size_max - 1] = '\0';
+
+	ret = rte_log_register_type_and_pick_level(lt_str, ll_default);
+	rte_free(lt_str);
+
+	return ret < 0 ? RTE_LOGTYPE_PMD : ret;
+}
+
+static struct rte_pci_id pci_id_sfc_vdpa_efx_map[] = {
+	{ RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) },
+	{ .vendor_id = 0, /* sentinel */ },
+};
+
+static int
+sfc_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+	struct rte_pci_device *pci_dev)
+{
+	struct sfc_vdpa_adapter *sva = NULL;
+	uint32_t logtype_main;
+	int ret = 0;
+
+	if (sfc_efx_dev_class_get(pci_dev->device.devargs) !=
+			SFC_EFX_DEV_CLASS_VDPA) {
+		SFC_VDPA_GENERIC_LOG(INFO,
+			"Incompatible device class: skip probing, should be probed

by other sfc driver.");

drivers/vdpa/sfc/sfc_vdpa.c new file +85 -0

+			return 1;
+	}
+
+	/*
+	 * It will not be probed in the secondary process. As device class
+	 * is vdpa so return 0 to avoid probe by other sfc driver
+	 */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	logtype_main = sfc_vdpa_register_logtype(&pci_dev->addr,
+						 SFC_VDPA_LOGTYPE_MAIN_STR,
+						 RTE_LOG_NOTICE);
+
+	sva = rte_zmalloc("sfc_vdpa", sizeof(struct sfc_vdpa_adapter), 0);
+	if (sva == NULL)
+		goto fail_zmalloc;
+
+	sva->pdev = pci_dev;
+	sva->logtype_main = logtype_main;
+
+	ret = sfc_vdpa_set_log_prefix(sva);
+	if (ret != 0)
+		goto fail_set_log_prefix;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "vfio init");
+	if (sfc_vdpa_vfio_setup(sva) < 0) {
+		sfc_vdpa_err(sva, "failed to setup device %s", pci_dev->name);
+		goto fail_vfio_setup;
+	}
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	sfc_vdpa_log_init(sva, "done");
+
+	return 0;
+
+fail_vfio_setup:
+fail_set_log_prefix:
+	rte_free(sva);
+
+fail_zmalloc:
+	return -1;
+}
+
+static int
+sfc_vdpa_pci_remove(struct rte_pci_device *pci_dev)
+{
+	struct sfc_vdpa_adapter *sva = NULL;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return -1;
+
+	sva = sfc_vdpa_get_adapter_by_dev(pci_dev);
+	if (sva == NULL) {
+		sfc_vdpa_info(sva, "invalid device: %s", pci_dev->name);
+		return -1;
+	}
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	sfc_vdpa_vfio_teardown(sva);
+
+	rte_free(sva);
+
+	return 0;
+}
+
+static struct rte_pci_driver rte_sfc_vdpa = {
+	.id_table = pci_id_sfc_vdpa_efx_map,
+	.drv_flags = 0,
+	.probe = sfc_vdpa_pci_probe,
+	.remove = sfc_vdpa_pci_remove,
+};
+
+RTE_PMD_REGISTER_PCI(net_sfc_vdpa, rte_sfc_vdpa);
+RTE_PMD_REGISTER_PCI_TABLE(net_sfc_vdpa, pci_id_sfc_vdpa_efx_map);
+RTE_PMD_REGISTER_KMOD_DEP(net_sfc_vdpa, "* vfio-pci");
+RTE_LOG_REGISTER_SUFFIX(sfc_vdpa_logtype_driver, driver, NOTICE);

drivers/vdpa/sfc/sfc_vdpa.h new file +10 -0

new file mode 100644
index 0000000..3b77900
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_H
+#define _SFC_VDPA_H
+
+#include <stdint.h>
+#include <sys/queue.h>

drivers/vdpa/sfc/sfc_vdpa.h line 20

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa.h new file +10 -0

+
+#include <rte_bus_pci.h>
+
+#include "sfc_vdpa_log.h"
+
+/* Adapter private data */
+struct sfc_vdpa_adapter {
+	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	struct rte_pci_device		*pdev;
+	struct rte_pci_addr		pci_addr;

I don't think pci_addr is needed, you can get it through pdev.

Thanks,
Chenbo

Quoted 94 lines

drivers/vdpa/sfc/sfc_vdpa.h new file +20 -0

+
+	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
+	uint32_t			logtype_main;
+
+	int				vfio_group_fd;
+	int				vfio_dev_fd;
+	int				vfio_container_fd;
+	int				iommu_group_num;
+};
+
+uint32_t
+sfc_vdpa_register_logtype(const struct rte_pci_addr *pci_addr,
+			  const char *lt_prefix_str,
+			  uint32_t ll_default);
+
+struct sfc_vdpa_adapter *
+sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+
+#endif  /* _SFC_VDPA_H */
+

drivers/vdpa/sfc/sfc_vdpa_log.h new file +21 -0

new file mode 100644
index 0000000..858e5ee
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_LOG_H_
+#define _SFC_VDPA_LOG_H_
+
+/** Generic driver log type */
+extern int sfc_vdpa_logtype_driver;
+
+/** Common log type name prefix */
+#define SFC_VDPA_LOGTYPE_PREFIX	"pmd.vdpa.sfc."
+
+/** Log PMD generic message, add a prefix and a line break */
+#define SFC_VDPA_GENERIC_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, sfc_vdpa_logtype_driver,		\
+		RTE_FMT("PMD: " RTE_FMT_HEAD(__VA_ARGS__ ,) "\n",	\
+			RTE_FMT_TAIL(__VA_ARGS__ ,)))
+
+/** Name prefix for the per-device log type used to report basic information

drivers/vdpa/sfc/sfc_vdpa_log.h new file +35 -0

+#define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
+
+#define SFC_VDPA_LOG_PREFIX_MAX	32
+
+/* Log PMD message, automatically add prefix and \n */
+#define SFC_VDPA_LOG(sva, level, ...) \
+	do {								\
+		const struct sfc_vdpa_adapter *_sva = (sva);		\
+									\
+		rte_log(RTE_LOG_ ## level, _sva->logtype_main,		\
+			RTE_FMT("%s" RTE_FMT_HEAD(__VA_ARGS__ ,) "\n",	\
+				_sva->log_prefix,			\
+				RTE_FMT_TAIL(__VA_ARGS__ ,)));		\
+	} while (0)
+
+#define sfc_vdpa_err(sva, ...) \
+	SFC_VDPA_LOG(sva, ERR, __VA_ARGS__)
+
+#define sfc_vdpa_warn(sva, ...) \
+	SFC_VDPA_LOG(sva, WARNING, __VA_ARGS__)
+
+#define sfc_vdpa_notice(sva, ...) \
+	SFC_VDPA_LOG(sva, NOTICE, __VA_ARGS__)
+
+#define sfc_vdpa_info(sva, ...) \
+	SFC_VDPA_LOG(sva, INFO, __VA_ARGS__)
+
+#define sfc_vdpa_log_init(sva, ...) \
+	SFC_VDPA_LOG(sva, INFO,				\
+		RTE_FMT("%s(): "			\
+			RTE_FMT_HEAD(__VA_ARGS__ ,),	\
+			__func__,			\
+			RTE_FMT_TAIL(__VA_ARGS__ ,)))
+
+#endif /* _SFC_VDPA_LOG_H_ */

drivers/vdpa/sfc/version.map new file +3 -1

new file mode 100644
index 0000000..4a76d1d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
--

1.8.3.1

Andrew Rybchenko

2021-11-01 08:59 UTC #66 lore raw

Hi Chenbo,

Thanks for review.
Few questions from me below on your review notes.

Andrew.

Quoted 21 lines · Chenbo

Hi Vijay,
Quoted 8 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver
Quoted 7 lines
From: Vijay Kumar Srivastava <[email protected]>
Add new vDPA PMD to support vDPA operations of Xilinx devices. This patch implements probe and remove functions.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
Note that your patch is not rebased on the latest commit. Reviewers may have trouble to review your patch.

[snip]

Quoted 5 lines

drivers/vdpa/sfc/sfc_vdpa.c line 73
Quoted 2 lines
drivers/vdpa/sfc/sfc_vdpa.c new file +2 -0
+	sva->vfio_group_fd =
+		rte_vfio_container_group_bind(sva->vfio_container_fd,
DPDK allows up to 100 chars per line. Please check all code that do not need new lines. It will make the code look better.

I'm sorry, but as far as I know permissive to have up to 100 chars does not enforce to stretch all lines. Or am I missing something in coding style? IMHO, it is still *allowed* to avoid lines longer than 80 chars.

[snip]

Xia, Chenbo

2021-11-01 09:10 UTC #67 lore raw

Quoted 46 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Monday, November 1, 2021 5:00 PM
To: Xia, Chenbo <[email protected]>; Vijay Srivastava <[email protected]>; [email protected]
Cc: [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: Re: [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver
Hi Chenbo,
Thanks for review.
Few questions from me below on your review notes.
Andrew.
Quoted 21 lines · Chenbo
Hi Vijay,
Quoted 8 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver
Quoted 7 lines
From: Vijay Kumar Srivastava <[email protected]>
Add new vDPA PMD to support vDPA operations of Xilinx devices. This patch implements probe and remove functions.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
Note that your patch is not rebased on the latest commit. Reviewers may have trouble to review your patch.
[snip]
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa.c line 73
Quoted 2 lines
drivers/vdpa/sfc/sfc_vdpa.c new file +2 -0
+	sva->vfio_group_fd =
+		rte_vfio_container_group_bind(sva->vfio_container_fd,
DPDK allows up to 100 chars per line. Please check all code that do not need new lines. It will make the code look better.

Quoted 4 lines

I'm sorry, but as far as I know permissive to have up to 100 chars does not enforce to stretch all lines. Or am I missing something in coding style? IMHO, it is still *allowed* to avoid lines longer than 80 chars.

You are correct. 80 chars is still allowed. You can keep it if it's your preference. I just thought most people would choose 100 chars for a case above. But you can choose on your own :)

/Chenbo

Quoted 2 lines

[snip]

Vijay Kumar Srivastava

2021-11-01 09:53 UTC #68 lore raw

In reply to Xia, Chenbo

Hi Chenbo,

Quoted 21 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, November 1, 2021 2:01 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver
Hi Vijay,
Quoted 11 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 01/10] vdpa/sfc: introduce Xilinx vDPA driver
From: Vijay Kumar Srivastava <[email protected]>

Quoted 8 lines

Quoted 5 lines
Add new vDPA PMD to support vDPA operations of Xilinx devices. This patch implements probe and remove functions.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
Note that your patch is not rebased on the latest commit. Reviewers may have trouble to review your patch.

Ack.

When I had prepared the patch series I needed patches from main (or next-net) due to changes in the intr_handle (internal) as at that time next-virtio was not updated.

Quoted 2 lines

Quoted 2 lines
---
v2:

[SNIP]

Quoted 11 lines

doc/guides/vdpadevs/index.rst which can be used from an application through vhost
Quoted 3 lines
doc/guides/vdpadevs/index.rst +0 -0
@@ -14,3 +14,5 @@ which can be used from an application through vhost
API.
doc/guides/vdpadevs/index.rst which can be used from an application through vhost API. · line 18
Quoted 5 lines
doc/guides/vdpadevs/index.rst +2 -0
     features_overview
     ifc
     mlx5
+    sfc
+
The above blank line not needed.

Ack.

Quoted 9 lines

doc/guides/vdpadevs/sfc.rst
Quoted 7 lines
doc/guides/vdpadevs/sfc.rst +6 -0
+sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva) {
+	struct rte_pci_device *dev = sva->pdev;
+	char dev_name[RTE_DEV_NAME_MAX_LEN] = {0};
+	int rc;
+
+	if (dev == NULL)
I think it will never happen

Yes, This check can be removed.

Quoted 4 lines

doc/guides/vdpadevs/sfc.rst
Quoted 3 lines
doc/guides/vdpadevs/sfc.rst +3 -0
+		goto fail_inval;
+
+	rte_pci_device_name(&dev->addr, dev_name,
RTE_DEV_NAME_MAX_LEN);

[SNIP]

Quoted 11 lines

doc/guides/vdpadevs/sfc.rst
Quoted 9 lines
doc/guides/vdpadevs/sfc.rst +9 -0
+#include <rte_bus_pci.h>
+
+#include "sfc_vdpa_log.h"
+
+/* Adapter private data */
+struct sfc_vdpa_adapter {
+	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	struct rte_pci_device		*pdev;
+	struct rte_pci_addr		pci_addr;
I don't think pci_addr is needed, you can get it through pdev.

Ack.

Quoted 2 lines

Quoted 2 lines
doc/guides/vdpadevs/sfc.rst +2 -0
+
+	char

Vijay Srivastava

author

2021-10-29 14:48 UTC #69 lore raw

PATCH v3 2/10 vdpa/sfc: add support for device initialization

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Used rte_memzone_reserve_aligned for mcdi buffer allocation.
* Freeing mcdi buff when DMA map fails.
* Fixed one typo.

 doc/guides/vdpadevs/sfc.rst       |   6 +
 drivers/vdpa/sfc/meson.build      |   3 +
 drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
 drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
 drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
 drivers/vdpa/sfc/sfc_vdpa_hw.c    | 327 ++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
 10 files changed, 670 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

doc/guides/vdpadevs/sfc.rst +6 -0

index 44e694f..d06c427 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types available for control:
   Matches a subset of per-port log types registered during runtime.
   A full name for a particular type may be obtained by appending a
   dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.
+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +3 -0

index 4255d65..dc333de 100644
@@ -19,4 +19,7 @@ endforeach
 deps += ['common_sfc_efx', 'bus_pci']
 sources = files(
 	'sfc_vdpa.c',
+	'sfc_vdpa_hw.c',
+	'sfc_vdpa_mcdi.c',
+	'sfc_vdpa_ops.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +23 -0

index d85c52b..b7eca56 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
 		goto fail_vfio_setup;
 	}
 
+	sfc_vdpa_log_init(sva, "hw init");
+	if (sfc_vdpa_hw_init(sva) != 0) {
+		sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+		goto fail_hw_init;
+	}
+
+	sfc_vdpa_log_init(sva, "dev init");
+	sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+	if (sva->ops_data == NULL) {
+		sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+		goto fail_dev_init;
+	}
+
 	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
 	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
 
 	return 0;
 
+fail_dev_init:
+	sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+	sfc_vdpa_vfio_teardown(sva);
+
 fail_vfio_setup:
 fail_set_log_prefix:
 	rte_free(sva);

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
 	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
 
+	sfc_vdpa_device_fini(sva->ops_data);
+
+	sfc_vdpa_hw_fini(sva);
+
 	sfc_vdpa_vfio_teardown(sva);
 
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +48 -1

73 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +21 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug check
+ * in the driver only.
+ */
+#define SFC_VDPA_ASSERT(exp)			RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)			RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +327 -0

327 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_log.h +3 -0

index 858e5ee..4e7a84f 100644
@@ -21,6 +21,9 @@
 /** Name prefix for the per-device log type used to report basic information */
 #define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
 
+/** Device MCDI log type name prefix */
+#define SFC_VDPA_LOGTYPE_MCDI_STR	SFC_VDPA_LOGTYPE_PREFIX "mcdi"
+
 #define SFC_VDPA_LOG_PREFIX_MAX	32
 
 /* Log PMD message, automatically add prefix and \n */

drivers/vdpa/sfc/sfc_vdpa_mcdi.c new file +74 -0

74 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +129 -0

129 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +36 -0

new file mode 100644
index 0000000..817b302
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_OPS_H
+#define _SFC_VDPA_OPS_H
+
+#include <rte_vdpa.h>
+
+#define SFC_VDPA_MAX_QUEUE_PAIRS		1
+
+enum sfc_vdpa_context {
+	SFC_VDPA_AS_PF = 0,
+	SFC_VDPA_AS_VF
+};
+
+enum sfc_vdpa_state {
+	SFC_VDPA_STATE_UNINITIALIZED = 0,
+	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_NSTATES
+};
+
+struct sfc_vdpa_ops_data {
+	void				*dev_handle;
+	struct rte_vdpa_device		*vdpa_dev;
+	enum sfc_vdpa_context		vdpa_context;
+	enum sfc_vdpa_state		state;
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *adapter, enum sfc_vdpa_context context);
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data);
+
+#endif /* _SFC_VDPA_OPS_H */

Signature · 1 line

1.8.3.1

Mattias Rönnblom

2021-10-29 20:21 UTC #70 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Quoted 234 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Used rte_memzone_reserve_aligned for mcdi buffer allocation.
* Freeing mcdi buff when DMA map fails.
* Fixed one typo.

  doc/guides/vdpadevs/sfc.rst       |   6 +
  drivers/vdpa/sfc/meson.build      |   3 +
  drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
  drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
  drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
  drivers/vdpa/sfc/sfc_vdpa_hw.c    | 327 ++++++++++++++++++++++++++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
  drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
  drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
  drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
  10 files changed, 670 insertions(+), 1 deletion(-)

create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

doc/guides/vdpadevs/sfc.rst +6 -0

index 44e694f..d06c427 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types available for control:
    Matches a subset of per-port log types registered during runtime.
    A full name for a particular type may be obtained by appending a
    dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.
+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +3 -0

index 4255d65..dc333de 100644
@@ -19,4 +19,7 @@ endforeach
  deps += ['common_sfc_efx', 'bus_pci']
  sources = files(
  	'sfc_vdpa.c',
+	'sfc_vdpa_hw.c',
+	'sfc_vdpa_mcdi.c',
+	'sfc_vdpa_ops.c',
  )

drivers/vdpa/sfc/sfc_vdpa.c +23 -0

index d85c52b..b7eca56 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
  		goto fail_vfio_setup;
  	}
 
+	sfc_vdpa_log_init(sva, "hw init");
+	if (sfc_vdpa_hw_init(sva) != 0) {
+		sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+		goto fail_hw_init;
+	}
+
+	sfc_vdpa_log_init(sva, "dev init");
+	sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+	if (sva->ops_data == NULL) {
+		sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+		goto fail_dev_init;
+	}
+
  	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
  	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
  	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
 
  	return 0;
 
+fail_dev_init:
+	sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+	sfc_vdpa_vfio_teardown(sva);
+
  fail_vfio_setup:
  fail_set_log_prefix:
  	rte_free(sva);

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
  	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
  	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
 
+	sfc_vdpa_device_fini(sva->ops_data);
+
+	sfc_vdpa_hw_fini(sva);
+
  	sfc_vdpa_vfio_teardown(sva);
 
  	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +48 -1

index 3b77900..046f25d 100644
@@ -11,14 +11,38 @@
 
  #include <rte_bus_pci.h>
 
+#include "sfc_efx.h"
+#include "sfc_efx_mcdi.h"
+#include "sfc_vdpa_debug.h"
  #include "sfc_vdpa_log.h"
+#include "sfc_vdpa_ops.h"
+
+#define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
  /* Adapter private data */
  struct sfc_vdpa_adapter {
  	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	/*
+	 * PMD setup and configuration is not thread safe. Since it is not
+	 * performance sensitive, it is better to guarantee thread-safety
+	 * and add device level lock. vDPA control operations which
+	 * change its state should acquire the lock.
+	 */
+	rte_spinlock_t			lock;
  	struct rte_pci_device		*pdev;
  	struct rte_pci_addr		pci_addr;
 
+	efx_family_t			family;
+	efx_nic_t			*nic;
+	rte_spinlock_t			nic_lock;
+
+	efsys_bar_t			mem_bar;
+
+	struct sfc_efx_mcdi		mcdi;
+	size_t				mcdi_buff_size;
+
+	uint32_t			max_queue_count;
+
  	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
  	uint32_t			logtype_main;

@@ -26,6 +50,7 @@ struct sfc_vdpa_adapter {
  	int				vfio_dev_fd;
  	int				vfio_container_fd;
  	int				iommu_group_num;
+	struct sfc_vdpa_ops_data	*ops_data;
  };
 
  uint32_t

@@ -36,5 +61,27 @@ struct sfc_vdpa_adapter {
  struct sfc_vdpa_adapter *
  sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
 
-#endif  /* _SFC_VDPA_H */
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sva);
 
+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva);
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp);
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
+
+static inline struct sfc_vdpa_adapter *
+sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
+{
+	return (struct sfc_vdpa_adapter *)dev_handle;
+}
+
+#endif  /* _SFC_VDPA_H */

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +21 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug check
+ * in the driver only.
+ */
+#define SFC_VDPA_ASSERT(exp)			RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)			RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +18 -0

new file mode 100644
index 0000000..7c256ff
@@ -0,0 +1,327 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_vfio.h>
+
+#include "efx.h"
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_ops.h"
+
+extern uint32_t sfc_logtype_driver;
+
+#ifndef PAGE_SIZE

drivers/vdpa/sfc/sfc_vdpa_hw.c line 28

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+#define PAGE_SIZE   (sysconf(_SC_PAGESIZE))
+#endif
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp)
+{
+	uint64_t mcdi_iova;
+	size_t mcdi_buff_size;
+	const struct rte_memzone *mz = NULL;

Redundant initialization.

Quoted 37 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +37 -0

+	int numa_node = sva->pdev->device.numa_node;
+	int ret;
+
+	mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+
+	sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+	mz = rte_memzone_reserve_aligned(name, mcdi_buff_size,
+					 numa_node,
+					 RTE_MEMZONE_IOVA_CONTIG,
+					 PAGE_SIZE);
+	if (mz == NULL) {
+		sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x: %s",
+			     name, (unsigned int)len, rte_strerror(rte_errno));
+		return -ENOMEM;
+	}
+
+	/* IOVA address for MCDI would be re-calculated if mapping
+	 * using default IOVA would fail.
+	 * TODO: Earlier there was no way to get valid IOVA range.
+	 * Recently a patch has been submitted to get the IOVA range
+	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+	 * in the kernel version >= 5.4. Support to get the default
+	 * IOVA address for MCDI buffer using available IOVA range
+	 * would be added later. Meanwhile default IOVA for MCDI buffer
+	 * is kept at high mem at 2TB. In case of overlap new available
+	 * addresses would be searched and same would be used.
+	 */
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mz->addr, mcdi_iova,
+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+

drivers/vdpa/sfc/sfc_vdpa_hw.c line 75

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			rte_memzone_free(mz);
+			return ret;
+		}
+
+	} while (ret < 0);

"ret < 0" will never evaluate to false here. Use "for (;;)" instead.

Quoted 76 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +76 -0

+
+	esmp->esm_addr = mcdi_iova;
+	esmp->esm_base = mz->addr;
+	sva->mcdi_buff_size = mcdi_buff_size;
+
+	sfc_vdpa_info(sva,
+		      "DMA name=%s len=%zu => virt=%p iova=%" PRIx64,
+		      name, len, esmp->esm_base, esmp->esm_addr);
+
+	return 0;
+}
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp)
+{
+	int ret;
+
+	sfc_vdpa_log_init(sva, "name=%s", esmp->esm_mz->name);
+
+	ret = rte_vfio_container_dma_unmap(sva->vfio_container_fd,
+					   (uint64_t)esmp->esm_base,
+					   esmp->esm_addr, sva->mcdi_buff_size);
+	if (ret < 0)
+		sfc_vdpa_err(sva, "DMA unmap failed for MCDI : %s",
+			     rte_strerror(rte_errno));
+
+	sfc_vdpa_info(sva,
+		      "DMA free name=%s => virt=%p iova=%" PRIx64,
+		      esmp->esm_mz->name, esmp->esm_base, esmp->esm_addr);
+
+	rte_free((void *)(esmp->esm_base));
+
+	sva->mcdi_buff_size = 0;
+	memset(esmp, 0, sizeof(*esmp));
+}
+
+static int
+sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
+		      const efx_bar_region_t *mem_ebrp)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	efsys_bar_t *ebp = &sva->mem_bar;
+	struct rte_mem_resource *res =
+		&pci_dev->mem_resource[mem_ebrp->ebr_index];
+
+	SFC_BAR_LOCK_INIT(ebp, pci_dev->name);
+	ebp->esb_rid = mem_ebrp->ebr_index;
+	ebp->esb_dev = pci_dev;
+	ebp->esb_base = res->addr;
+
+	return 0;
+}
+
+static void
+sfc_vdpa_mem_bar_fini(struct sfc_vdpa_adapter *sva)
+{
+	efsys_bar_t *ebp = &sva->mem_bar;
+
+	SFC_BAR_LOCK_DESTROY(ebp);
+	memset(ebp, 0, sizeof(*ebp));
+}
+
+static int
+sfc_vdpa_nic_probe(struct sfc_vdpa_adapter *sva)
+{
+	efx_nic_t *enp = sva->nic;
+	int rc;
+
+	rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE);
+	if (rc != 0)
+		sfc_vdpa_err(sva, "nic probe failed: %s", rte_strerror(rc));
+
+	return rc;
+}
+
+static int

drivers/vdpa/sfc/sfc_vdpa_hw.c line 161

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+sfc_vdpa_estimate_resource_limits(struct sfc_vdpa_adapter *sva)
+{
+	efx_drv_limits_t limits;
+	int rc;
+	uint32_t evq_allocated;
+	uint32_t rxq_allocated;
+	uint32_t txq_allocated;
+	uint32_t max_queue_cnt;
+
+	memset(&limits, 0, sizeof(limits));

Avoid using typedefs for structs. This is a struct?

Quoted 405 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +166 -0

+
+	/* Request at least one Rx and Tx queue */
+	limits.edl_min_rxq_count = 1;
+	limits.edl_min_txq_count = 1;
+	/* Management event queue plus event queue for Tx/Rx queue */
+	limits.edl_min_evq_count =
+		1 + RTE_MAX(limits.edl_min_rxq_count, limits.edl_min_txq_count);
+
+	limits.edl_max_rxq_count = SFC_VDPA_MAX_QUEUE_PAIRS;
+	limits.edl_max_txq_count = SFC_VDPA_MAX_QUEUE_PAIRS;
+	limits.edl_max_evq_count = 1 + SFC_VDPA_MAX_QUEUE_PAIRS;
+
+	SFC_VDPA_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count);
+	SFC_VDPA_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count);
+	SFC_VDPA_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count);
+
+	/* Configure the minimum required resources needed for the
+	 * driver to operate, and the maximum desired resources that the
+	 * driver is capable of using.
+	 */
+	sfc_vdpa_log_init(sva, "set drv limit");
+	efx_nic_set_drv_limits(sva->nic, &limits);
+
+	sfc_vdpa_log_init(sva, "init nic");
+	rc = efx_nic_init(sva->nic);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "nic init failed: %s", rte_strerror(rc));
+		goto fail_nic_init;
+	}
+
+	/* Find resource dimensions assigned by firmware to this function */
+	rc = efx_nic_get_vi_pool(sva->nic, &evq_allocated, &rxq_allocated,
+				 &txq_allocated);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "vi pool get failed: %s", rte_strerror(rc));
+		goto fail_get_vi_pool;
+	}
+
+	/* It still may allocate more than maximum, ensure limit */
+	evq_allocated = RTE_MIN(evq_allocated, limits.edl_max_evq_count);
+	rxq_allocated = RTE_MIN(rxq_allocated, limits.edl_max_rxq_count);
+	txq_allocated = RTE_MIN(txq_allocated, limits.edl_max_txq_count);
+
+
+	max_queue_cnt = RTE_MIN(rxq_allocated, txq_allocated);
+	/* Subtract management EVQ not used for traffic */
+	max_queue_cnt = RTE_MIN(evq_allocated - 1, max_queue_cnt);
+
+	SFC_VDPA_ASSERT(max_queue_cnt > 0);
+
+	sva->max_queue_count = max_queue_cnt;
+
+	return 0;
+
+fail_get_vi_pool:
+	efx_nic_fini(sva->nic);
+fail_nic_init:
+	sfc_vdpa_log_init(sva, "failed: %s", rte_strerror(rc));
+	return rc;
+}
+
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva)
+{
+	efx_bar_region_t mem_ebr;
+	efx_nic_t *enp;
+	int rc;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "get family");
+	rc = sfc_efx_family(sva->pdev, &mem_ebr, &sva->family);
+	if (rc != 0)
+		goto fail_family;
+	sfc_vdpa_log_init(sva,
+			  "family is %u, membar is %u,"
+			  "function control window offset is %#" PRIx64,
+			  sva->family, mem_ebr.ebr_index, mem_ebr.ebr_offset);
+
+	sfc_vdpa_log_init(sva, "init mem bar");
+	rc = sfc_vdpa_mem_bar_init(sva, &mem_ebr);
+	if (rc != 0)
+		goto fail_mem_bar_init;
+
+	sfc_vdpa_log_init(sva, "create nic");
+	rte_spinlock_init(&sva->nic_lock);
+	rc = efx_nic_create(sva->family, (efsys_identifier_t *)sva,
+			    &sva->mem_bar, mem_ebr.ebr_offset,
+			    &sva->nic_lock, &enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "nic create failed: %s", rte_strerror(rc));
+		goto fail_nic_create;
+	}
+	sva->nic = enp;
+
+	sfc_vdpa_log_init(sva, "init mcdi");
+	rc = sfc_vdpa_mcdi_init(sva);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "mcdi init failed: %s", rte_strerror(rc));
+		goto fail_mcdi_init;
+	}
+
+	sfc_vdpa_log_init(sva, "probe nic");
+	rc = sfc_vdpa_nic_probe(sva);
+	if (rc != 0)
+		goto fail_nic_probe;
+
+	sfc_vdpa_log_init(sva, "reset nic");
+	rc = efx_nic_reset(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "nic reset failed: %s", rte_strerror(rc));
+		goto fail_nic_reset;
+	}
+
+	sfc_vdpa_log_init(sva, "estimate resource limits");
+	rc = sfc_vdpa_estimate_resource_limits(sva);
+	if (rc != 0)
+		goto fail_estimate_rsrc_limits;
+
+	sfc_vdpa_log_init(sva, "done");
+
+	return 0;
+
+fail_estimate_rsrc_limits:
+fail_nic_reset:
+	efx_nic_unprobe(enp);
+
+fail_nic_probe:
+	sfc_vdpa_mcdi_fini(sva);
+
+fail_mcdi_init:
+	sfc_vdpa_log_init(sva, "destroy nic");
+	sva->nic = NULL;
+	efx_nic_destroy(enp);
+
+fail_nic_create:
+	sfc_vdpa_mem_bar_fini(sva);
+
+fail_mem_bar_init:
+fail_family:
+	sfc_vdpa_log_init(sva, "failed: %s", rte_strerror(rc));
+	return rc;
+}
+
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sva)
+{
+	efx_nic_t *enp = sva->nic;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "unprobe nic");
+	efx_nic_unprobe(enp);
+
+	sfc_vdpa_log_init(sva, "mcdi fini");
+	sfc_vdpa_mcdi_fini(sva);
+
+	sfc_vdpa_log_init(sva, "nic fini");
+	efx_nic_fini(enp);
+
+	sfc_vdpa_log_init(sva, "destroy nic");
+	sva->nic = NULL;
+	efx_nic_destroy(enp);
+
+	sfc_vdpa_mem_bar_fini(sva);
+}

drivers/vdpa/sfc/sfc_vdpa_log.h +3 -0

index 858e5ee..4e7a84f 100644
@@ -21,6 +21,9 @@
  /** Name prefix for the per-device log type used to report basic information */
  #define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
 
+/** Device MCDI log type name prefix */
+#define SFC_VDPA_LOGTYPE_MCDI_STR	SFC_VDPA_LOGTYPE_PREFIX "mcdi"
+
  #define SFC_VDPA_LOG_PREFIX_MAX	32
 
  /* Log PMD message, automatically add prefix and \n */

drivers/vdpa/sfc/sfc_vdpa_mcdi.c new file +74 -0

new file mode 100644
index 0000000..961d2d3
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include "sfc_efx_mcdi.h"
+
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_debug.h"
+#include "sfc_vdpa_log.h"
+
+static sfc_efx_mcdi_dma_alloc_cb sfc_vdpa_mcdi_dma_alloc;
+static int
+sfc_vdpa_mcdi_dma_alloc(void *cookie, const char *name, size_t len,
+			efsys_mem_t *esmp)
+{
+	struct sfc_vdpa_adapter *sva = cookie;
+
+	return sfc_vdpa_dma_alloc(sva, name, len, esmp);
+}
+
+static sfc_efx_mcdi_dma_free_cb sfc_vdpa_mcdi_dma_free;
+static void
+sfc_vdpa_mcdi_dma_free(void *cookie, efsys_mem_t *esmp)
+{
+	struct sfc_vdpa_adapter *sva = cookie;
+
+	sfc_vdpa_dma_free(sva, esmp);
+}
+
+static sfc_efx_mcdi_sched_restart_cb sfc_vdpa_mcdi_sched_restart;
+static void
+sfc_vdpa_mcdi_sched_restart(void *cookie)
+{
+	RTE_SET_USED(cookie);
+}
+
+static sfc_efx_mcdi_mgmt_evq_poll_cb sfc_vdpa_mcdi_mgmt_evq_poll;
+static void
+sfc_vdpa_mcdi_mgmt_evq_poll(void *cookie)
+{
+	RTE_SET_USED(cookie);
+}
+
+static const struct sfc_efx_mcdi_ops sfc_vdpa_mcdi_ops = {
+	.dma_alloc	= sfc_vdpa_mcdi_dma_alloc,
+	.dma_free	= sfc_vdpa_mcdi_dma_free,
+	.sched_restart  = sfc_vdpa_mcdi_sched_restart,
+	.mgmt_evq_poll  = sfc_vdpa_mcdi_mgmt_evq_poll,
+
+};
+
+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva)
+{
+	uint32_t logtype;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	logtype = sfc_vdpa_register_logtype(&(sva->pdev->addr),
+					    SFC_VDPA_LOGTYPE_MCDI_STR,
+					    RTE_LOG_NOTICE);
+
+	return sfc_efx_mcdi_init(&sva->mcdi, logtype,
+				 sva->log_prefix, sva->nic,
+				 &sfc_vdpa_mcdi_ops, sva);
+}
+
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva)
+{
+	sfc_vdpa_log_init(sva, "entry");
+	sfc_efx_mcdi_fini(&sva->mcdi);
+}

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +129 -0

new file mode 100644
index 0000000..71696be
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <rte_malloc.h>
+#include <rte_vdpa.h>
+#include <rte_vdpa_dev.h>
+#include <rte_vhost.h>
+
+#include "sfc_vdpa_ops.h"
+#include "sfc_vdpa.h"
+
+/* Dummy functions for mandatory vDPA ops to pass vDPA device registration.
+ * In subsequent patches these ops would be implemented.
+ */
+static int
+sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
+{
+	RTE_SET_USED(vdpa_dev);
+	RTE_SET_USED(queue_num);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t *features)
+{
+	RTE_SET_USED(vdpa_dev);
+	RTE_SET_USED(features);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_get_protocol_features(struct rte_vdpa_device *vdpa_dev,
+			       uint64_t *features)
+{
+	RTE_SET_USED(vdpa_dev);
+	RTE_SET_USED(features);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_dev_config(int vid)
+{
+	RTE_SET_USED(vid);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_dev_close(int vid)
+{
+	RTE_SET_USED(vid);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_set_vring_state(int vid, int vring, int state)
+{
+	RTE_SET_USED(vid);
+	RTE_SET_USED(vring);
+	RTE_SET_USED(state);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_set_features(int vid)
+{
+	RTE_SET_USED(vid);
+
+	return -1;
+}
+
+static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
+	.get_queue_num = sfc_vdpa_get_queue_num,
+	.get_features = sfc_vdpa_get_features,
+	.get_protocol_features = sfc_vdpa_get_protocol_features,
+	.dev_conf = sfc_vdpa_dev_config,
+	.dev_close = sfc_vdpa_dev_close,
+	.set_vring_state = sfc_vdpa_set_vring_state,
+	.set_features = sfc_vdpa_set_features,
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *dev_handle, enum sfc_vdpa_context context)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	struct rte_pci_device *pci_dev;
+
+	/* Create vDPA ops context */
+	ops_data = rte_zmalloc("vdpa", sizeof(struct sfc_vdpa_ops_data), 0);
+	if (ops_data == NULL)
+		return NULL;
+
+	ops_data->vdpa_context = context;
+	ops_data->dev_handle = dev_handle;
+
+	pci_dev = sfc_vdpa_adapter_by_dev_handle(dev_handle)->pdev;
+
+	/* Register vDPA Device */
+	sfc_vdpa_log_init(dev_handle, "register vDPA device");
+	ops_data->vdpa_dev =
+		rte_vdpa_register_device(&pci_dev->device, &sfc_vdpa_ops);
+	if (ops_data->vdpa_dev == NULL) {
+		sfc_vdpa_err(dev_handle, "vDPA device registration failed");
+		goto fail_register_device;
+	}
+
+	ops_data->state = SFC_VDPA_STATE_INITIALIZED;
+
+	return ops_data;
+
+fail_register_device:
+	rte_free(ops_data);
+	return NULL;
+}
+
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data)
+{
+	rte_vdpa_unregister_device(ops_data->vdpa_dev);
+
+	rte_free(ops_data);
+}

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +4 -0

new file mode 100644
index 0000000..817b302
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */

drivers/vdpa/sfc/sfc_vdpa_ops.h line 14

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +10 -0

+
+#ifndef _SFC_VDPA_OPS_H
+#define _SFC_VDPA_OPS_H
+
+#include <rte_vdpa.h>
+
+#define SFC_VDPA_MAX_QUEUE_PAIRS		1
+
+enum sfc_vdpa_context {
+	SFC_VDPA_AS_PF = 0,

0 is the default.

drivers/vdpa/sfc/sfc_vdpa_ops.h line 19

Quoted 5 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +5 -0

+	SFC_VDPA_AS_VF
+};
+
+enum sfc_vdpa_state {
+	SFC_VDPA_STATE_UNINITIALIZED = 0,

Same here.

Quoted 18 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +17 -0

+	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_NSTATES
+};
+
+struct sfc_vdpa_ops_data {
+	void				*dev_handle;
+	struct rte_vdpa_device		*vdpa_dev;
+	enum sfc_vdpa_context		vdpa_context;
+	enum sfc_vdpa_state		state;
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *adapter, enum sfc_vdpa_context context);
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data);
+
+#endif /* _SFC_VDPA_OPS_H */

Andrew Rybchenko

2021-11-01 08:09 UTC #71 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Hi Mattias,

Many thanks for review. Please, see below.

Andrew.

Quoted 229 lines · Mattias Rönnblom

Quoted 228 lines · Vijay Srivastava

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Used rte_memzone_reserve_aligned for mcdi buffer allocation.
* Freeing mcdi buff when DMA map fails.
* Fixed one typo.

  10 files changed, 670 insertions(+), 1 deletion(-)

doc/guides/vdpadevs/sfc.rst +0 -0

index 44e694f..d06c427 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types

available for control:
Matches a subset of per-port log types registered during runtime. A full name for a particular type may be obtained by appending a dot and a PCI device identifier (``XXXX:XX:XX.X``) to the prefix.

doc/guides/vdpadevs/sfc.rst +6 -0

+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +0 -0

index 4255d65..dc333de 100644
@@ -19,4 +19,7 @@ endforeach

deps += ['common_sfc_efx', 'bus_pci']
sources = files(
      'sfc_vdpa.c',
+    'sfc_vdpa_hw.c',
+    'sfc_vdpa_mcdi.c',
+    'sfc_vdpa_ops.c',
)

drivers/vdpa/sfc/sfc_vdpa.c +19 -0

index d85c52b..b7eca56 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
           goto fail_vfio_setup;
       }
+    sfc_vdpa_log_init(sva, "hw init");
+    if (sfc_vdpa_hw_init(sva) != 0) {
+        sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+        goto fail_hw_init;
+    }
+
+    sfc_vdpa_log_init(sva, "dev init");
+    sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+    if (sva->ops_data == NULL) {
+        sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+        goto fail_dev_init;
+    }
+
       pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
       TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
       pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
       return 0;
+fail_dev_init:
+    sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+    sfc_vdpa_vfio_teardown(sva);
+

fail_vfio_setup:
fail_set_log_prefix:
rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.c +4 -0

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
       TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
       pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+    sfc_vdpa_device_fini(sva->ops_data);
+
+    sfc_vdpa_hw_fini(sva);
+
       sfc_vdpa_vfio_teardown(sva);
       rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +25 -0

index 3b77900..046f25d 100644
@@ -11,14 +11,38 @@
   #include <rte_bus_pci.h>
+#include "sfc_efx.h"
+#include "sfc_efx_mcdi.h"
+#include "sfc_vdpa_debug.h"
   #include "sfc_vdpa_log.h"
+#include "sfc_vdpa_ops.h"
+
+#define SFC_VDPA_DEFAULT_MCDI_IOVA        0x200000000000
   /* Adapter private data */
   struct sfc_vdpa_adapter {
       TAILQ_ENTRY(sfc_vdpa_adapter)    next;
+    /*
+     * PMD setup and configuration is not thread safe. Since it is not
+     * performance sensitive, it is better to guarantee thread-safety
+     * and add device level lock. vDPA control operations which
+     * change its state should acquire the lock.
+     */
+    rte_spinlock_t            lock;
       struct rte_pci_device        *pdev;
       struct rte_pci_addr        pci_addr;
+    efx_family_t            family;
+    efx_nic_t            *nic;
+    rte_spinlock_t            nic_lock;
+
+    efsys_bar_t            mem_bar;
+
+    struct sfc_efx_mcdi        mcdi;
+    size_t                mcdi_buff_size;
+
+    uint32_t            max_queue_count;
+
       char                log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
       uint32_t            logtype_main;

@@ -26,6 +50,7 @@ struct sfc_vdpa_adapter {
       int                vfio_dev_fd;
       int                vfio_container_fd;
       int                iommu_group_num;
+    struct sfc_vdpa_ops_data    *ops_data;
   };

uint32_t

drivers/vdpa/sfc/sfc_vdpa.h +0 -0

@@ -36,5 +61,27 @@ struct sfc_vdpa_adapter {

struct sfc_vdpa_adapter * sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);

drivers/vdpa/sfc/sfc_vdpa.h +23 -1

-#endif  /* _SFC_VDPA_H */
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sva);
+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva);
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+           size_t len, efsys_mem_t *esmp);
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
+
+static inline struct sfc_vdpa_adapter *
+sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
+{
+    return (struct sfc_vdpa_adapter *)dev_handle;
+}
+
+#endif  /* _SFC_VDPA_H */

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +12 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug

check
+ * in the driver only.
+ */

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +7 -0

+#define SFC_VDPA_ASSERT(exp)            RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)            RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +22 -0

new file mode 100644
index 0000000..7c256ff
@@ -0,0 +1,327 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_vfio.h>
+
+#include "efx.h"
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_ops.h"
+
+extern uint32_t sfc_logtype_driver;
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE   (sysconf(_SC_PAGESIZE))
+#endif
+
+int

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c line 28
Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +6 -0
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+           size_t len, efsys_mem_t *esmp)
+{
+    uint64_t mcdi_iova;
+    size_t mcdi_buff_size;
+    const struct rte_memzone *mz = NULL;
Redundant initialization.

Is it a problem? Is it defined in DPDK coding style to avoid it?

Quoted 42 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +41 -0

+    int numa_node = sva->pdev->device.numa_node;
+    int ret;
+
+    mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+
+    sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+    mz = rte_memzone_reserve_aligned(name, mcdi_buff_size,
+                     numa_node,
+                     RTE_MEMZONE_IOVA_CONTIG,
+                     PAGE_SIZE);
+    if (mz == NULL) {
+        sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x: %s",
+                 name, (unsigned int)len, rte_strerror(rte_errno));
+        return -ENOMEM;
+    }
+
+    /* IOVA address for MCDI would be re-calculated if mapping
+     * using default IOVA would fail.
+     * TODO: Earlier there was no way to get valid IOVA range.
+     * Recently a patch has been submitted to get the IOVA range
+     * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+     * in the kernel version >= 5.4. Support to get the default
+     * IOVA address for MCDI buffer using available IOVA range
+     * would be added later. Meanwhile default IOVA for MCDI buffer
+     * is kept at high mem at 2TB. In case of overlap new available
+     * addresses would be searched and same would be used.
+     */
+    mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+    do {
+        ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+                         (uint64_t)mz->addr, mcdi_iova,
+                         mcdi_buff_size);
+        if (ret == 0)
+            break;
+
+        mcdi_iova = mcdi_iova >> 1;
+        if (mcdi_iova < mcdi_buff_size)    {
+            sfc_vdpa_err(sva,
+                     "DMA mapping failed for MCDI : %s",

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c line 75
Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +6 -0
+                     rte_strerror(rte_errno));
+            rte_memzone_free(mz);
+            return ret;
+        }
+
+    } while (ret < 0);
"ret < 0" will never evaluate to false here. Use "for (;;)" instead.

Ack.

Quoted 81 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +80 -0

+
+    esmp->esm_addr = mcdi_iova;
+    esmp->esm_base = mz->addr;
+    sva->mcdi_buff_size = mcdi_buff_size;
+
+    sfc_vdpa_info(sva,
+              "DMA name=%s len=%zu => virt=%p iova=%" PRIx64,
+              name, len, esmp->esm_base, esmp->esm_addr);
+
+    return 0;
+}
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp)
+{
+    int ret;
+
+    sfc_vdpa_log_init(sva, "name=%s", esmp->esm_mz->name);
+
+    ret = rte_vfio_container_dma_unmap(sva->vfio_container_fd,
+                       (uint64_t)esmp->esm_base,
+                       esmp->esm_addr, sva->mcdi_buff_size);
+    if (ret < 0)
+        sfc_vdpa_err(sva, "DMA unmap failed for MCDI : %s",
+                 rte_strerror(rte_errno));
+
+    sfc_vdpa_info(sva,
+              "DMA free name=%s => virt=%p iova=%" PRIx64,
+              esmp->esm_mz->name, esmp->esm_base, esmp->esm_addr);
+
+    rte_free((void *)(esmp->esm_base));
+
+    sva->mcdi_buff_size = 0;
+    memset(esmp, 0, sizeof(*esmp));
+}
+
+static int
+sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
+              const efx_bar_region_t *mem_ebrp)
+{
+    struct rte_pci_device *pci_dev = sva->pdev;
+    efsys_bar_t *ebp = &sva->mem_bar;
+    struct rte_mem_resource *res =
+        &pci_dev->mem_resource[mem_ebrp->ebr_index];
+
+    SFC_BAR_LOCK_INIT(ebp, pci_dev->name);
+    ebp->esb_rid = mem_ebrp->ebr_index;
+    ebp->esb_dev = pci_dev;
+    ebp->esb_base = res->addr;
+
+    return 0;
+}
+
+static void
+sfc_vdpa_mem_bar_fini(struct sfc_vdpa_adapter *sva)
+{
+    efsys_bar_t *ebp = &sva->mem_bar;
+
+    SFC_BAR_LOCK_DESTROY(ebp);
+    memset(ebp, 0, sizeof(*ebp));
+}
+
+static int
+sfc_vdpa_nic_probe(struct sfc_vdpa_adapter *sva)
+{
+    efx_nic_t *enp = sva->nic;
+    int rc;
+
+    rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE);
+    if (rc != 0)
+        sfc_vdpa_err(sva, "nic probe failed: %s", rte_strerror(rc));
+
+    return rc;
+}
+
+static int
+sfc_vdpa_estimate_resource_limits(struct sfc_vdpa_adapter *sva)
+{
+    efx_drv_limits_t limits;
+    int rc;

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c line 161
Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +6 -0
+    uint32_t evq_allocated;
+    uint32_t rxq_allocated;
+    uint32_t txq_allocated;
+    uint32_t max_queue_cnt;
+
+    memset(&limits, 0, sizeof(limits));
Avoid using typedefs for structs. This is a struct?

Yes, it is a struct. But it is an interface of the base driver which is out of scope of DPDK coding style.

Quoted 414 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +13 -0

+
+    /* Request at least one Rx and Tx queue */
+    limits.edl_min_rxq_count = 1;
+    limits.edl_min_txq_count = 1;
+    /* Management event queue plus event queue for Tx/Rx queue */
+    limits.edl_min_evq_count =
+        1 + RTE_MAX(limits.edl_min_rxq_count, limits.edl_min_txq_count);
+
+    limits.edl_max_rxq_count = SFC_VDPA_MAX_QUEUE_PAIRS;
+    limits.edl_max_txq_count = SFC_VDPA_MAX_QUEUE_PAIRS;
+    limits.edl_max_evq_count = 1 + SFC_VDPA_MAX_QUEUE_PAIRS;
+
+    SFC_VDPA_ASSERT(limits.edl_max_evq_count >=

limits.edl_min_rxq_count);
+ SFC_VDPA_ASSERT(limits.edl_max_rxq_count >=
limits.edl_min_rxq_count);
+ SFC_VDPA_ASSERT(limits.edl_max_txq_count >=
limits.edl_min_rxq_count);

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +151 -0

+
+    /* Configure the minimum required resources needed for the
+     * driver to operate, and the maximum desired resources that the
+     * driver is capable of using.
+     */
+    sfc_vdpa_log_init(sva, "set drv limit");
+    efx_nic_set_drv_limits(sva->nic, &limits);
+
+    sfc_vdpa_log_init(sva, "init nic");
+    rc = efx_nic_init(sva->nic);
+    if (rc != 0) {
+        sfc_vdpa_err(sva, "nic init failed: %s", rte_strerror(rc));
+        goto fail_nic_init;
+    }
+
+    /* Find resource dimensions assigned by firmware to this function */
+    rc = efx_nic_get_vi_pool(sva->nic, &evq_allocated, &rxq_allocated,
+                 &txq_allocated);
+    if (rc != 0) {
+        sfc_vdpa_err(sva, "vi pool get failed: %s", rte_strerror(rc));
+        goto fail_get_vi_pool;
+    }
+
+    /* It still may allocate more than maximum, ensure limit */
+    evq_allocated = RTE_MIN(evq_allocated, limits.edl_max_evq_count);
+    rxq_allocated = RTE_MIN(rxq_allocated, limits.edl_max_rxq_count);
+    txq_allocated = RTE_MIN(txq_allocated, limits.edl_max_txq_count);
+
+
+    max_queue_cnt = RTE_MIN(rxq_allocated, txq_allocated);
+    /* Subtract management EVQ not used for traffic */
+    max_queue_cnt = RTE_MIN(evq_allocated - 1, max_queue_cnt);
+
+    SFC_VDPA_ASSERT(max_queue_cnt > 0);
+
+    sva->max_queue_count = max_queue_cnt;
+
+    return 0;
+
+fail_get_vi_pool:
+    efx_nic_fini(sva->nic);
+fail_nic_init:
+    sfc_vdpa_log_init(sva, "failed: %s", rte_strerror(rc));
+    return rc;
+}
+
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva)
+{
+    efx_bar_region_t mem_ebr;
+    efx_nic_t *enp;
+    int rc;
+
+    sfc_vdpa_log_init(sva, "entry");
+
+    sfc_vdpa_log_init(sva, "get family");
+    rc = sfc_efx_family(sva->pdev, &mem_ebr, &sva->family);
+    if (rc != 0)
+        goto fail_family;
+    sfc_vdpa_log_init(sva,
+              "family is %u, membar is %u,"
+              "function control window offset is %#" PRIx64,
+              sva->family, mem_ebr.ebr_index, mem_ebr.ebr_offset);
+
+    sfc_vdpa_log_init(sva, "init mem bar");
+    rc = sfc_vdpa_mem_bar_init(sva, &mem_ebr);
+    if (rc != 0)
+        goto fail_mem_bar_init;
+
+    sfc_vdpa_log_init(sva, "create nic");
+    rte_spinlock_init(&sva->nic_lock);
+    rc = efx_nic_create(sva->family, (efsys_identifier_t *)sva,
+                &sva->mem_bar, mem_ebr.ebr_offset,
+                &sva->nic_lock, &enp);
+    if (rc != 0) {
+        sfc_vdpa_err(sva, "nic create failed: %s", rte_strerror(rc));
+        goto fail_nic_create;
+    }
+    sva->nic = enp;
+
+    sfc_vdpa_log_init(sva, "init mcdi");
+    rc = sfc_vdpa_mcdi_init(sva);
+    if (rc != 0) {
+        sfc_vdpa_err(sva, "mcdi init failed: %s", rte_strerror(rc));
+        goto fail_mcdi_init;
+    }
+
+    sfc_vdpa_log_init(sva, "probe nic");
+    rc = sfc_vdpa_nic_probe(sva);
+    if (rc != 0)
+        goto fail_nic_probe;
+
+    sfc_vdpa_log_init(sva, "reset nic");
+    rc = efx_nic_reset(enp);
+    if (rc != 0) {
+        sfc_vdpa_err(sva, "nic reset failed: %s", rte_strerror(rc));
+        goto fail_nic_reset;
+    }
+
+    sfc_vdpa_log_init(sva, "estimate resource limits");
+    rc = sfc_vdpa_estimate_resource_limits(sva);
+    if (rc != 0)
+        goto fail_estimate_rsrc_limits;
+
+    sfc_vdpa_log_init(sva, "done");
+
+    return 0;
+
+fail_estimate_rsrc_limits:
+fail_nic_reset:
+    efx_nic_unprobe(enp);
+
+fail_nic_probe:
+    sfc_vdpa_mcdi_fini(sva);
+
+fail_mcdi_init:
+    sfc_vdpa_log_init(sva, "destroy nic");
+    sva->nic = NULL;
+    efx_nic_destroy(enp);
+
+fail_nic_create:
+    sfc_vdpa_mem_bar_fini(sva);
+
+fail_mem_bar_init:
+fail_family:
+    sfc_vdpa_log_init(sva, "failed: %s", rte_strerror(rc));
+    return rc;
+}
+
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sva)
+{
+    efx_nic_t *enp = sva->nic;
+
+    sfc_vdpa_log_init(sva, "entry");
+
+    sfc_vdpa_log_init(sva, "unprobe nic");
+    efx_nic_unprobe(enp);
+
+    sfc_vdpa_log_init(sva, "mcdi fini");
+    sfc_vdpa_mcdi_fini(sva);
+
+    sfc_vdpa_log_init(sva, "nic fini");
+    efx_nic_fini(enp);
+
+    sfc_vdpa_log_init(sva, "destroy nic");
+    sva->nic = NULL;
+    efx_nic_destroy(enp);
+
+    sfc_vdpa_mem_bar_fini(sva);
+}

drivers/vdpa/sfc/sfc_vdpa_log.h +3 -0

index 858e5ee..4e7a84f 100644
@@ -21,6 +21,9 @@
   /** Name prefix for the per-device log type used to report basic information */
   #define SFC_VDPA_LOGTYPE_MAIN_STR    SFC_VDPA_LOGTYPE_PREFIX "main"
+/** Device MCDI log type name prefix */
+#define SFC_VDPA_LOGTYPE_MCDI_STR    SFC_VDPA_LOGTYPE_PREFIX "mcdi"
+
   #define SFC_VDPA_LOG_PREFIX_MAX    32
   /* Log PMD message, automatically add prefix and \n */

drivers/vdpa/sfc/sfc_vdpa_mcdi.c new file +74 -0

new file mode 100644
index 0000000..961d2d3
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include "sfc_efx_mcdi.h"
+
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_debug.h"
+#include "sfc_vdpa_log.h"
+
+static sfc_efx_mcdi_dma_alloc_cb sfc_vdpa_mcdi_dma_alloc;
+static int
+sfc_vdpa_mcdi_dma_alloc(void *cookie, const char *name, size_t len,
+            efsys_mem_t *esmp)
+{
+    struct sfc_vdpa_adapter *sva = cookie;
+
+    return sfc_vdpa_dma_alloc(sva, name, len, esmp);
+}
+
+static sfc_efx_mcdi_dma_free_cb sfc_vdpa_mcdi_dma_free;
+static void
+sfc_vdpa_mcdi_dma_free(void *cookie, efsys_mem_t *esmp)
+{
+    struct sfc_vdpa_adapter *sva = cookie;
+
+    sfc_vdpa_dma_free(sva, esmp);
+}
+
+static sfc_efx_mcdi_sched_restart_cb sfc_vdpa_mcdi_sched_restart;
+static void
+sfc_vdpa_mcdi_sched_restart(void *cookie)
+{
+    RTE_SET_USED(cookie);
+}
+
+static sfc_efx_mcdi_mgmt_evq_poll_cb sfc_vdpa_mcdi_mgmt_evq_poll;
+static void
+sfc_vdpa_mcdi_mgmt_evq_poll(void *cookie)
+{
+    RTE_SET_USED(cookie);
+}
+
+static const struct sfc_efx_mcdi_ops sfc_vdpa_mcdi_ops = {
+    .dma_alloc    = sfc_vdpa_mcdi_dma_alloc,
+    .dma_free    = sfc_vdpa_mcdi_dma_free,
+    .sched_restart  = sfc_vdpa_mcdi_sched_restart,
+    .mgmt_evq_poll  = sfc_vdpa_mcdi_mgmt_evq_poll,
+
+};
+
+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva)
+{
+    uint32_t logtype;
+
+    sfc_vdpa_log_init(sva, "entry");
+
+    logtype = sfc_vdpa_register_logtype(&(sva->pdev->addr),
+                        SFC_VDPA_LOGTYPE_MCDI_STR,
+                        RTE_LOG_NOTICE);
+
+    return sfc_efx_mcdi_init(&sva->mcdi, logtype,
+                 sva->log_prefix, sva->nic,
+                 &sfc_vdpa_mcdi_ops, sva);
+}
+
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva)
+{
+    sfc_vdpa_log_init(sva, "entry");
+    sfc_efx_mcdi_fini(&sva->mcdi);
+}

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +14 -0

new file mode 100644
index 0000000..71696be
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <rte_malloc.h>
+#include <rte_vdpa.h>
+#include <rte_vdpa_dev.h>
+#include <rte_vhost.h>
+
+#include "sfc_vdpa_ops.h"
+#include "sfc_vdpa.h"
+
+/* Dummy functions for mandatory vDPA ops to pass vDPA device

registration.
+ * In subsequent patches these ops would be implemented.
+ */

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +2 -0

+static int
+sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t

*queue_num)

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +9 -0

+{
+    RTE_SET_USED(vdpa_dev);
+    RTE_SET_USED(queue_num);
+
+    return -1;
+}
+
+static int
+sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t

*features)

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +102 -0

+{
+    RTE_SET_USED(vdpa_dev);
+    RTE_SET_USED(features);
+
+    return -1;
+}
+
+static int
+sfc_vdpa_get_protocol_features(struct rte_vdpa_device *vdpa_dev,
+                   uint64_t *features)
+{
+    RTE_SET_USED(vdpa_dev);
+    RTE_SET_USED(features);
+
+    return -1;
+}
+
+static int
+sfc_vdpa_dev_config(int vid)
+{
+    RTE_SET_USED(vid);
+
+    return -1;
+}
+
+static int
+sfc_vdpa_dev_close(int vid)
+{
+    RTE_SET_USED(vid);
+
+    return -1;
+}
+
+static int
+sfc_vdpa_set_vring_state(int vid, int vring, int state)
+{
+    RTE_SET_USED(vid);
+    RTE_SET_USED(vring);
+    RTE_SET_USED(state);
+
+    return -1;
+}
+
+static int
+sfc_vdpa_set_features(int vid)
+{
+    RTE_SET_USED(vid);
+
+    return -1;
+}
+
+static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
+    .get_queue_num = sfc_vdpa_get_queue_num,
+    .get_features = sfc_vdpa_get_features,
+    .get_protocol_features = sfc_vdpa_get_protocol_features,
+    .dev_conf = sfc_vdpa_dev_config,
+    .dev_close = sfc_vdpa_dev_close,
+    .set_vring_state = sfc_vdpa_set_vring_state,
+    .set_features = sfc_vdpa_set_features,
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *dev_handle, enum sfc_vdpa_context context)
+{
+    struct sfc_vdpa_ops_data *ops_data;
+    struct rte_pci_device *pci_dev;
+
+    /* Create vDPA ops context */
+    ops_data = rte_zmalloc("vdpa", sizeof(struct sfc_vdpa_ops_data), 0);
+    if (ops_data == NULL)
+        return NULL;
+
+    ops_data->vdpa_context = context;
+    ops_data->dev_handle = dev_handle;
+
+    pci_dev = sfc_vdpa_adapter_by_dev_handle(dev_handle)->pdev;
+
+    /* Register vDPA Device */
+    sfc_vdpa_log_init(dev_handle, "register vDPA device");
+    ops_data->vdpa_dev =
+        rte_vdpa_register_device(&pci_dev->device, &sfc_vdpa_ops);
+    if (ops_data->vdpa_dev == NULL) {
+        sfc_vdpa_err(dev_handle, "vDPA device registration failed");
+        goto fail_register_device;
+    }
+
+    ops_data->state = SFC_VDPA_STATE_INITIALIZED;
+
+    return ops_data;
+
+fail_register_device:
+    rte_free(ops_data);
+    return NULL;
+}
+
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data)
+{
+    rte_vdpa_unregister_device(ops_data->vdpa_dev);
+
+    rte_free(ops_data);
+}

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +8 -0

new file mode 100644
index 0000000..817b302
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_OPS_H
+#define _SFC_VDPA_OPS_H
+

Quoted 8 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h line 14
Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa_ops.h new file +6 -0
+#include <rte_vdpa.h>
+
+#define SFC_VDPA_MAX_QUEUE_PAIRS        1
+
+enum sfc_vdpa_context {
+    SFC_VDPA_AS_PF = 0,
0 is the default.

There are a number of coding standards which recommnd to initalize the first member to make it easier for less skilled reader to calculate enum values.

So, I don't think it is a problem to initalize.

Quoted 27 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h line 19

Quoted 6 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +5 -0

+    SFC_VDPA_AS_VF
+};
+
+enum sfc_vdpa_state {
+    SFC_VDPA_STATE_UNINITIALIZED = 0,

Same here.

Quoted 18 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +17 -0

+    SFC_VDPA_STATE_INITIALIZED,
+    SFC_VDPA_STATE_NSTATES
+};
+
+struct sfc_vdpa_ops_data {
+    void                *dev_handle;
+    struct rte_vdpa_device        *vdpa_dev;
+    enum sfc_vdpa_context        vdpa_context;
+    enum sfc_vdpa_state        state;
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *adapter, enum sfc_vdpa_context context);
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data);
+
+#endif /* _SFC_VDPA_OPS_H */

Xia, Chenbo

2021-11-01 11:48 UTC #72 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Hi Vijay,

Quoted 292 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 02/10] vdpa/sfc: add support for device initialization

From: Vijay Kumar Srivastava <[email protected]>

Add HW initialization and vDPA device registration support.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Used rte_memzone_reserve_aligned for mcdi buffer allocation.
* Freeing mcdi buff when DMA map fails.
* Fixed one typo.

 doc/guides/vdpadevs/sfc.rst       |   6 +
 drivers/vdpa/sfc/meson.build      |   3 +
 drivers/vdpa/sfc/sfc_vdpa.c       |  23 +++
 drivers/vdpa/sfc/sfc_vdpa.h       |  49 +++++-
 drivers/vdpa/sfc/sfc_vdpa_debug.h |  21 +++
 drivers/vdpa/sfc/sfc_vdpa_hw.c    | 327
++++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_log.h   |   3 +
 drivers/vdpa/sfc/sfc_vdpa_mcdi.c  |  74 +++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c   | 129 +++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.h   |  36 +++++
 10 files changed, 670 insertions(+), 1 deletion(-)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_debug.h
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_hw.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_mcdi.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.c
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_ops.h

doc/guides/vdpadevs/sfc.rst +0 -0

index 44e694f..d06c427 100644
@@ -95,3 +95,9 @@ SFC vDPA PMD provides the following log types available for

doc/guides/vdpadevs/sfc.rst +6 -0

+
+- ``pmd.vdpa.sfc.mcdi`` (default level is **notice**)
+
+  Extra logging of the communication with the NIC's management CPU.
+  The format of the log is consumed by the netlogdecode cross-platform
+  tool. May be managed per-port, as explained above.

drivers/vdpa/sfc/meson.build +3 -0

index 4255d65..dc333de 100644
@@ -19,4 +19,7 @@ endforeach
 deps += ['common_sfc_efx', 'bus_pci']
 sources = files(
 	'sfc_vdpa.c',
+	'sfc_vdpa_hw.c',
+	'sfc_vdpa_mcdi.c',
+	'sfc_vdpa_ops.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +23 -0

index d85c52b..b7eca56 100644
@@ -232,6 +232,19 @@ struct sfc_vdpa_adapter *
 		goto fail_vfio_setup;
 	}
 
+	sfc_vdpa_log_init(sva, "hw init");
+	if (sfc_vdpa_hw_init(sva) != 0) {
+		sfc_vdpa_err(sva, "failed to init HW %s", pci_dev->name);
+		goto fail_hw_init;
+	}
+
+	sfc_vdpa_log_init(sva, "dev init");
+	sva->ops_data = sfc_vdpa_device_init(sva, SFC_VDPA_AS_VF);
+	if (sva->ops_data == NULL) {
+		sfc_vdpa_err(sva, "failed vDPA dev init %s", pci_dev->name);
+		goto fail_dev_init;
+	}
+
 	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
 	TAILQ_INSERT_TAIL(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);

@@ -240,6 +253,12 @@ struct sfc_vdpa_adapter *
 
 	return 0;
 
+fail_dev_init:
+	sfc_vdpa_hw_fini(sva);
+
+fail_hw_init:
+	sfc_vdpa_vfio_teardown(sva);
+
 fail_vfio_setup:
 fail_set_log_prefix:
 	rte_free(sva);

@@ -266,6 +285,10 @@ struct sfc_vdpa_adapter *
 	TAILQ_REMOVE(&sfc_vdpa_adapter_list, sva, next);
 	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
 
+	sfc_vdpa_device_fini(sva->ops_data);
+
+	sfc_vdpa_hw_fini(sva);
+
 	sfc_vdpa_vfio_teardown(sva);
 
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +48 -1

index 3b77900..046f25d 100644
@@ -11,14 +11,38 @@
 
 #include <rte_bus_pci.h>
 
+#include "sfc_efx.h"
+#include "sfc_efx_mcdi.h"
+#include "sfc_vdpa_debug.h"
 #include "sfc_vdpa_log.h"
+#include "sfc_vdpa_ops.h"
+
+#define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
 /* Adapter private data */
 struct sfc_vdpa_adapter {
 	TAILQ_ENTRY(sfc_vdpa_adapter)	next;
+	/*
+	 * PMD setup and configuration is not thread safe. Since it is not
+	 * performance sensitive, it is better to guarantee thread-safety
+	 * and add device level lock. vDPA control operations which
+	 * change its state should acquire the lock.
+	 */
+	rte_spinlock_t			lock;
 	struct rte_pci_device		*pdev;
 	struct rte_pci_addr		pci_addr;
 
+	efx_family_t			family;
+	efx_nic_t			*nic;
+	rte_spinlock_t			nic_lock;
+
+	efsys_bar_t			mem_bar;
+
+	struct sfc_efx_mcdi		mcdi;
+	size_t				mcdi_buff_size;
+
+	uint32_t			max_queue_count;
+
 	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
 	uint32_t			logtype_main;

@@ -26,6 +50,7 @@ struct sfc_vdpa_adapter {
 	int				vfio_dev_fd;
 	int				vfio_container_fd;
 	int				iommu_group_num;
+	struct sfc_vdpa_ops_data	*ops_data;
 };
 
 uint32_t

@@ -36,5 +61,27 @@ struct sfc_vdpa_adapter {
 struct sfc_vdpa_adapter *
 sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
 
-#endif  /* _SFC_VDPA_H */
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sva);
 
+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva);
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva);
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp);
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
+
+static inline struct sfc_vdpa_adapter *
+sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
+{
+	return (struct sfc_vdpa_adapter *)dev_handle;
+}
+
+#endif  /* _SFC_VDPA_H */

drivers/vdpa/sfc/sfc_vdpa_debug.h new file +21 -0

new file mode 100644
index 0000000..cfa8cc5
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#ifndef _SFC_VDPA_DEBUG_H_
+#define _SFC_VDPA_DEBUG_H_
+
+#include <rte_debug.h>
+
+#ifdef RTE_LIBRTE_SFC_VDPA_DEBUG
+/* Avoid dependency from RTE_LOG_DP_LEVEL to be able to enable debug check
+ * in the driver only.
+ */
+#define SFC_VDPA_ASSERT(exp)			RTE_VERIFY(exp)
+#else
+/* If the driver debug is not enabled, follow DPDK debug/non-debug */
+#define SFC_VDPA_ASSERT(exp)			RTE_ASSERT(exp)
+#endif
+
+#endif /* _SFC_VDPA_DEBUG_H_ */

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +65 -0

new file mode 100644
index 0000000..7c256ff
@@ -0,0 +1,327 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_vfio.h>
+
+#include "efx.h"
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_ops.h"
+
+extern uint32_t sfc_logtype_driver;
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE   (sysconf(_SC_PAGESIZE))
+#endif
+
+int
+sfc_vdpa_dma_alloc(struct sfc_vdpa_adapter *sva, const char *name,
+		   size_t len, efsys_mem_t *esmp)
+{
+	uint64_t mcdi_iova;
+	size_t mcdi_buff_size;
+	const struct rte_memzone *mz = NULL;
+	int numa_node = sva->pdev->device.numa_node;
+	int ret;
+
+	mcdi_buff_size = RTE_ALIGN_CEIL(len, PAGE_SIZE);
+
+	sfc_vdpa_log_init(sva, "name=%s, len=%zu", name, len);
+
+	mz = rte_memzone_reserve_aligned(name, mcdi_buff_size,
+					 numa_node,
+					 RTE_MEMZONE_IOVA_CONTIG,
+					 PAGE_SIZE);
+	if (mz == NULL) {
+		sfc_vdpa_err(sva, "cannot reserve memory for %s: len=%#x: %s",
+			     name, (unsigned int)len, rte_strerror(rte_errno));
+		return -ENOMEM;
+	}
+
+	/* IOVA address for MCDI would be re-calculated if mapping
+	 * using default IOVA would fail.
+	 * TODO: Earlier there was no way to get valid IOVA range.
+	 * Recently a patch has been submitted to get the IOVA range
+	 * using ioctl. VFIO_IOMMU_GET_INFO. This patch is available
+	 * in the kernel version >= 5.4. Support to get the default
+	 * IOVA address for MCDI buffer using available IOVA range
+	 * would be added later. Meanwhile default IOVA for MCDI buffer
+	 * is kept at high mem at 2TB. In case of overlap new available
+	 * addresses would be searched and same would be used.
+	 */
+	mcdi_iova = SFC_VDPA_DEFAULT_MCDI_IOVA;
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mz->addr, mcdi_iova,
+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+

drivers/vdpa/sfc/sfc_vdpa_hw.c line 75

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			rte_memzone_free(mz);
+			return ret;
+		}
+
+	} while (ret < 0);

So when QEMU iova and mcdi_iova conflicts, you just let vdpa dev failed to configure, right?

Why not use re-mapping mcdi dma region as the solution? Any side-effect? Or you just assume conflict can hardly happen?

drivers/vdpa/sfc/sfc_vdpa_hw.c line 82

Quoted 7 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +7 -0

+
+	esmp->esm_addr = mcdi_iova;
+	esmp->esm_base = mz->addr;
+	sva->mcdi_buff_size = mcdi_buff_size;
+
+	sfc_vdpa_info(sva,
+		      "DMA name=%s len=%zu => virt=%p iova=%" PRIx64,

I believe 0x%" PRIx64 or %#" PRIx64 will be more friendly

Quoted 12 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +12 -0

+		      name, len, esmp->esm_base, esmp->esm_addr);
+
+	return 0;
+}
+
+void
+sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp)
+{
+	int ret;
+
+	sfc_vdpa_log_init(sva, "name=%s", esmp->esm_mz->name);
+

drivers/vdpa/sfc/sfc_vdpa_hw.c line 104

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	ret = rte_vfio_container_dma_unmap(sva->vfio_container_fd,
+					   (uint64_t)esmp->esm_base,
+					   esmp->esm_addr, sva->mcdi_buff_size);
+	if (ret < 0)
+		sfc_vdpa_err(sva, "DMA unmap failed for MCDI : %s",
+			     rte_strerror(rte_errno));
+
+	sfc_vdpa_info(sva,
+		      "DMA free name=%s => virt=%p iova=%" PRIx64,
+		      esmp->esm_mz->name, esmp->esm_base, esmp->esm_addr);

Ditto.

Quoted 125 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +125 -0

+
+	rte_free((void *)(esmp->esm_base));
+
+	sva->mcdi_buff_size = 0;
+	memset(esmp, 0, sizeof(*esmp));
+}
+
+static int
+sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
+		      const efx_bar_region_t *mem_ebrp)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	efsys_bar_t *ebp = &sva->mem_bar;
+	struct rte_mem_resource *res =
+		&pci_dev->mem_resource[mem_ebrp->ebr_index];
+
+	SFC_BAR_LOCK_INIT(ebp, pci_dev->name);
+	ebp->esb_rid = mem_ebrp->ebr_index;
+	ebp->esb_dev = pci_dev;
+	ebp->esb_base = res->addr;
+
+	return 0;
+}
+
+static void
+sfc_vdpa_mem_bar_fini(struct sfc_vdpa_adapter *sva)
+{
+	efsys_bar_t *ebp = &sva->mem_bar;
+
+	SFC_BAR_LOCK_DESTROY(ebp);
+	memset(ebp, 0, sizeof(*ebp));
+}
+
+static int
+sfc_vdpa_nic_probe(struct sfc_vdpa_adapter *sva)
+{
+	efx_nic_t *enp = sva->nic;
+	int rc;
+
+	rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE);
+	if (rc != 0)
+		sfc_vdpa_err(sva, "nic probe failed: %s", rte_strerror(rc));
+
+	return rc;
+}
+
+static int
+sfc_vdpa_estimate_resource_limits(struct sfc_vdpa_adapter *sva)
+{
+	efx_drv_limits_t limits;
+	int rc;
+	uint32_t evq_allocated;
+	uint32_t rxq_allocated;
+	uint32_t txq_allocated;
+	uint32_t max_queue_cnt;
+
+	memset(&limits, 0, sizeof(limits));
+
+	/* Request at least one Rx and Tx queue */
+	limits.edl_min_rxq_count = 1;
+	limits.edl_min_txq_count = 1;
+	/* Management event queue plus event queue for Tx/Rx queue */
+	limits.edl_min_evq_count =
+		1 + RTE_MAX(limits.edl_min_rxq_count, limits.edl_min_txq_count);
+
+	limits.edl_max_rxq_count = SFC_VDPA_MAX_QUEUE_PAIRS;
+	limits.edl_max_txq_count = SFC_VDPA_MAX_QUEUE_PAIRS;
+	limits.edl_max_evq_count = 1 + SFC_VDPA_MAX_QUEUE_PAIRS;
+
+	SFC_VDPA_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count);
+	SFC_VDPA_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count);
+	SFC_VDPA_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count);
+
+	/* Configure the minimum required resources needed for the
+	 * driver to operate, and the maximum desired resources that the
+	 * driver is capable of using.
+	 */
+	sfc_vdpa_log_init(sva, "set drv limit");
+	efx_nic_set_drv_limits(sva->nic, &limits);
+
+	sfc_vdpa_log_init(sva, "init nic");
+	rc = efx_nic_init(sva->nic);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "nic init failed: %s", rte_strerror(rc));
+		goto fail_nic_init;
+	}
+
+	/* Find resource dimensions assigned by firmware to this function */
+	rc = efx_nic_get_vi_pool(sva->nic, &evq_allocated, &rxq_allocated,
+				 &txq_allocated);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "vi pool get failed: %s", rte_strerror(rc));
+		goto fail_get_vi_pool;
+	}
+
+	/* It still may allocate more than maximum, ensure limit */
+	evq_allocated = RTE_MIN(evq_allocated, limits.edl_max_evq_count);
+	rxq_allocated = RTE_MIN(rxq_allocated, limits.edl_max_rxq_count);
+	txq_allocated = RTE_MIN(txq_allocated, limits.edl_max_txq_count);
+
+
+	max_queue_cnt = RTE_MIN(rxq_allocated, txq_allocated);
+	/* Subtract management EVQ not used for traffic */
+	max_queue_cnt = RTE_MIN(evq_allocated - 1, max_queue_cnt);
+
+	SFC_VDPA_ASSERT(max_queue_cnt > 0);
+
+	sva->max_queue_count = max_queue_cnt;
+
+	return 0;
+
+fail_get_vi_pool:
+	efx_nic_fini(sva->nic);
+fail_nic_init:
+	sfc_vdpa_log_init(sva, "failed: %s", rte_strerror(rc));
+	return rc;
+}
+
+int
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva)
+{
+	efx_bar_region_t mem_ebr;
+	efx_nic_t *enp;
+	int rc;
+

drivers/vdpa/sfc/sfc_vdpa_hw.c line 239

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "get family");
+	rc = sfc_efx_family(sva->pdev, &mem_ebr, &sva->family);
+	if (rc != 0)
+		goto fail_family;
+	sfc_vdpa_log_init(sva,
+			  "family is %u, membar is %u,"
+			  "function control window offset is %#" PRIx64,
+			  sva->family, mem_ebr.ebr_index, mem_ebr.ebr_offset);

If ebr_idx is int, then %u -> %d. But if it's a bar index, its type should be unsigned int and you should change the definition in sfc common code.

Quoted 329 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +88 -0

+
+	sfc_vdpa_log_init(sva, "init mem bar");
+	rc = sfc_vdpa_mem_bar_init(sva, &mem_ebr);
+	if (rc != 0)
+		goto fail_mem_bar_init;
+
+	sfc_vdpa_log_init(sva, "create nic");
+	rte_spinlock_init(&sva->nic_lock);
+	rc = efx_nic_create(sva->family, (efsys_identifier_t *)sva,
+			    &sva->mem_bar, mem_ebr.ebr_offset,
+			    &sva->nic_lock, &enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "nic create failed: %s", rte_strerror(rc));
+		goto fail_nic_create;
+	}
+	sva->nic = enp;
+
+	sfc_vdpa_log_init(sva, "init mcdi");
+	rc = sfc_vdpa_mcdi_init(sva);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "mcdi init failed: %s", rte_strerror(rc));
+		goto fail_mcdi_init;
+	}
+
+	sfc_vdpa_log_init(sva, "probe nic");
+	rc = sfc_vdpa_nic_probe(sva);
+	if (rc != 0)
+		goto fail_nic_probe;
+
+	sfc_vdpa_log_init(sva, "reset nic");
+	rc = efx_nic_reset(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "nic reset failed: %s", rte_strerror(rc));
+		goto fail_nic_reset;
+	}
+
+	sfc_vdpa_log_init(sva, "estimate resource limits");
+	rc = sfc_vdpa_estimate_resource_limits(sva);
+	if (rc != 0)
+		goto fail_estimate_rsrc_limits;
+
+	sfc_vdpa_log_init(sva, "done");
+
+	return 0;
+
+fail_estimate_rsrc_limits:
+fail_nic_reset:
+	efx_nic_unprobe(enp);
+
+fail_nic_probe:
+	sfc_vdpa_mcdi_fini(sva);
+
+fail_mcdi_init:
+	sfc_vdpa_log_init(sva, "destroy nic");
+	sva->nic = NULL;
+	efx_nic_destroy(enp);
+
+fail_nic_create:
+	sfc_vdpa_mem_bar_fini(sva);
+
+fail_mem_bar_init:
+fail_family:
+	sfc_vdpa_log_init(sva, "failed: %s", rte_strerror(rc));
+	return rc;
+}
+
+void
+sfc_vdpa_hw_fini(struct sfc_vdpa_adapter *sva)
+{
+	efx_nic_t *enp = sva->nic;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "unprobe nic");
+	efx_nic_unprobe(enp);
+
+	sfc_vdpa_log_init(sva, "mcdi fini");
+	sfc_vdpa_mcdi_fini(sva);
+
+	sfc_vdpa_log_init(sva, "nic fini");
+	efx_nic_fini(enp);
+
+	sfc_vdpa_log_init(sva, "destroy nic");
+	sva->nic = NULL;
+	efx_nic_destroy(enp);
+
+	sfc_vdpa_mem_bar_fini(sva);
+}

drivers/vdpa/sfc/sfc_vdpa_log.h +3 -0

index 858e5ee..4e7a84f 100644
@@ -21,6 +21,9 @@
 /** Name prefix for the per-device log type used to report basic information
 */
 #define SFC_VDPA_LOGTYPE_MAIN_STR	SFC_VDPA_LOGTYPE_PREFIX "main"
 
+/** Device MCDI log type name prefix */
+#define SFC_VDPA_LOGTYPE_MCDI_STR	SFC_VDPA_LOGTYPE_PREFIX "mcdi"
+
 #define SFC_VDPA_LOG_PREFIX_MAX	32

/* Log PMD message, automatically add prefix and \n */

drivers/vdpa/sfc/sfc_vdpa_mcdi.c new file +74 -0

new file mode 100644
index 0000000..961d2d3
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include "sfc_efx_mcdi.h"
+
+#include "sfc_vdpa.h"
+#include "sfc_vdpa_debug.h"
+#include "sfc_vdpa_log.h"
+
+static sfc_efx_mcdi_dma_alloc_cb sfc_vdpa_mcdi_dma_alloc;
+static int
+sfc_vdpa_mcdi_dma_alloc(void *cookie, const char *name, size_t len,
+			efsys_mem_t *esmp)
+{
+	struct sfc_vdpa_adapter *sva = cookie;
+
+	return sfc_vdpa_dma_alloc(sva, name, len, esmp);
+}
+
+static sfc_efx_mcdi_dma_free_cb sfc_vdpa_mcdi_dma_free;
+static void
+sfc_vdpa_mcdi_dma_free(void *cookie, efsys_mem_t *esmp)
+{
+	struct sfc_vdpa_adapter *sva = cookie;
+
+	sfc_vdpa_dma_free(sva, esmp);
+}
+
+static sfc_efx_mcdi_sched_restart_cb sfc_vdpa_mcdi_sched_restart;
+static void
+sfc_vdpa_mcdi_sched_restart(void *cookie)
+{
+	RTE_SET_USED(cookie);
+}
+
+static sfc_efx_mcdi_mgmt_evq_poll_cb sfc_vdpa_mcdi_mgmt_evq_poll;
+static void
+sfc_vdpa_mcdi_mgmt_evq_poll(void *cookie)
+{
+	RTE_SET_USED(cookie);
+}
+
+static const struct sfc_efx_mcdi_ops sfc_vdpa_mcdi_ops = {
+	.dma_alloc	= sfc_vdpa_mcdi_dma_alloc,
+	.dma_free	= sfc_vdpa_mcdi_dma_free,
+	.sched_restart  = sfc_vdpa_mcdi_sched_restart,
+	.mgmt_evq_poll  = sfc_vdpa_mcdi_mgmt_evq_poll,
+
+};
+
+int
+sfc_vdpa_mcdi_init(struct sfc_vdpa_adapter *sva)
+{
+	uint32_t logtype;
+
+	sfc_vdpa_log_init(sva, "entry");
+
+	logtype = sfc_vdpa_register_logtype(&(sva->pdev->addr),
+					    SFC_VDPA_LOGTYPE_MCDI_STR,
+					    RTE_LOG_NOTICE);
+
+	return sfc_efx_mcdi_init(&sva->mcdi, logtype,
+				 sva->log_prefix, sva->nic,
+				 &sfc_vdpa_mcdi_ops, sva);
+}
+
+void
+sfc_vdpa_mcdi_fini(struct sfc_vdpa_adapter *sva)
+{
+	sfc_vdpa_log_init(sva, "entry");
+	sfc_efx_mcdi_fini(&sva->mcdi);
+}

drivers/vdpa/sfc/sfc_vdpa_ops.c new file +129 -0

new file mode 100644
index 0000000..71696be
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <rte_malloc.h>
+#include <rte_vdpa.h>
+#include <rte_vdpa_dev.h>
+#include <rte_vhost.h>
+
+#include "sfc_vdpa_ops.h"
+#include "sfc_vdpa.h"
+
+/* Dummy functions for mandatory vDPA ops to pass vDPA device registration.
+ * In subsequent patches these ops would be implemented.
+ */
+static int
+sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
+{
+	RTE_SET_USED(vdpa_dev);
+	RTE_SET_USED(queue_num);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t *features)
+{
+	RTE_SET_USED(vdpa_dev);
+	RTE_SET_USED(features);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_get_protocol_features(struct rte_vdpa_device *vdpa_dev,
+			       uint64_t *features)
+{
+	RTE_SET_USED(vdpa_dev);
+	RTE_SET_USED(features);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_dev_config(int vid)
+{
+	RTE_SET_USED(vid);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_dev_close(int vid)
+{
+	RTE_SET_USED(vid);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_set_vring_state(int vid, int vring, int state)
+{
+	RTE_SET_USED(vid);
+	RTE_SET_USED(vring);
+	RTE_SET_USED(state);
+
+	return -1;
+}
+
+static int
+sfc_vdpa_set_features(int vid)
+{
+	RTE_SET_USED(vid);
+
+	return -1;
+}
+
+static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
+	.get_queue_num = sfc_vdpa_get_queue_num,
+	.get_features = sfc_vdpa_get_features,
+	.get_protocol_features = sfc_vdpa_get_protocol_features,
+	.dev_conf = sfc_vdpa_dev_config,
+	.dev_close = sfc_vdpa_dev_close,
+	.set_vring_state = sfc_vdpa_set_vring_state,
+	.set_features = sfc_vdpa_set_features,
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *dev_handle, enum sfc_vdpa_context context)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	struct rte_pci_device *pci_dev;
+
+	/* Create vDPA ops context */
+	ops_data = rte_zmalloc("vdpa", sizeof(struct sfc_vdpa_ops_data), 0);
+	if (ops_data == NULL)
+		return NULL;
+
+	ops_data->vdpa_context = context;
+	ops_data->dev_handle = dev_handle;
+
+	pci_dev = sfc_vdpa_adapter_by_dev_handle(dev_handle)->pdev;
+
+	/* Register vDPA Device */
+	sfc_vdpa_log_init(dev_handle, "register vDPA device");
+	ops_data->vdpa_dev =
+		rte_vdpa_register_device(&pci_dev->device, &sfc_vdpa_ops);
+	if (ops_data->vdpa_dev == NULL) {
+		sfc_vdpa_err(dev_handle, "vDPA device registration failed");
+		goto fail_register_device;
+	}
+
+	ops_data->state = SFC_VDPA_STATE_INITIALIZED;
+
+	return ops_data;
+
+fail_register_device:
+	rte_free(ops_data);
+	return NULL;
+}
+
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data)
+{
+	rte_vdpa_unregister_device(ops_data->vdpa_dev);
+
+	rte_free(ops_data);
+}

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +4 -0

new file mode 100644
index 0000000..817b302
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */

drivers/vdpa/sfc/sfc_vdpa_ops.h line 14

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +10 -0

+
+#ifndef _SFC_VDPA_OPS_H
+#define _SFC_VDPA_OPS_H
+
+#include <rte_vdpa.h>
+
+#define SFC_VDPA_MAX_QUEUE_PAIRS		1
+
+enum sfc_vdpa_context {
+	SFC_VDPA_AS_PF = 0,

If you don't use SFC_VDPA_AS_PF for this version of driver, why not introduce it when you need it?

drivers/vdpa/sfc/sfc_vdpa_ops.h line 21

Quoted 7 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +7 -0

+	SFC_VDPA_AS_VF
+};
+
+enum sfc_vdpa_state {
+	SFC_VDPA_STATE_UNINITIALIZED = 0,
+	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_NSTATES

Similar comment: if you don't need info like this 'number of states', you can just remove it.

Thanks,
Chenbo

Quoted 17 lines

drivers/vdpa/sfc/sfc_vdpa_ops.h new file +15 -1

+};
+
+struct sfc_vdpa_ops_data {
+	void				*dev_handle;
+	struct rte_vdpa_device		*vdpa_dev;
+	enum sfc_vdpa_context		vdpa_context;
+	enum sfc_vdpa_state		state;
+};
+
+struct sfc_vdpa_ops_data *
+sfc_vdpa_device_init(void *adapter, enum sfc_vdpa_context context);
+void
+sfc_vdpa_device_fini(struct sfc_vdpa_ops_data *ops_data);
+
+#endif /* _SFC_VDPA_OPS_H */
--

1.8.3.1

Vijay Kumar Srivastava

2021-11-02 04:38 UTC #73 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Hi Chenbo,

Quoted 27 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, November 1, 2021 5:19 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 17 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---

[SNIP]

Quoted 23 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c
Quoted 4 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +4 -0
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mz->addr,
mcdi_iova,
Quoted 4 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +4 -0
+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+
drivers/vdpa/sfc/sfc_vdpa_hw.c line 75
Quoted 10 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			rte_memzone_free(mz);
+			return ret;
+		}
+
+	} while (ret < 0);
So when QEMU iova and mcdi_iova conflicts, you just let vdpa dev failed to configure, right?

Quoted 2 lines

Why not use re-mapping mcdi dma region as the solution? Any side-effect? Or you just assume conflict can hardly happen?

MCDI configuration is being done at the very early point of initialization.
Conflict would be detected later when rte_vhost_get_mem_table() would be invoked in .dev_conf callback and then MCDI re-mapping can be done in case of conflict, for this a patch is in progress which would be submitted separately.

Regards,
Vijay

[SNIP]

Xia, Chenbo

2021-11-02 05:16 UTC #74 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Quoted 61 lines

-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Tuesday, November 2, 2021 12:38 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 27 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, November 1, 2021 5:19 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 17 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---
[SNIP]
Quoted 23 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c
Quoted 4 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +4 -0
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mz->addr,
mcdi_iova,
Quoted 4 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +4 -0
+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+
drivers/vdpa/sfc/sfc_vdpa_hw.c line 75
Quoted 10 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			rte_memzone_free(mz);
+			return ret;
+		}
+
+	} while (ret < 0);
So when QEMU iova and mcdi_iova conflicts, you just let vdpa dev failed to configure, right?

Quoted 7 lines

Quoted 2 lines
Why not use re-mapping mcdi dma region as the solution? Any side-effect? Or you just assume conflict can hardly happen?
MCDI configuration is being done at the very early point of initialization. Conflict would be detected later when rte_vhost_get_mem_table() would be invoked in .dev_conf callback and
then MCDI re-mapping can be done in case of conflict,

Agree. It should be done in dev_conf callback.

Quoted 2 lines

for this a patch is in
progress which would be submitted separately.

OK for me, as the initial version, you can just let dev_conf fail if conflict happens.

/Chenbo

Quoted 5 lines

Regards,
Vijay
[SNIP]

Vijay Kumar Srivastava

2021-11-02 09:50 UTC #75 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Hi Chenbo,

Quoted 79 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Tuesday, November 2, 2021 10:47 AM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Quoted 63 lines
-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Tuesday, November 2, 2021 12:38 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Praveen Kumar Jain <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 28 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, November 1, 2021 5:19 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 17 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---
[SNIP]
Quoted 23 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c
Quoted 4 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +4 -0
+
+	do {
+		ret = rte_vfio_container_dma_map(sva->vfio_container_fd,
+						 (uint64_t)mz->addr,
mcdi_iova,
Quoted 4 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +4 -0
+						 mcdi_buff_size);
+		if (ret == 0)
+			break;
+
drivers/vdpa/sfc/sfc_vdpa_hw.c line 75
Quoted 10 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0
+		mcdi_iova = mcdi_iova >> 1;
+		if (mcdi_iova < mcdi_buff_size)	{
+			sfc_vdpa_err(sva,
+				     "DMA mapping failed for MCDI : %s",
+				     rte_strerror(rte_errno));
+			rte_memzone_free(mz);
+			return ret;
+		}
+
+	} while (ret < 0);
So when QEMU iova and mcdi_iova conflicts, you just let vdpa dev failed to configure, right?
Quoted 7 lines
Quoted 2 lines
Why not use re-mapping mcdi dma region as the solution? Any side-effect? Or you just assume conflict can hardly happen?
MCDI configuration is being done at the very early point of initialization. Conflict would be detected later when rte_vhost_get_mem_table() would be invoked in .dev_conf callback and then MCDI re-mapping can be done in case of conflict,

Quoted 7 lines

Agree. It should be done in dev_conf callback.
Quoted 2 lines
for this a patch is in
progress which would be submitted separately.
OK for me, as the initial version, you can just let dev_conf fail if conflict happens.

Yes. In case of conflict dev_conf would fail.

Regards,
Vijay

Quoted 1 line

Quoted 1 line
[SNIP]

Vijay Kumar Srivastava

2021-11-02 07:42 UTC #76 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

In reply to Xia, Chenbo

Hi Chenbo,

Quoted 27 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, November 1, 2021 5:19 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 17 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---

[SNIP]

Quoted 17 lines

Quoted 5 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0

+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva) {
+	efx_bar_region_t mem_ebr;
+	efx_nic_t *enp;
+	int rc;
+

drivers/vdpa/sfc/sfc_vdpa_hw.c

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0

+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "get family");
+	rc = sfc_efx_family(sva->pdev, &mem_ebr, &sva->family);
+	if (rc != 0)
+		goto fail_family;
+	sfc_vdpa_log_init(sva,
+			  "family is %u, membar is %u,"
+			  "function control window offset is %#" PRIx64,
+			  sva->family, mem_ebr.ebr_index,

mem_ebr.ebr_offset);

Quoted 2 lines

If ebr_idx is int, then %u -> %d. But if it's a bar index, its type should be unsigned int and you should change the definition in sfc common code.

Yes. It’s BAR index.
Thanks for the catch. I agree that usage of 'unsigned int' looks better for BAR index, however it should result in more changes in similar places. Is it OK if we use %d here right now to be consistent with current type and address the review note in a follow up patches for common/sfc_efx, net/sfc and vdpa/sfc?

Xia, Chenbo

2021-11-02 07:50 UTC #77 lore raw

Re: [dpdk-dev] [PATCH v3 02/10] vdpa/sfc: add support for device initialization

Quoted 55 lines

-----Original Message-----
From: Vijay Kumar Srivastava <[email protected]>
Sent: Tuesday, November 2, 2021 3:43 PM
To: Xia, Chenbo <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Praveen Kumar Jain <[email protected]>; Harpreet Singh Anand <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Chenbo,
Quoted 27 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Monday, November 1, 2021 5:19 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
Hi Vijay,
Quoted 17 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 02/10] vdpa/sfc: add support for device initialization
From: Vijay Kumar Srivastava <[email protected]>
Add HW initialization and vDPA device registration support.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---
[SNIP]
Quoted 17 lines
Quoted 5 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +5 -0
+sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva) {
+	efx_bar_region_t mem_ebr;
+	efx_nic_t *enp;
+	int rc;
+
drivers/vdpa/sfc/sfc_vdpa_hw.c
Quoted 10 lines
drivers/vdpa/sfc/sfc_vdpa_hw.c new file +10 -0
+	sfc_vdpa_log_init(sva, "entry");
+
+	sfc_vdpa_log_init(sva, "get family");
+	rc = sfc_efx_family(sva->pdev, &mem_ebr, &sva->family);
+	if (rc != 0)
+		goto fail_family;
+	sfc_vdpa_log_init(sva,
+			  "family is %u, membar is %u,"
+			  "function control window offset is %#" PRIx64,
+			  sva->family, mem_ebr.ebr_index,
mem_ebr.ebr_offset);

Quoted 8 lines

Quoted 2 lines
If ebr_idx is int, then %u -> %d. But if it's a bar index, its type should be unsigned int and you should change the definition in sfc common code.
Yes. It’s BAR index.
Thanks for the catch. I agree that usage of 'unsigned int' looks better for BAR index, however it should result in more changes in similar places. Is it OK if we use %d here right now to be consistent with current type and address the review note in a follow up patches for common/sfc_efx, net/sfc and vdpa/sfc?

Sure. Ok for me.

/Chenbo

Vijay Srivastava

author

2021-10-29 14:48 UTC #78 lore raw

PATCH v3 3/10 vdpa/sfc: add support to get device and protocol features

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_feature and get_protocol_features. This patch retrieves device supported features and enables protocol features.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/features/sfc.ini | 10 ++++
 drivers/common/sfc_efx/efsys.h       |  2 +-
 drivers/common/sfc_efx/version.map   | 10 ++++
 drivers/vdpa/sfc/sfc_vdpa.c          | 20 ++++++++
 drivers/vdpa/sfc/sfc_vdpa.h          |  2 +
 drivers/vdpa/sfc/sfc_vdpa_hw.c       | 13 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c      | 91 ++++++++++++++++++++++++++++++++----
 drivers/vdpa/sfc/sfc_vdpa_ops.h      |  3 ++
 8 files changed, 142 insertions(+), 9 deletions(-)

doc/guides/vdpadevs/features/sfc.ini +10 -0

index 71b6158..700d061 100644
@@ -4,6 +4,16 @@
 ; Refer to default.ini for the full list of available driver features.
 ;
 [Features]
+csum			= Y
+guest csum		= Y
+host tso4		= Y
+host tso6		= Y
+version 1		= Y
+mrg rxbuf		= Y
+any layout		= Y
+in_order		= Y
+proto host notifier	= Y
+IOMMU platform		= Y
 Linux			= Y
 x86-64			= Y
 Usage doc		= Y

drivers/common/sfc_efx/efsys.h +1 -1

index d133d61..37ec6b9 100644
@@ -187,7 +187,7 @@
 
 #define EFSYS_OPT_MAE 1
 
-#define EFSYS_OPT_VIRTIO 0
+#define EFSYS_OPT_VIRTIO 1
 
 /* ID */

drivers/common/sfc_efx/version.map +10 -0

index 642a62e..ec86220 100644
@@ -247,6 +247,16 @@ INTERNAL {
 	efx_txq_nbufs;
 	efx_txq_size;
 
+	efx_virtio_fini;
+	efx_virtio_get_doorbell_offset;
+	efx_virtio_get_features;
+	efx_virtio_init;
+	efx_virtio_qcreate;
+	efx_virtio_qdestroy;
+	efx_virtio_qstart;
+	efx_virtio_qstop;
+	efx_virtio_verify_features;
+
 	sfc_efx_dev_class_get;
 	sfc_efx_family;

drivers/vdpa/sfc/sfc_vdpa.c +20 -0

index b7eca56..ccbd243 100644
@@ -43,6 +43,26 @@ struct sfc_vdpa_adapter *
 	return found ? sva : NULL;
 }
 
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev)
+{
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (vdpa_dev == sva->ops_data->vdpa_dev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva->ops_data : NULL;
+}
+
 static int
 sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva)
 {

drivers/vdpa/sfc/sfc_vdpa.h +2 -0

index 046f25d..c10c3d3 100644
@@ -60,6 +60,8 @@ struct sfc_vdpa_adapter {
 
 struct sfc_vdpa_adapter *
 sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev);
 
 int
 sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);

drivers/vdpa/sfc/sfc_vdpa_hw.c +13 -0

index 7c256ff..7a67bd8 100644
@@ -278,10 +278,20 @@
 	if (rc != 0)
 		goto fail_estimate_rsrc_limits;
 
+	sfc_vdpa_log_init(sva, "init virtio");
+	rc = efx_virtio_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "virtio init failed: %s", rte_strerror(rc));
+		goto fail_virtio_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_virtio_init:
+	efx_nic_fini(enp);
+
 fail_estimate_rsrc_limits:
 fail_nic_reset:
 	efx_nic_unprobe(enp);

@@ -310,6 +320,9 @@
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_log_init(sva, "virtio fini");
+	efx_virtio_fini(enp);
+
 	sfc_vdpa_log_init(sva, "unprobe nic");
 	efx_nic_unprobe(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +83 -8

138 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h +3 -0

index 817b302..21cbb73 100644
@@ -26,6 +26,9 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+
+	uint64_t			dev_features;
+	uint64_t			drv_features;
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Xia, Chenbo

Reviewed-by on v3 3/10

2021-11-02 07:09 UTC #79 lore raw

Re: [dpdk-dev] [PATCH v3 03/10] vdpa/sfc: add support to get device and protocol features

Quoted 327 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 03/10] vdpa/sfc: add support to get device and protocol features

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_feature and get_protocol_features. This patch retrieves device supported features and enables protocol features.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/features/sfc.ini | 10 ++++
 drivers/common/sfc_efx/efsys.h       |  2 +-
 drivers/common/sfc_efx/version.map   | 10 ++++
 drivers/vdpa/sfc/sfc_vdpa.c          | 20 ++++++++
 drivers/vdpa/sfc/sfc_vdpa.h          |  2 +
 drivers/vdpa/sfc/sfc_vdpa_hw.c       | 13 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c      | 91 ++++++++++++++++++++++++++++++++---

(quoted diff) +0 -1

-
 drivers/vdpa/sfc/sfc_vdpa_ops.h      |  3 ++
 8 files changed, 142 insertions(+), 9 deletions(-)

doc/guides/vdpadevs/features/sfc.ini +10 -0

index 71b6158..700d061 100644
@@ -4,6 +4,16 @@
 ; Refer to default.ini for the full list of available driver features.
 ;
 [Features]
+csum			= Y
+guest csum		= Y
+host tso4		= Y
+host tso6		= Y
+version 1		= Y
+mrg rxbuf		= Y
+any layout		= Y
+in_order		= Y
+proto host notifier	= Y
+IOMMU platform		= Y
 Linux			= Y
 x86-64			= Y
 Usage doc		= Y

drivers/common/sfc_efx/efsys.h +1 -1

index d133d61..37ec6b9 100644
@@ -187,7 +187,7 @@
 
 #define EFSYS_OPT_MAE 1
 
-#define EFSYS_OPT_VIRTIO 0
+#define EFSYS_OPT_VIRTIO 1
 
 /* ID */

drivers/common/sfc_efx/version.map +10 -0

index 642a62e..ec86220 100644
@@ -247,6 +247,16 @@ INTERNAL {
 	efx_txq_nbufs;
 	efx_txq_size;
 
+	efx_virtio_fini;
+	efx_virtio_get_doorbell_offset;
+	efx_virtio_get_features;
+	efx_virtio_init;
+	efx_virtio_qcreate;
+	efx_virtio_qdestroy;
+	efx_virtio_qstart;
+	efx_virtio_qstop;
+	efx_virtio_verify_features;
+
 	sfc_efx_dev_class_get;
 	sfc_efx_family;

drivers/vdpa/sfc/sfc_vdpa.c +20 -0

index b7eca56..ccbd243 100644
@@ -43,6 +43,26 @@ struct sfc_vdpa_adapter *
 	return found ? sva : NULL;
 }
 
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev)
+{
+	bool found = false;
+	struct sfc_vdpa_adapter *sva;
+
+	pthread_mutex_lock(&sfc_vdpa_adapter_list_lock);
+
+	TAILQ_FOREACH(sva, &sfc_vdpa_adapter_list, next) {
+		if (vdpa_dev == sva->ops_data->vdpa_dev) {
+			found = true;
+			break;
+		}
+	}
+
+	pthread_mutex_unlock(&sfc_vdpa_adapter_list_lock);
+
+	return found ? sva->ops_data : NULL;
+}
+
 static int
 sfc_vdpa_vfio_setup(struct sfc_vdpa_adapter *sva)
 {

drivers/vdpa/sfc/sfc_vdpa.h +2 -0

index 046f25d..c10c3d3 100644
@@ -60,6 +60,8 @@ struct sfc_vdpa_adapter {
 
 struct sfc_vdpa_adapter *
 sfc_vdpa_get_adapter_by_dev(struct rte_pci_device *pdev);
+struct sfc_vdpa_ops_data *
+sfc_vdpa_get_data_by_dev(struct rte_vdpa_device *vdpa_dev);
 
 int
 sfc_vdpa_hw_init(struct sfc_vdpa_adapter *sva);

drivers/vdpa/sfc/sfc_vdpa_hw.c +13 -0

index 7c256ff..7a67bd8 100644
@@ -278,10 +278,20 @@
 	if (rc != 0)
 		goto fail_estimate_rsrc_limits;
 
+	sfc_vdpa_log_init(sva, "init virtio");
+	rc = efx_virtio_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "virtio init failed: %s", rte_strerror(rc));
+		goto fail_virtio_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_virtio_init:
+	efx_nic_fini(enp);
+
 fail_estimate_rsrc_limits:
 fail_nic_reset:
 	efx_nic_unprobe(enp);

@@ -310,6 +320,9 @@
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_log_init(sva, "virtio fini");
+	efx_virtio_fini(enp);
+
 	sfc_vdpa_log_init(sva, "unprobe nic");
 	efx_nic_unprobe(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +83 -8

index 71696be..5750944 100644
@@ -3,17 +3,31 @@
  * Copyright(c) 2020-2021 Xilinx, Inc.
  */
 
+#include <rte_errno.h>
 #include <rte_malloc.h>
 #include <rte_vdpa.h>
 #include <rte_vdpa_dev.h>
 #include <rte_vhost.h>
 
+#include "efx.h"
 #include "sfc_vdpa_ops.h"
 #include "sfc_vdpa.h"
 
-/* Dummy functions for mandatory vDPA ops to pass vDPA device registration.
- * In subsequent patches these ops would be implemented.
+/* These protocol features are needed to enable notifier ctrl */
+#define SFC_VDPA_PROTOCOL_FEATURES \
+		((1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK) | \
+		 (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ) | \
+		 (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) | \
+		 (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) | \
+		 (1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD))
+
+/*
+ * Set of features which are enabled by default.
+ * Protocol feature bit is needed to enable notification notifier ctrl.
  */
+#define SFC_VDPA_DEFAULT_FEATURES \
+		(1ULL << VHOST_USER_F_PROTOCOL_FEATURES)
+
 static int
 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
 {

@@ -24,22 +38,67 @@
 }
 
 static int
+sfc_vdpa_get_device_features(struct sfc_vdpa_ops_data *ops_data)
+{
+	int rc;
+	uint64_t dev_features;
+	efx_nic_t *nic;
+
+	nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
+
+	rc = efx_virtio_get_features(nic, EFX_VIRTIO_DEVICE_TYPE_NET,
+				     &dev_features);
+	if (rc != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "could not read device feature: %s",
+			     rte_strerror(rc));
+		return rc;
+	}
+
+	ops_data->dev_features = dev_features;
+
+	sfc_vdpa_info(ops_data->dev_handle,
+		      "device supported virtio features : 0x%" PRIx64,
+		      ops_data->dev_features);
+
+	return 0;
+}
+
+static int
 sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t *features)
 {
-	RTE_SET_USED(vdpa_dev);
-	RTE_SET_USED(features);
+	struct sfc_vdpa_ops_data *ops_data;
 
-	return -1;
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	*features = ops_data->drv_features;
+
+	sfc_vdpa_info(ops_data->dev_handle,
+		      "vDPA ops get_feature :: features : 0x%" PRIx64,
+		      *features);
+
+	return 0;
 }
 
 static int
 sfc_vdpa_get_protocol_features(struct rte_vdpa_device *vdpa_dev,
 			       uint64_t *features)
 {
-	RTE_SET_USED(vdpa_dev);
-	RTE_SET_USED(features);
+	struct sfc_vdpa_ops_data *ops_data;
 
-	return -1;
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	*features = SFC_VDPA_PROTOCOL_FEATURES;
+
+	sfc_vdpa_info(ops_data->dev_handle,
+		      "vDPA ops get_protocol_feature :: features : 0x%" PRIx64,
+		      *features);
+
+	return 0;
 }
 
 static int

@@ -91,6 +150,7 @@ struct sfc_vdpa_ops_data *
 {
 	struct sfc_vdpa_ops_data *ops_data;
 	struct rte_pci_device *pci_dev;
+	int rc;
 
 	/* Create vDPA ops context */
 	ops_data = rte_zmalloc("vdpa", sizeof(struct sfc_vdpa_ops_data), 0);

@@ -111,10 +171,25 @@ struct sfc_vdpa_ops_data *
 		goto fail_register_device;
 	}
 
+	/* Read supported device features */
+	sfc_vdpa_log_init(dev_handle, "get device feature");
+	rc = sfc_vdpa_get_device_features(ops_data);
+	if (rc != 0)
+		goto fail_get_dev_feature;
+
+	/* Driver features are superset of device supported feature
+	 * and any additional features supported by the driver.
+	 */
+	ops_data->drv_features =
+		ops_data->dev_features | SFC_VDPA_DEFAULT_FEATURES;
+
 	ops_data->state = SFC_VDPA_STATE_INITIALIZED;
 
 	return ops_data;
 
+fail_get_dev_feature:
+	rte_vdpa_unregister_device(ops_data->vdpa_dev);
+
 fail_register_device:
 	rte_free(ops_data);
 	return NULL;

drivers/vdpa/sfc/sfc_vdpa_ops.h +3 -1

index 817b302..21cbb73 100644
@@ -26,6 +26,9 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+
+	uint64_t			dev_features;
+	uint64_t			drv_features;
 };
 
 struct sfc_vdpa_ops_data *
--

Quoted 1 line

1.8.3.1

Reviewed-by Chenbo Xia <[email protected]>

Vijay Srivastava

author

2021-10-29 14:49 UTC #80 lore raw

PATCH v3 4/10 vdpa/sfc: get device supported max queue count

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_queue_num to get the maximum number of queues supported by the device.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +13 -3

index 5750944..6c702e1 100644
@@ -31,10 +31,20 @@
 static int
 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
 {
-	RTE_SET_USED(vdpa_dev);
-	RTE_SET_USED(queue_num);
+	struct sfc_vdpa_ops_data *ops_data;
+	void *dev;
 
-	return -1;
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	*queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
+
+	sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %d",
+		      *queue_num);
+
+	return 0;
 }
 
 static int

Signature · 1 line

1.8.3.1

Xia, Chenbo

Reviewed-by on v3 4/10

2021-11-02 07:19 UTC #81 lore raw

Re: [dpdk-dev] [PATCH v3 04/10] vdpa/sfc: get device supported max queue count

Quoted 33 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 04/10] vdpa/sfc: get device supported max queue count
From: Vijay Kumar Srivastava <[email protected]>
Implement vDPA ops get_queue_num to get the maximum number of queues supported by the device.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
Reviewed-by Maxime Coquelin <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
drivers/vdpa/sfc/sfc_vdpa_ops.c +2 -2
index 5750944..6c702e1 100644
@@ -31,10 +31,20 @@
 static int
 sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
 {
-	RTE_SET_USED(vdpa_dev);
-	RTE_SET_USED(queue_num);
+	struct sfc_vdpa_ops_data *ops_data;
+	void *dev;
 

drivers/vdpa/sfc/sfc_vdpa_ops.c line 45

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +9 -1

-	return -1;
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	*queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
+
+	sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %d",
+		      *queue_num);

%d -> %u

With this fixed:

Reviewed-by Chenbo Xia <[email protected]>

Quoted 7 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +2 -1
+
+	return 0;
 }
 
 static int
--
1.8.3.1

Vijay Srivastava

author

2021-10-29 14:48 UTC #82 lore raw

PATCH v3 5/10 vdpa/sfc: add support to get VFIO device fd

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops get_vfio_device_fd to get the VFIO device fd.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

drivers/vdpa/sfc/sfc_vdpa_ops.c +24 -0

index 6c702e1..5253adb 100644
@@ -145,6 +145,29 @@
 	return -1;
 }
 
+static int
+sfc_vdpa_get_vfio_device_fd(int vid)
+{
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	int vfio_dev_fd;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+
+	sfc_vdpa_info(dev, "vDPA ops get_vfio_device_fd :: vfio fd : %d",
+		      vfio_dev_fd);
+
+	return vfio_dev_fd;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -153,6 +176,7 @@
 	.dev_close = sfc_vdpa_dev_close,
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
+	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Xia, Chenbo

Reviewed-by on v3 5/10

2021-11-02 07:10 UTC #83 lore raw

Re: [dpdk-dev] [PATCH v3 05/10] vdpa/sfc: add support to get VFIO device fd

Quoted 62 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 05/10] vdpa/sfc: add support to get VFIO device fd
From: Vijay Kumar Srivastava <[email protected]>
Implement vDPA ops get_vfio_device_fd to get the VFIO device fd.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
Reviewed-by Maxime Coquelin <[email protected]>
---
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
drivers/vdpa/sfc/sfc_vdpa_ops.c +24 -1
index 6c702e1..5253adb 100644
@@ -145,6 +145,29 @@
 	return -1;
 }
 
+static int
+sfc_vdpa_get_vfio_device_fd(int vid)
+{
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	int vfio_dev_fd;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+
+	sfc_vdpa_info(dev, "vDPA ops get_vfio_device_fd :: vfio fd : %d",
+		      vfio_dev_fd);
+
+	return vfio_dev_fd;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,
@@ -153,6 +176,7 @@
 	.dev_close = sfc_vdpa_dev_close,
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
+	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
 };
 
 struct sfc_vdpa_ops_data *
--

Quoted 1 line

1.8.3.1

Reviewed-by Chenbo Xia <[email protected]>

Vijay Srivastava

author

2021-10-29 14:49 UTC #84 lore raw

PATCH v3 6/10 vdpa/sfc: add support for dev conf and dev close ops

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops dev_conf and dev_close for DMA mapping, interrupt and virtqueue configurations.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Removed redundant null check while calling free().
* Added error handling for rte_vhost_get_vhost_vring().

 drivers/vdpa/sfc/sfc_vdpa.c     |   6 +
 drivers/vdpa/sfc/sfc_vdpa.h     |  43 ++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c  |  69 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 530 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |  28 +++
 5 files changed, 656 insertions(+), 20 deletions(-)

drivers/vdpa/sfc/sfc_vdpa.c +6 -0

index ccbd243..b3c82e5 100644
@@ -246,6 +246,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_adapter_lock_init(sva);
+
 	sfc_vdpa_log_init(sva, "vfio init");
 	if (sfc_vdpa_vfio_setup(sva) < 0) {
 		sfc_vdpa_err(sva, "failed to setup device %s", pci_dev->name);

@@ -280,6 +282,8 @@ struct sfc_vdpa_ops_data *
 	sfc_vdpa_vfio_teardown(sva);
 
 fail_vfio_setup:
+	sfc_vdpa_adapter_lock_fini(sva);
+
 fail_set_log_prefix:
 	rte_free(sva);

@@ -311,6 +315,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_vfio_teardown(sva);
 
+	sfc_vdpa_adapter_lock_fini(sva);
+
 	rte_free(sva);
 
 	return 0;

drivers/vdpa/sfc/sfc_vdpa.h +43 -0

53 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_hw.c +69 -0

81 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.c +510 ~~-20~~

566 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_ops.h +28 -0

index 21cbb73..8d553c5 100644
@@ -18,17 +18,45 @@ enum sfc_vdpa_context {
 enum sfc_vdpa_state {
 	SFC_VDPA_STATE_UNINITIALIZED = 0,
 	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_CONFIGURING,
+	SFC_VDPA_STATE_CONFIGURED,
+	SFC_VDPA_STATE_CLOSING,
+	SFC_VDPA_STATE_CLOSED,
+	SFC_VDPA_STATE_STARTING,
+	SFC_VDPA_STATE_STARTED,
+	SFC_VDPA_STATE_STOPPING,
 	SFC_VDPA_STATE_NSTATES
 };
 
+struct sfc_vdpa_vring_info {
+	uint64_t	desc;
+	uint64_t	avail;
+	uint64_t	used;
+	uint64_t	size;
+	uint16_t	last_avail_idx;
+	uint16_t	last_used_idx;
+};
+
+typedef struct sfc_vdpa_vq_context_s {
+	uint8_t				enable;
+	uint32_t			pidx;
+	uint32_t			cidx;
+	efx_virtio_vq_t			*vq;
+} sfc_vdpa_vq_context_t;
+
 struct sfc_vdpa_ops_data {
 	void				*dev_handle;
+	int				vid;
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;
+	uint64_t			req_features;
+
+	uint16_t			vq_count;
+	struct sfc_vdpa_vq_context_s	vq_cxt[SFC_VDPA_MAX_QUEUE_PAIRS * 2];
 };
 
 struct sfc_vdpa_ops_data *

Signature · 1 line

1.8.3.1

Xia, Chenbo

2021-11-02 07:14 UTC #85 lore raw

Re: [dpdk-dev] [PATCH v3 06/10] vdpa/sfc: add support for dev conf and dev close ops

Hi Vijay,

Quoted 183 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 06/10] vdpa/sfc: add support for dev conf and dev close ops

From: Vijay Kumar Srivastava <[email protected]>

Implement vDPA ops dev_conf and dev_close for DMA mapping, interrupt and virtqueue configurations.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Removed redundant null check while calling free().
* Added error handling for rte_vhost_get_vhost_vring().

 drivers/vdpa/sfc/sfc_vdpa.c     |   6 +
 drivers/vdpa/sfc/sfc_vdpa.h     |  43 ++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c  |  69 ++++++
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 530 ++++++++++++++++++++++++++++++++++++++-

(quoted diff) +0 -1

-
 drivers/vdpa/sfc/sfc_vdpa_ops.h |  28 +++
 5 files changed, 656 insertions(+), 20 deletions(-)

drivers/vdpa/sfc/sfc_vdpa.c +6 -0

index ccbd243..b3c82e5 100644
@@ -246,6 +246,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_log_init(sva, "entry");
 
+	sfc_vdpa_adapter_lock_init(sva);
+
 	sfc_vdpa_log_init(sva, "vfio init");
 	if (sfc_vdpa_vfio_setup(sva) < 0) {
 		sfc_vdpa_err(sva, "failed to setup device %s", pci_dev->name);

@@ -280,6 +282,8 @@ struct sfc_vdpa_ops_data *
 	sfc_vdpa_vfio_teardown(sva);
 
 fail_vfio_setup:
+	sfc_vdpa_adapter_lock_fini(sva);
+
 fail_set_log_prefix:
 	rte_free(sva);

@@ -311,6 +315,8 @@ struct sfc_vdpa_ops_data *
 
 	sfc_vdpa_vfio_teardown(sva);
 
+	sfc_vdpa_adapter_lock_fini(sva);
+
 	rte_free(sva);
 
 	return 0;

drivers/vdpa/sfc/sfc_vdpa.h +43 -0

index c10c3d3..1bf96e7 100644
@@ -80,10 +80,53 @@ struct sfc_vdpa_ops_data *
 void
 sfc_vdpa_dma_free(struct sfc_vdpa_adapter *sva, efsys_mem_t *esmp);
 
+int
+sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *vdpa_data, bool do_map);
+
 static inline struct sfc_vdpa_adapter *
 sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
 {
 	return (struct sfc_vdpa_adapter *)dev_handle;
 }
 
+/*
+ * Add wrapper functions to acquire/release lock to be able to remove or
+ * change the lock in one place.
+ */
+static inline void
+sfc_vdpa_adapter_lock_init(struct sfc_vdpa_adapter *sva)
+{
+	rte_spinlock_init(&sva->lock);
+}
+
+static inline int
+sfc_vdpa_adapter_is_locked(struct sfc_vdpa_adapter *sva)
+{
+	return rte_spinlock_is_locked(&sva->lock);
+}
+
+static inline void
+sfc_vdpa_adapter_lock(struct sfc_vdpa_adapter *sva)
+{
+	rte_spinlock_lock(&sva->lock);
+}
+
+static inline int
+sfc_vdpa_adapter_trylock(struct sfc_vdpa_adapter *sva)
+{
+	return rte_spinlock_trylock(&sva->lock);
+}
+
+static inline void
+sfc_vdpa_adapter_unlock(struct sfc_vdpa_adapter *sva)
+{
+	rte_spinlock_unlock(&sva->lock);
+}
+
+static inline void
+sfc_vdpa_adapter_lock_fini(__rte_unused struct sfc_vdpa_adapter *sva)
+{
+	/* Just for symmetry of the API */
+}
+
 #endif  /* _SFC_VDPA_H */

drivers/vdpa/sfc/sfc_vdpa_hw.c +51 -0

index 7a67bd8..b473708 100644
@@ -8,6 +8,7 @@
 #include <rte_common.h>
 #include <rte_errno.h>
 #include <rte_vfio.h>
+#include <rte_vhost.h>
 
 #include "efx.h"
 #include "sfc_vdpa.h"

@@ -109,6 +110,74 @@
 	memset(esmp, 0, sizeof(*esmp));
 }
 
+int
+sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *ops_data, bool do_map)
+{
+	uint32_t i, j;
+	int rc;
+	struct rte_vhost_memory *vhost_mem = NULL;
+	struct rte_vhost_mem_region *mem_reg = NULL;
+	int vfio_container_fd;
+	void *dev;
+
+	dev = ops_data->dev_handle;
+	vfio_container_fd =
+		sfc_vdpa_adapter_by_dev_handle(dev)->vfio_container_fd;
+
+	rc = rte_vhost_get_mem_table(ops_data->vid, &vhost_mem);
+	if (rc < 0) {
+		sfc_vdpa_err(dev,
+			     "failed to get VM memory layout");
+		goto error;
+	}
+
+	for (i = 0; i < vhost_mem->nregions; i++) {
+		mem_reg = &vhost_mem->regions[i];
+
+		if (do_map) {
+			rc = rte_vfio_container_dma_map(vfio_container_fd,
+						mem_reg->host_user_addr,
+						mem_reg->guest_phys_addr,
+						mem_reg->size);
+			if (rc < 0) {
+				sfc_vdpa_err(dev,
+					     "DMA map failed : %s",
+					     rte_strerror(rte_errno));
+				goto failed_vfio_dma_map;
+			}
+		} else {
+			rc = rte_vfio_container_dma_unmap(vfio_container_fd,
+						mem_reg->host_user_addr,
+						mem_reg->guest_phys_addr,
+						mem_reg->size);
+			if (rc < 0) {
+				sfc_vdpa_err(dev,
+					     "DMA unmap failed : %s",
+					     rte_strerror(rte_errno));
+				goto error;
+			}
+		}
+	}
+
+	free(vhost_mem);

drivers/vdpa/sfc/sfc_vdpa_hw.c line 172

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +10 -0

+
+	return 0;
+
+failed_vfio_dma_map:
+	for (j = 0; j < i; j++) {
+		mem_reg = &vhost_mem->regions[j];
+		rc = rte_vfio_container_dma_unmap(vfio_container_fd,
+						  mem_reg->host_user_addr,
+						  mem_reg->guest_phys_addr,
+						  mem_reg->size);

'rc =' not needed and will make the logic incorrect. Think of dma map failed but unmap here all succeeded. This function will return 0, and the function that calls it will think dma map succeed.

Quoted 146 lines

drivers/vdpa/sfc/sfc_vdpa_hw.c +8 -0

+	}
+
+error:
+	free(vhost_mem);
+
+	return rc;
+}
+
 static int
 sfc_vdpa_mem_bar_init(struct sfc_vdpa_adapter *sva,
 		      const efx_bar_region_t *mem_ebrp)

drivers/vdpa/sfc/sfc_vdpa_ops.c +91 ~~-17~~

index 5253adb..de1c81a 100644
@@ -3,10 +3,13 @@
  * Copyright(c) 2020-2021 Xilinx, Inc.
  */
 
+#include <sys/ioctl.h>
+
 #include <rte_errno.h>
 #include <rte_malloc.h>
 #include <rte_vdpa.h>
 #include <rte_vdpa_dev.h>
+#include <rte_vfio.h>
 #include <rte_vhost.h>
 
 #include "efx.h"

@@ -28,24 +31,12 @@
 #define SFC_VDPA_DEFAULT_FEATURES \
 		(1ULL << VHOST_USER_F_PROTOCOL_FEATURES)
 
-static int
-sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
-{
-	struct sfc_vdpa_ops_data *ops_data;
-	void *dev;
-
-	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
-	if (ops_data == NULL)
-		return -1;
-
-	dev = ops_data->dev_handle;
-	*queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
+#define SFC_VDPA_MSIX_IRQ_SET_BUF_LEN \
+		(sizeof(struct vfio_irq_set) + \
+		sizeof(int) * (SFC_VDPA_MAX_QUEUE_PAIRS * 2 + 1))
 
-	sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %d",
-		      *queue_num);
-
-	return 0;
-}
+/* It will be used for target VF when calling function is not PF */
+#define SFC_VDPA_VF_NULL		0xFFFF
 
 static int
 sfc_vdpa_get_device_features(struct sfc_vdpa_ops_data *ops_data)

@@ -74,6 +65,441 @@
 	return 0;
 }
 
+static uint64_t
+hva_to_gpa(int vid, uint64_t hva)
+{
+	struct rte_vhost_memory *vhost_mem = NULL;
+	struct rte_vhost_mem_region *mem_reg = NULL;
+	uint32_t i;
+	uint64_t gpa = 0;
+
+	if (rte_vhost_get_mem_table(vid, &vhost_mem) < 0)
+		goto error;
+
+	for (i = 0; i < vhost_mem->nregions; i++) {
+		mem_reg = &vhost_mem->regions[i];
+
+		if (hva >= mem_reg->host_user_addr &&
+				hva < mem_reg->host_user_addr + mem_reg->size) {
+			gpa = (hva - mem_reg->host_user_addr) +
+				mem_reg->guest_phys_addr;
+			break;
+		}
+	}
+
+error:
+	free(vhost_mem);
+	return gpa;
+}
+
+static int
+sfc_vdpa_enable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
+{
+	int rc;
+	int *irq_fd_ptr;
+	int vfio_dev_fd;
+	uint32_t i, num_vring;
+	struct rte_vhost_vring vring;
+	struct vfio_irq_set *irq_set;
+	struct rte_pci_device *pci_dev;
+	char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
+	void *dev;
+
+	num_vring = rte_vhost_get_vring_num(ops_data->vid);
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	pci_dev = sfc_vdpa_adapter_by_dev_handle(dev)->pdev;
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = sizeof(irq_set_buf);
+	irq_set->count = num_vring + 1;
+	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+			 VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = 0;
+	irq_fd_ptr = (int *)&irq_set->data;
+	irq_fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] =
+		rte_intr_fd_get(pci_dev->intr_handle);
+
+	for (i = 0; i < num_vring; i++) {
+		rc = rte_vhost_get_vhost_vring(ops_data->vid, i, &vring);
+		if (rc)
+			return -1;
+
+		irq_fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd;
+	}
+
+	rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (rc) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "error enabling MSI-X interrupts: %s",
+			     strerror(errno));
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+sfc_vdpa_disable_vfio_intr(struct sfc_vdpa_ops_data *ops_data)
+{
+	int rc;
+	int vfio_dev_fd;
+	struct vfio_irq_set *irq_set;
+	char irq_set_buf[SFC_VDPA_MSIX_IRQ_SET_BUF_LEN];
+	void *dev;

drivers/vdpa/sfc/sfc_vdpa_ops.c line 160

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0

+
+	dev = ops_data->dev_handle;
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+
+	irq_set = (struct vfio_irq_set *)irq_set_buf;
+	irq_set->argsz = sizeof(irq_set_buf);
+	irq_set->count = 0;
+	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
+	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+	irq_set->start = 0;

I think disabling only needs a struct vfio_irq_set instead of irq_set_buf, because you don't have data when disabling. and argsz should be sizeof(struct vfio_irq_set)

Quoted 52 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +52 -0

+
+	rc = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+	if (rc) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "error disabling MSI-X interrupts: %s",
+			     strerror(errno));
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+sfc_vdpa_get_vring_info(struct sfc_vdpa_ops_data *ops_data,
+			int vq_num, struct sfc_vdpa_vring_info *vring)
+{
+	int rc;
+	uint64_t gpa;
+	struct rte_vhost_vring vq;
+
+	rc = rte_vhost_get_vhost_vring(ops_data->vid, vq_num, &vq);
+	if (rc < 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "get vhost vring failed: %s", rte_strerror(rc));
+		return rc;
+	}
+
+	gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.desc);
+	if (gpa == 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "fail to get GPA for descriptor ring.");
+		goto fail_vring_map;
+	}
+	vring->desc = gpa;
+
+	gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.avail);
+	if (gpa == 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "fail to get GPA for available ring.");
+		goto fail_vring_map;
+	}
+	vring->avail = gpa;
+
+	gpa = hva_to_gpa(ops_data->vid, (uint64_t)(uintptr_t)vq.used);
+	if (gpa == 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "fail to get GPA for used ring.");
+		goto fail_vring_map;
+	}
+	vring->used = gpa;
+
+	vring->size = vq.size;

drivers/vdpa/sfc/sfc_vdpa_ops.c line 222

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0

+
+	rc = rte_vhost_get_vring_base(ops_data->vid, vq_num,
+				      &vring->last_avail_idx,
+				      &vring->last_used_idx);
+
+	return rc;
+
+fail_vring_map:
+	return -1;
+}

I don't think you need 'fail_vring_map'. Just use 'return -1' to replace goto fail_vring_map

Thanks,
Chenbo

Quoted 420 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +347 -3

+
+static int
+sfc_vdpa_virtq_start(struct sfc_vdpa_ops_data *ops_data, int vq_num)
+{
+	int rc;
+	efx_virtio_vq_t *vq;
+	struct sfc_vdpa_vring_info vring;
+	efx_virtio_vq_cfg_t vq_cfg;
+	efx_virtio_vq_dyncfg_t vq_dyncfg;
+
+	vq = ops_data->vq_cxt[vq_num].vq;
+	if (vq == NULL)
+		return -1;
+
+	rc = sfc_vdpa_get_vring_info(ops_data, vq_num, &vring);
+	if (rc < 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "get vring info failed: %s", rte_strerror(rc));
+		goto fail_vring_info;
+	}
+
+	vq_cfg.evvc_target_vf = SFC_VDPA_VF_NULL;
+
+	/* even virtqueue for RX and odd for TX */
+	if (vq_num % 2) {
+		vq_cfg.evvc_type = EFX_VIRTIO_VQ_TYPE_NET_TXQ;
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "configure virtqueue # %d (TXQ)", vq_num);
+	} else {
+		vq_cfg.evvc_type = EFX_VIRTIO_VQ_TYPE_NET_RXQ;
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "configure virtqueue # %d (RXQ)", vq_num);
+	}
+
+	vq_cfg.evvc_vq_num = vq_num;
+	vq_cfg.evvc_desc_tbl_addr   = vring.desc;
+	vq_cfg.evvc_avail_ring_addr = vring.avail;
+	vq_cfg.evvc_used_ring_addr  = vring.used;
+	vq_cfg.evvc_vq_size = vring.size;
+
+	vq_dyncfg.evvd_vq_pidx = vring.last_used_idx;
+	vq_dyncfg.evvd_vq_cidx = vring.last_avail_idx;
+
+	/* MSI-X vector is function-relative */
+	vq_cfg.evvc_msix_vector = RTE_INTR_VEC_RXTX_OFFSET + vq_num;
+	if (ops_data->vdpa_context == SFC_VDPA_AS_VF)
+		vq_cfg.evvc_pas_id = 0;
+	vq_cfg.evcc_features = ops_data->dev_features &
+			       ops_data->req_features;
+
+	/* Start virtqueue */
+	rc = efx_virtio_qstart(vq, &vq_cfg, &vq_dyncfg);
+	if (rc != 0) {
+		/* destroy virtqueue */
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "virtqueue start failed: %s",
+			     rte_strerror(rc));
+		efx_virtio_qdestroy(vq);
+		goto fail_virtio_qstart;
+	}
+
+	sfc_vdpa_info(ops_data->dev_handle,
+		      "virtqueue started successfully for vq_num %d", vq_num);
+
+	ops_data->vq_cxt[vq_num].enable = B_TRUE;
+
+	return rc;
+
+fail_virtio_qstart:
+fail_vring_info:
+	return rc;
+}
+
+static int
+sfc_vdpa_virtq_stop(struct sfc_vdpa_ops_data *ops_data, int vq_num)
+{
+	int rc;
+	efx_virtio_vq_dyncfg_t vq_idx;
+	efx_virtio_vq_t *vq;
+
+	if (ops_data->vq_cxt[vq_num].enable != B_TRUE)
+		return -1;
+
+	vq = ops_data->vq_cxt[vq_num].vq;
+	if (vq == NULL)
+		return -1;
+
+	/* stop the vq */
+	rc = efx_virtio_qstop(vq, &vq_idx);
+	if (rc == 0) {
+		ops_data->vq_cxt[vq_num].cidx = vq_idx.evvd_vq_cidx;
+		ops_data->vq_cxt[vq_num].pidx = vq_idx.evvd_vq_pidx;
+	}
+	ops_data->vq_cxt[vq_num].enable = B_FALSE;
+
+	return rc;
+}
+
+static int
+sfc_vdpa_configure(struct sfc_vdpa_ops_data *ops_data)
+{
+	int rc, i;
+	int nr_vring;
+	int max_vring_cnt;
+	efx_virtio_vq_t *vq;
+	efx_nic_t *nic;
+	void *dev;
+
+	dev = ops_data->dev_handle;
+	nic = sfc_vdpa_adapter_by_dev_handle(dev)->nic;
+
+	SFC_EFX_ASSERT(ops_data->state == SFC_VDPA_STATE_INITIALIZED);
+
+	ops_data->state = SFC_VDPA_STATE_CONFIGURING;
+
+	nr_vring = rte_vhost_get_vring_num(ops_data->vid);
+	max_vring_cnt =
+		(sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	/* number of vring should not be more than supported max vq count */
+	if (nr_vring > max_vring_cnt) {
+		sfc_vdpa_err(dev,
+			     "nr_vring (%d) is > max vring count (%d)",
+			     nr_vring, max_vring_cnt);
+		goto fail_vring_num;
+	}
+
+	rc = sfc_vdpa_dma_map(ops_data, true);
+	if (rc) {
+		sfc_vdpa_err(dev,
+			     "DMA map failed: %s", rte_strerror(rc));
+		goto fail_dma_map;
+	}
+
+	for (i = 0; i < nr_vring; i++) {
+		rc = efx_virtio_qcreate(nic, &vq);
+		if ((rc != 0) || (vq == NULL)) {
+			sfc_vdpa_err(dev,
+				     "virtqueue create failed: %s",
+				     rte_strerror(rc));
+			goto fail_vq_create;
+		}
+
+		/* store created virtqueue context */
+		ops_data->vq_cxt[i].vq = vq;
+	}
+
+	ops_data->vq_count = i;
+
+	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
+
+	return 0;
+
+fail_vq_create:
+	sfc_vdpa_dma_map(ops_data, false);
+
+fail_dma_map:
+fail_vring_num:
+	ops_data->state = SFC_VDPA_STATE_INITIALIZED;
+
+	return -1;
+}
+
+static void
+sfc_vdpa_close(struct sfc_vdpa_ops_data *ops_data)
+{
+	int i;
+
+	if (ops_data->state != SFC_VDPA_STATE_CONFIGURED)
+		return;
+
+	ops_data->state = SFC_VDPA_STATE_CLOSING;
+
+	for (i = 0; i < ops_data->vq_count; i++) {
+		if (ops_data->vq_cxt[i].vq == NULL)
+			continue;
+
+		efx_virtio_qdestroy(ops_data->vq_cxt[i].vq);
+	}
+
+	sfc_vdpa_dma_map(ops_data, false);
+
+	ops_data->state = SFC_VDPA_STATE_INITIALIZED;
+}
+
+static void
+sfc_vdpa_stop(struct sfc_vdpa_ops_data *ops_data)
+{
+	int i;
+	int rc;
+
+	if (ops_data->state != SFC_VDPA_STATE_STARTED)
+		return;
+
+	ops_data->state = SFC_VDPA_STATE_STOPPING;
+
+	for (i = 0; i < ops_data->vq_count; i++) {
+		rc = sfc_vdpa_virtq_stop(ops_data, i);
+		if (rc != 0)
+			continue;
+	}
+
+	sfc_vdpa_disable_vfio_intr(ops_data);
+
+	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
+}
+
+static int
+sfc_vdpa_start(struct sfc_vdpa_ops_data *ops_data)
+{
+	int i, j;
+	int rc;
+
+	SFC_EFX_ASSERT(ops_data->state == SFC_VDPA_STATE_CONFIGURED);
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "entry");
+
+	ops_data->state = SFC_VDPA_STATE_STARTING;
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "enable interrupts");
+	rc = sfc_vdpa_enable_vfio_intr(ops_data);
+	if (rc < 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "vfio intr allocation failed: %s",
+			     rte_strerror(rc));
+		goto fail_enable_vfio_intr;
+	}
+
+	rte_vhost_get_negotiated_features(ops_data->vid,
+					  &ops_data->req_features);
+
+	sfc_vdpa_info(ops_data->dev_handle,
+		      "negotiated feature : 0x%" PRIx64,
+		      ops_data->req_features);
+
+	for (i = 0; i < ops_data->vq_count; i++) {
+		sfc_vdpa_log_init(ops_data->dev_handle,
+				  "starting vq# %d", i);
+		rc = sfc_vdpa_virtq_start(ops_data, i);
+		if (rc != 0)
+			goto fail_vq_start;
+	}
+
+	ops_data->state = SFC_VDPA_STATE_STARTED;
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "done");
+
+	return 0;
+
+fail_vq_start:
+	/* stop already started virtqueues */
+	for (j = 0; j < i; j++)
+		sfc_vdpa_virtq_stop(ops_data, j);
+	sfc_vdpa_disable_vfio_intr(ops_data);
+
+fail_enable_vfio_intr:
+	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
+
+	return rc;
+}
+
+static int
+sfc_vdpa_get_queue_num(struct rte_vdpa_device *vdpa_dev, uint32_t *queue_num)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	void *dev;
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+	*queue_num = sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count;
+
+	sfc_vdpa_info(dev, "vDPA ops get_queue_num :: supported queue num : %d",
+		      *queue_num);
+
+	return 0;
+}
+
 static int
 sfc_vdpa_get_features(struct rte_vdpa_device *vdpa_dev, uint64_t *features)
 {

@@ -114,7 +540,53 @@
 static int
 sfc_vdpa_dev_config(int vid)
 {
-	RTE_SET_USED(vid);
+	struct rte_vdpa_device *vdpa_dev;
+	int rc;
+	struct sfc_vdpa_ops_data *ops_data;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "entry");
+
+	ops_data->vid = vid;
+
+	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "configuring");
+	rc = sfc_vdpa_configure(ops_data);
+	if (rc != 0)
+		goto fail_vdpa_config;
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "starting");
+	rc = sfc_vdpa_start(ops_data);
+	if (rc != 0)
+		goto fail_vdpa_start;
+
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "vhost notifier ctrl");
+	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "vDPA (%s): software relay for notify is used.",
+			      vdpa_dev->device->name);
+
+	sfc_vdpa_log_init(ops_data->dev_handle, "done");
+
+	return 0;
+
+fail_vdpa_start:
+	sfc_vdpa_close(ops_data);
+
+fail_vdpa_config:
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
 
 	return -1;
 }

@@ -122,9 +594,27 @@
 static int
 sfc_vdpa_dev_close(int vid)
 {
-	RTE_SET_USED(vid);
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
 
-	return -1;
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}
+
+	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+
+	sfc_vdpa_stop(ops_data);
+	sfc_vdpa_close(ops_data);
+
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+
+	return 0;
 }
 
 static int

drivers/vdpa/sfc/sfc_vdpa_ops.h +28 -1

index 21cbb73..8d553c5 100644
@@ -18,17 +18,45 @@ enum sfc_vdpa_context {
 enum sfc_vdpa_state {
 	SFC_VDPA_STATE_UNINITIALIZED = 0,
 	SFC_VDPA_STATE_INITIALIZED,
+	SFC_VDPA_STATE_CONFIGURING,
+	SFC_VDPA_STATE_CONFIGURED,
+	SFC_VDPA_STATE_CLOSING,
+	SFC_VDPA_STATE_CLOSED,
+	SFC_VDPA_STATE_STARTING,
+	SFC_VDPA_STATE_STARTED,
+	SFC_VDPA_STATE_STOPPING,
 	SFC_VDPA_STATE_NSTATES
 };
 
+struct sfc_vdpa_vring_info {
+	uint64_t	desc;
+	uint64_t	avail;
+	uint64_t	used;
+	uint64_t	size;
+	uint16_t	last_avail_idx;
+	uint16_t	last_used_idx;
+};
+
+typedef struct sfc_vdpa_vq_context_s {
+	uint8_t				enable;
+	uint32_t			pidx;
+	uint32_t			cidx;
+	efx_virtio_vq_t			*vq;
+} sfc_vdpa_vq_context_t;
+
 struct sfc_vdpa_ops_data {
 	void				*dev_handle;
+	int				vid;
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;
+	uint64_t			req_features;
+
+	uint16_t			vq_count;
+	struct sfc_vdpa_vq_context_s	vq_cxt[SFC_VDPA_MAX_QUEUE_PAIRS * 2];
 };
 
 struct sfc_vdpa_ops_data *
--

1.8.3.1

Vijay Srivastava

author

2021-10-29 14:49 UTC #86 lore raw

PATCH v3 7/10 vdpa/sfc: add support to get queue notify area info

From: Vijay Kumar Srivastava <[email protected]>

Implement the vDPA ops get_notify_area to get the notify area info of the queue.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
v2:
* Added error log in sfc_vdpa_get_notify_area.

 drivers/vdpa/sfc/sfc_vdpa_ops.c | 168 ++++++++++++++++++++++++++++++++++++++--
 drivers/vdpa/sfc/sfc_vdpa_ops.h |   2 +
 2 files changed, 164 insertions(+), 6 deletions(-)

drivers/vdpa/sfc/sfc_vdpa_ops.c +162 -6

index de1c81a..774d73e 100644
@@ -3,6 +3,8 @@
  * Copyright(c) 2020-2021 Xilinx, Inc.
  */
 
+#include <pthread.h>
+#include <unistd.h>
 #include <sys/ioctl.h>
 
 #include <rte_errno.h>

@@ -537,6 +539,67 @@
 	return 0;
 }
 
+static void *
+sfc_vdpa_notify_ctrl(void *arg)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	int vid;
+
+	ops_data = arg;
+	if (ops_data == NULL)
+		return NULL;
+
+	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+
+	vid = ops_data->vid;
+
+	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "vDPA (%s): Notifier could not get configured",
+			      ops_data->vdpa_dev->device->name);
+
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+
+	return NULL;
+}
+
+static int
+sfc_vdpa_setup_notify_ctrl(int vid)
+{
+	int ret;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}
+
+	ops_data->is_notify_thread_started = false;
+
+	/*
+	 * Use rte_vhost_host_notifier_ctrl in a thread to avoid
+	 * dead lock scenario when multiple VFs are used in single vdpa
+	 * application and multiple VFs are passed to a single VM.
+	 */
+	ret = pthread_create(&ops_data->notify_tid, NULL,
+			     sfc_vdpa_notify_ctrl, ops_data);
+	if (ret != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "failed to create notify_ctrl thread: %s",
+			     rte_strerror(ret));
+		return -1;
+	}
+	ops_data->is_notify_thread_started = true;
+
+	return 0;
+}
+
 static int
 sfc_vdpa_dev_config(int vid)
 {

@@ -570,18 +633,19 @@
 	if (rc != 0)
 		goto fail_vdpa_start;
 
-	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+	rc = sfc_vdpa_setup_notify_ctrl(vid);
+	if (rc != 0)
+		goto fail_vdpa_notify;
 
-	sfc_vdpa_log_init(ops_data->dev_handle, "vhost notifier ctrl");
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		sfc_vdpa_info(ops_data->dev_handle,
-			      "vDPA (%s): software relay for notify is used.",
-			      vdpa_dev->device->name);
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_vdpa_notify:
+	sfc_vdpa_stop(ops_data);
+
 fail_vdpa_start:
 	sfc_vdpa_close(ops_data);

@@ -594,6 +658,7 @@
 static int
 sfc_vdpa_dev_close(int vid)
 {
+	int ret;
 	struct rte_vdpa_device *vdpa_dev;
 	struct sfc_vdpa_ops_data *ops_data;

@@ -608,6 +673,23 @@
 	}
 
 	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+	if (ops_data->is_notify_thread_started == true) {
+		void *status;
+		ret = pthread_cancel(ops_data->notify_tid);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to cancel notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+
+		ret = pthread_join(ops_data->notify_tid, &status);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to join terminated notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+	}
+	ops_data->is_notify_thread_started = false;
 
 	sfc_vdpa_stop(ops_data);
 	sfc_vdpa_close(ops_data);

@@ -658,6 +740,79 @@
 	return vfio_dev_fd;
 }
 
+static int
+sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
+{
+	int ret;
+	efx_nic_t *nic;
+	int vfio_dev_fd;
+	efx_rc_t rc;
+	unsigned int bar_offset;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	struct vfio_region_info reg = { .argsz = sizeof(reg) };
+	const efx_nic_cfg_t *encp;
+	int max_vring_cnt;
+	int64_t len;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	max_vring_cnt =
+		(sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
+	encp = efx_nic_cfg_get(nic);
+
+	if (qid >= max_vring_cnt) {
+		sfc_vdpa_err(dev, "invalid qid : %d", qid);
+		return -1;
+	}
+
+	if (ops_data->vq_cxt[qid].enable != B_TRUE) {
+		sfc_vdpa_err(dev, "vq is not enabled");
+		return -1;
+	}
+
+	rc = efx_virtio_get_doorbell_offset(ops_data->vq_cxt[qid].vq,
+					    &bar_offset);
+	if (rc != 0) {
+		sfc_vdpa_err(dev, "failed to get doorbell offset: %s",
+			     rte_strerror(rc));
+		return rc;
+	}
+
+	reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
+	if (ret != 0) {
+		sfc_vdpa_err(dev, "could not get device region info: %s",
+			     strerror(errno));
+		return ret;
+	}
+
+	*offset = reg.offset + bar_offset;
+
+	len = (1U << encp->enc_vi_window_shift) / 2;
+	if (len >= sysconf(_SC_PAGESIZE)) {
+		*size = sysconf(_SC_PAGESIZE);
+	} else {
+		sfc_vdpa_err(dev, "invalid VI window size : 0x%" PRIx64, len);
+		return -1;
+	}
+
+	sfc_vdpa_info(dev, "vDPA ops get_notify_area :: offset : 0x%" PRIx64,
+		      *offset);
+
+	return 0;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -667,6 +822,7 @@
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
 	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
+	.get_notify_area = sfc_vdpa_get_notify_area,
 };
 
 struct sfc_vdpa_ops_data *

drivers/vdpa/sfc/sfc_vdpa_ops.h +2 -0

index 8d553c5..f7523ef 100644
@@ -50,6 +50,8 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+	pthread_t			notify_tid;
+	bool				is_notify_thread_started;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;

Signature · 1 line

1.8.3.1

Xia, Chenbo

2021-11-02 07:36 UTC #87 lore raw

Re: [dpdk-dev] [PATCH v3 07/10] vdpa/sfc: add support to get queue notify area info

Hi Vijay,

Quoted 71 lines

-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 07/10] vdpa/sfc: add support to get queue notify area info
From: Vijay Kumar Srivastava <[email protected]>
Implement the vDPA ops get_notify_area to get the notify area info of the queue.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---
v2:
* Added error log in sfc_vdpa_get_notify_area.
 drivers/vdpa/sfc/sfc_vdpa_ops.c | 168 ++++++++++++++++++++++++++++++++++++++-
(quoted diff) +0 -1
-
 drivers/vdpa/sfc/sfc_vdpa_ops.h |   2 +
 2 files changed, 164 insertions(+), 6 deletions(-)
drivers/vdpa/sfc/sfc_vdpa_ops.c +32 -0
index de1c81a..774d73e 100644
@@ -3,6 +3,8 @@
  * Copyright(c) 2020-2021 Xilinx, Inc.
  */
 
+#include <pthread.h>
+#include <unistd.h>
 #include <sys/ioctl.h>
 
 #include <rte_errno.h>
@@ -537,6 +539,67 @@
 	return 0;
 }
 
+static void *
+sfc_vdpa_notify_ctrl(void *arg)
+{
+	struct sfc_vdpa_ops_data *ops_data;
+	int vid;
+
+	ops_data = arg;
+	if (ops_data == NULL)
+		return NULL;
+
+	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+
+	vid = ops_data->vid;
+
+	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
+		sfc_vdpa_info(ops_data->dev_handle,
+			      "vDPA (%s): Notifier could not get configured",
+			      ops_data->vdpa_dev->device->name);
+
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+
+	return NULL;
+}
+
+static int
+sfc_vdpa_setup_notify_ctrl(int vid)
+{
+	int ret;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;

drivers/vdpa/sfc/sfc_vdpa_ops.c line 581

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0

+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}

Why not use struct sfc_vdpa_ops_data * as the input param rather than vid, then use ops_data->vdpa_dev to get vdpa_dev?

As ops_data is checked as non-NULL before the func, it will make things easier.

Thanks,
Chenbo

Quoted 186 lines

drivers/vdpa/sfc/sfc_vdpa_ops.c +46 -6

+
+	ops_data->is_notify_thread_started = false;
+
+	/*
+	 * Use rte_vhost_host_notifier_ctrl in a thread to avoid
+	 * dead lock scenario when multiple VFs are used in single vdpa
+	 * application and multiple VFs are passed to a single VM.
+	 */
+	ret = pthread_create(&ops_data->notify_tid, NULL,
+			     sfc_vdpa_notify_ctrl, ops_data);
+	if (ret != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "failed to create notify_ctrl thread: %s",
+			     rte_strerror(ret));
+		return -1;
+	}
+	ops_data->is_notify_thread_started = true;
+
+	return 0;
+}
+
 static int
 sfc_vdpa_dev_config(int vid)
 {

@@ -570,18 +633,19 @@
 	if (rc != 0)
 		goto fail_vdpa_start;
 
-	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
+	rc = sfc_vdpa_setup_notify_ctrl(vid);
+	if (rc != 0)
+		goto fail_vdpa_notify;
 
-	sfc_vdpa_log_init(ops_data->dev_handle, "vhost notifier ctrl");
-	if (rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true) != 0)
-		sfc_vdpa_info(ops_data->dev_handle,
-			      "vDPA (%s): software relay for notify is used.",
-			      vdpa_dev->device->name);
+	sfc_vdpa_adapter_unlock(ops_data->dev_handle);
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_vdpa_notify:
+	sfc_vdpa_stop(ops_data);
+
 fail_vdpa_start:
 	sfc_vdpa_close(ops_data);

@@ -594,6 +658,7 @@
 static int
 sfc_vdpa_dev_close(int vid)
 {
+	int ret;
 	struct rte_vdpa_device *vdpa_dev;
 	struct sfc_vdpa_ops_data *ops_data;

@@ -608,6 +673,23 @@
 	}
 
 	sfc_vdpa_adapter_lock(ops_data->dev_handle);
+	if (ops_data->is_notify_thread_started == true) {
+		void *status;
+		ret = pthread_cancel(ops_data->notify_tid);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to cancel notify_ctrl thread: %s",
+				     rte_strerror(ret));
+		}
+
+		ret = pthread_join(ops_data->notify_tid, &status);
+		if (ret != 0) {
+			sfc_vdpa_err(ops_data->dev_handle,
+				     "failed to join terminated notify_ctrl
 thread: %s",
+				     rte_strerror(ret));
+		}
+	}
+	ops_data->is_notify_thread_started = false;
 
 	sfc_vdpa_stop(ops_data);

drivers/vdpa/sfc/sfc_vdpa_ops.c +74 -0

 	sfc_vdpa_close(ops_data);

@@ -658,6 +740,79 @@
 	return vfio_dev_fd;
 }
 
+static int
+sfc_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
+{
+	int ret;
+	efx_nic_t *nic;
+	int vfio_dev_fd;
+	efx_rc_t rc;
+	unsigned int bar_offset;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
+	struct vfio_region_info reg = { .argsz = sizeof(reg) };
+	const efx_nic_cfg_t *encp;
+	int max_vring_cnt;
+	int64_t len;
+	void *dev;
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL)
+		return -1;
+
+	dev = ops_data->dev_handle;
+
+	vfio_dev_fd = sfc_vdpa_adapter_by_dev_handle(dev)->vfio_dev_fd;
+	max_vring_cnt =
+		(sfc_vdpa_adapter_by_dev_handle(dev)->max_queue_count * 2);
+
+	nic = sfc_vdpa_adapter_by_dev_handle(ops_data->dev_handle)->nic;
+	encp = efx_nic_cfg_get(nic);
+
+	if (qid >= max_vring_cnt) {
+		sfc_vdpa_err(dev, "invalid qid : %d", qid);
+		return -1;
+	}
+
+	if (ops_data->vq_cxt[qid].enable != B_TRUE) {
+		sfc_vdpa_err(dev, "vq is not enabled");
+		return -1;
+	}
+
+	rc = efx_virtio_get_doorbell_offset(ops_data->vq_cxt[qid].vq,
+					    &bar_offset);
+	if (rc != 0) {
+		sfc_vdpa_err(dev, "failed to get doorbell offset: %s",
+			     rte_strerror(rc));
+		return rc;
+	}
+
+	reg.index = sfc_vdpa_adapter_by_dev_handle(dev)->mem_bar.esb_rid;
+	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg);
+	if (ret != 0) {
+		sfc_vdpa_err(dev, "could not get device region info: %s",
+			     strerror(errno));
+		return ret;
+	}
+
+	*offset = reg.offset + bar_offset;
+
+	len = (1U << encp->enc_vi_window_shift) / 2;
+	if (len >= sysconf(_SC_PAGESIZE)) {
+		*size = sysconf(_SC_PAGESIZE);
+	} else {
+		sfc_vdpa_err(dev, "invalid VI window size : 0x%" PRIx64, len);
+		return -1;
+	}
+
+	sfc_vdpa_info(dev, "vDPA ops get_notify_area :: offset : 0x%" PRIx64,
+		      *offset);
+
+	return 0;
+}
+
 static struct rte_vdpa_dev_ops sfc_vdpa_ops = {
 	.get_queue_num = sfc_vdpa_get_queue_num,
 	.get_features = sfc_vdpa_get_features,

@@ -667,6 +822,7 @@
 	.set_vring_state = sfc_vdpa_set_vring_state,
 	.set_features = sfc_vdpa_set_features,
 	.get_vfio_device_fd = sfc_vdpa_get_vfio_device_fd,
+	.get_notify_area = sfc_vdpa_get_notify_area,
 };
 
 struct sfc_vdpa_ops_data *

drivers/vdpa/sfc/sfc_vdpa_ops.h +2 -1

index 8d553c5..f7523ef 100644
@@ -50,6 +50,8 @@ struct sfc_vdpa_ops_data {
 	struct rte_vdpa_device		*vdpa_dev;
 	enum sfc_vdpa_context		vdpa_context;
 	enum sfc_vdpa_state		state;
+	pthread_t			notify_tid;
+	bool				is_notify_thread_started;
 
 	uint64_t			dev_features;
 	uint64_t			drv_features;
--

1.8.3.1

Vijay Kumar Srivastava

2021-11-02 09:47 UTC #88 lore raw

Re: [dpdk-dev] [PATCH v3 07/10] vdpa/sfc: add support to get queue notify area info

Hi Chenbo,

Quoted 29 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Tuesday, November 2, 2021 1:05 PM
To: Vijay Kumar Srivastava <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: RE: [PATCH v3 07/10] vdpa/sfc: add support to get queue notify area info
Hi Vijay,
Quoted 18 lines
-----Original Message-----
From: Vijay Srivastava <[email protected]>
Sent: Friday, October 29, 2021 10:47 PM
To: [email protected]
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Vijay Kumar Srivastava <[email protected]>
Subject: [PATCH v3 07/10] vdpa/sfc: add support to get queue notify area info
From: Vijay Kumar Srivastava <[email protected]>
Implement the vDPA ops get_notify_area to get the notify area info of the queue.
Signed-off-by Vijay Kumar Srivastava <[email protected]>
Acked-by Andrew Rybchenko <[email protected]>
---

[SNIP]

Quoted 20 lines

Quoted 6 lines
drivers/vdpa/sfc/sfc_vdpa_ops.c +6 -0
+static int
+sfc_vdpa_setup_notify_ctrl(int vid)
+{
+	int ret;
+	struct rte_vdpa_device *vdpa_dev;
+	struct sfc_vdpa_ops_data *ops_data;
drivers/vdpa/sfc/sfc_vdpa_ops.c line 581
Quoted 10 lines
drivers/vdpa/sfc/sfc_vdpa_ops.c +10 -0
+
+	vdpa_dev = rte_vhost_get_vdpa_device(vid);
+
+	ops_data = sfc_vdpa_get_data_by_dev(vdpa_dev);
+	if (ops_data == NULL) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "invalid vDPA device : %p, vid : %d",
+			     vdpa_dev, vid);
+		return -1;
+	}
Why not use struct sfc_vdpa_ops_data * as the input param rather than vid, then use ops_data->vdpa_dev to get vdpa_dev?

Quoted 2 lines

As ops_data is checked as non-NULL before the func, it will make things easier.

Yes. ops_data can be used as input param.
I will include this change.

Vijay Srivastava

author

2021-10-29 14:49 UTC #89 lore raw

PATCH v3 8/10 vdpa/sfc: add support for MAC filter config

From: Vijay Kumar Srivastava <[email protected]>

Add support for unicast and broadcast MAC filter configuration.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst        |   4 ++
 drivers/vdpa/sfc/meson.build       |   1 +
 drivers/vdpa/sfc/sfc_vdpa.c        |  32 +++++++++
 drivers/vdpa/sfc/sfc_vdpa.h        |  30 ++++++++
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 144 +++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c     |  10 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c    |  17 +++++
 7 files changed, 238 insertions(+)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c

doc/guides/vdpadevs/sfc.rst +4 -0

index d06c427..512f23e 100644
@@ -71,6 +71,10 @@ boolean parameters value.
   **vdpa** device will work as vdpa device and will be probed by vdpa/sfc driver.
   If this parameter is not specified then ef100 device will operate as network device.
 
+- ``mac`` [mac address]
+
+  Configures MAC address which would be used to setup MAC filters.
+
 
 Dynamic Logging Parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

drivers/vdpa/sfc/meson.build +1 -0

index dc333de..2ca33bc 100644
@@ -22,4 +22,5 @@ sources = files(
 	'sfc_vdpa_hw.c',
 	'sfc_vdpa_mcdi.c',
 	'sfc_vdpa_ops.c',
+	'sfc_vdpa_filter.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +32 -0

index b3c82e5..d18cd61 100644
@@ -8,7 +8,9 @@
 #include <sys/queue.h>
 
 #include <rte_common.h>
+#include <rte_devargs.h>
 #include <rte_errno.h>
+#include <rte_kvargs.h>
 #include <rte_string_fns.h>
 #include <rte_vfio.h>
 #include <rte_vhost.h>

@@ -202,6 +204,31 @@ struct sfc_vdpa_ops_data *
 	return ret < 0 ? RTE_LOGTYPE_PMD : ret;
 }
 
+static int
+sfc_vdpa_kvargs_parse(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	struct rte_devargs *devargs = pci_dev->device.devargs;
+	/*
+	 * To get the device class a mandatory param 'class' is being
+	 * used so included SFC_EFX_KVARG_DEV_CLASS in the param list.
+	 */
+	const char **params = (const char *[]){
+		RTE_DEVARGS_KEY_CLASS,
+		SFC_VDPA_MAC_ADDR,
+		NULL,
+	};
+
+	if (devargs == NULL)
+		return 0;
+
+	sva->kvargs = rte_kvargs_parse(devargs->args, params);
+	if (sva->kvargs == NULL)
+		return -EINVAL;
+
+	return 0;
+}
+
 static struct rte_pci_id pci_id_sfc_vdpa_efx_map[] = {
 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) },
 	{ .vendor_id = 0, /* sentinel */ },

@@ -244,6 +271,10 @@ struct sfc_vdpa_ops_data *
 	if (ret != 0)
 		goto fail_set_log_prefix;
 
+	ret = sfc_vdpa_kvargs_parse(sva);
+	if (ret != 0)
+		goto fail_kvargs_parse;
+
 	sfc_vdpa_log_init(sva, "entry");
 
 	sfc_vdpa_adapter_lock_init(sva);

@@ -284,6 +315,7 @@ struct sfc_vdpa_ops_data *
 fail_vfio_setup:
 	sfc_vdpa_adapter_lock_fini(sva);
 
+fail_kvargs_parse:
 fail_set_log_prefix:
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +30 -0

index 1bf96e7..dbd099f 100644
@@ -17,8 +17,29 @@
 #include "sfc_vdpa_log.h"
 #include "sfc_vdpa_ops.h"
 
+#define SFC_VDPA_MAC_ADDR			"mac"
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
+/* Broadcast & Unicast MAC filters are supported */
+#define SFC_MAX_SUPPORTED_FILTERS		2
+
+/*
+ * Get function-local index of the associated VI from the
+ * virtqueue number. Queue 0 is reserved for MCDI
+ */
+#define SFC_VDPA_GET_VI_INDEX(vq_num) (((vq_num) / 2) + 1)
+
+enum sfc_vdpa_filter_type {
+	SFC_VDPA_BCAST_MAC_FILTER = 0,
+	SFC_VDPA_UCAST_MAC_FILTER = 1,
+	SFC_VDPA_FILTER_NTYPE
+};
+
+typedef struct sfc_vdpa_filter_s {
+	int				filter_cnt;
+	efx_filter_spec_t		spec[SFC_MAX_SUPPORTED_FILTERS];
+} sfc_vdpa_filter_t;
+
 /* Adapter private data */
 struct sfc_vdpa_adapter {
 	TAILQ_ENTRY(sfc_vdpa_adapter)	next;

@@ -32,6 +53,8 @@ struct sfc_vdpa_adapter {
 	struct rte_pci_device		*pdev;
 	struct rte_pci_addr		pci_addr;
 
+	struct rte_kvargs		*kvargs;
+
 	efx_family_t			family;
 	efx_nic_t			*nic;
 	rte_spinlock_t			nic_lock;

@@ -46,6 +69,8 @@ struct sfc_vdpa_adapter {
 	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
 	uint32_t			logtype_main;
 
+	sfc_vdpa_filter_t		filters;
+
 	int				vfio_group_fd;
 	int				vfio_dev_fd;
 	int				vfio_container_fd;

@@ -83,6 +108,11 @@ struct sfc_vdpa_ops_data *
 int
 sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *vdpa_data, bool do_map);
 
+int
+sfc_vdpa_filter_remove(struct sfc_vdpa_ops_data *ops_data);
+int
+sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data);
+
 static inline struct sfc_vdpa_adapter *
 sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
 {

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +144 -0

144 lines not on this page · raw message

drivers/vdpa/sfc/sfc_vdpa_hw.c +10 -0

index b473708..5307b03 100644
@@ -354,10 +354,20 @@
 		goto fail_virtio_init;
 	}
 
+	sfc_vdpa_log_init(sva, "init filter");
+	rc = efx_filter_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "filter init failed: %s", rte_strerror(rc));
+		goto fail_filter_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_filter_init:
+	efx_virtio_fini(enp);
+
 fail_virtio_init:
 	efx_nic_fini(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +17 -0

index 774d73e..8551b65 100644
@@ -426,6 +426,8 @@
 
 	sfc_vdpa_disable_vfio_intr(ops_data);
 
+	sfc_vdpa_filter_remove(ops_data);
+
 	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
 }

@@ -465,12 +467,27 @@
 			goto fail_vq_start;
 	}
 
+	ops_data->vq_count = i;
+
+	sfc_vdpa_log_init(ops_data->dev_handle,
+			  "configure MAC filters");
+	rc = sfc_vdpa_filter_config(ops_data);
+	if (rc != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "MAC filter config failed: %s",
+			     rte_strerror(rc));
+		goto fail_filter_cfg;
+	}
+
 	ops_data->state = SFC_VDPA_STATE_STARTED;
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_filter_cfg:
+	/* remove already created filters */
+	sfc_vdpa_filter_remove(ops_data);
 fail_vq_start:
 	/* stop already started virtqueues */
 	for (j = 0; j < i; j++)

Signature · 1 line

1.8.3.1

Xia, Chenbo

2021-11-02 08:18 UTC #90 lore raw

Re: [dpdk-dev] [PATCH v3 08/10] vdpa/sfc: add support for MAC filter config

Hi Vijay,

Quoted 244 lines

From: Vijay Kumar Srivastava <[email protected]>

Add support for unicast and broadcast MAC filter configuration.

Signed-off-by Vijay Kumar Srivastava <[email protected]>

---
 doc/guides/vdpadevs/sfc.rst        |   4 ++
 drivers/vdpa/sfc/meson.build       |   1 +
 drivers/vdpa/sfc/sfc_vdpa.c        |  32 +++++++++
 drivers/vdpa/sfc/sfc_vdpa.h        |  30 ++++++++
 drivers/vdpa/sfc/sfc_vdpa_filter.c | 144
+++++++++++++++++++++++++++++++++++++
 drivers/vdpa/sfc/sfc_vdpa_hw.c     |  10 +++
 drivers/vdpa/sfc/sfc_vdpa_ops.c    |  17 +++++
 7 files changed, 238 insertions(+)
 create mode 100644 drivers/vdpa/sfc/sfc_vdpa_filter.c

doc/guides/vdpadevs/sfc.rst +0 -0

index d06c427..512f23e 100644
@@ -71,6 +71,10 @@ boolean parameters value.
   **vdpa** device will work as vdpa device and will be probed by vdpa/sfc

driver.
If this parameter is not specified then ef100 device will operate as
network device.

+- ``mac`` [mac address]

doc/guides/vdpadevs/sfc.rst +3 -0

+
+  Configures MAC address which would be used to setup MAC filters.
+
 
 Dynamic Logging Parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

drivers/vdpa/sfc/meson.build +1 -0

index dc333de..2ca33bc 100644
@@ -22,4 +22,5 @@ sources = files(
 	'sfc_vdpa_hw.c',
 	'sfc_vdpa_mcdi.c',
 	'sfc_vdpa_ops.c',
+	'sfc_vdpa_filter.c',
 )

drivers/vdpa/sfc/sfc_vdpa.c +32 -0

index b3c82e5..d18cd61 100644
@@ -8,7 +8,9 @@
 #include <sys/queue.h>
 
 #include <rte_common.h>
+#include <rte_devargs.h>
 #include <rte_errno.h>
+#include <rte_kvargs.h>
 #include <rte_string_fns.h>
 #include <rte_vfio.h>
 #include <rte_vhost.h>

@@ -202,6 +204,31 @@ struct sfc_vdpa_ops_data *
 	return ret < 0 ? RTE_LOGTYPE_PMD : ret;
 }
 
+static int
+sfc_vdpa_kvargs_parse(struct sfc_vdpa_adapter *sva)
+{
+	struct rte_pci_device *pci_dev = sva->pdev;
+	struct rte_devargs *devargs = pci_dev->device.devargs;
+	/*
+	 * To get the device class a mandatory param 'class' is being
+	 * used so included SFC_EFX_KVARG_DEV_CLASS in the param list.
+	 */
+	const char **params = (const char *[]){
+		RTE_DEVARGS_KEY_CLASS,
+		SFC_VDPA_MAC_ADDR,
+		NULL,
+	};
+
+	if (devargs == NULL)
+		return 0;
+
+	sva->kvargs = rte_kvargs_parse(devargs->args, params);
+	if (sva->kvargs == NULL)
+		return -EINVAL;
+
+	return 0;
+}
+
 static struct rte_pci_id pci_id_sfc_vdpa_efx_map[] = {
 	{ RTE_PCI_DEVICE(EFX_PCI_VENID_XILINX, EFX_PCI_DEVID_RIVERHEAD_VF) },
 	{ .vendor_id = 0, /* sentinel */ },

@@ -244,6 +271,10 @@ struct sfc_vdpa_ops_data *
 	if (ret != 0)
 		goto fail_set_log_prefix;
 
+	ret = sfc_vdpa_kvargs_parse(sva);
+	if (ret != 0)
+		goto fail_kvargs_parse;
+
 	sfc_vdpa_log_init(sva, "entry");
 
 	sfc_vdpa_adapter_lock_init(sva);

@@ -284,6 +315,7 @@ struct sfc_vdpa_ops_data *
 fail_vfio_setup:
 	sfc_vdpa_adapter_lock_fini(sva);
 
+fail_kvargs_parse:
 fail_set_log_prefix:
 	rte_free(sva);

drivers/vdpa/sfc/sfc_vdpa.h +30 -0

index 1bf96e7..dbd099f 100644
@@ -17,8 +17,29 @@
 #include "sfc_vdpa_log.h"
 #include "sfc_vdpa_ops.h"
 
+#define SFC_VDPA_MAC_ADDR			"mac"
 #define SFC_VDPA_DEFAULT_MCDI_IOVA		0x200000000000
 
+/* Broadcast & Unicast MAC filters are supported */
+#define SFC_MAX_SUPPORTED_FILTERS		2
+
+/*
+ * Get function-local index of the associated VI from the
+ * virtqueue number. Queue 0 is reserved for MCDI
+ */
+#define SFC_VDPA_GET_VI_INDEX(vq_num) (((vq_num) / 2) + 1)
+
+enum sfc_vdpa_filter_type {
+	SFC_VDPA_BCAST_MAC_FILTER = 0,
+	SFC_VDPA_UCAST_MAC_FILTER = 1,
+	SFC_VDPA_FILTER_NTYPE
+};
+
+typedef struct sfc_vdpa_filter_s {
+	int				filter_cnt;
+	efx_filter_spec_t		spec[SFC_MAX_SUPPORTED_FILTERS];
+} sfc_vdpa_filter_t;
+
 /* Adapter private data */
 struct sfc_vdpa_adapter {
 	TAILQ_ENTRY(sfc_vdpa_adapter)	next;

@@ -32,6 +53,8 @@ struct sfc_vdpa_adapter {
 	struct rte_pci_device		*pdev;
 	struct rte_pci_addr		pci_addr;
 
+	struct rte_kvargs		*kvargs;
+
 	efx_family_t			family;
 	efx_nic_t			*nic;
 	rte_spinlock_t			nic_lock;

@@ -46,6 +69,8 @@ struct sfc_vdpa_adapter {
 	char				log_prefix[SFC_VDPA_LOG_PREFIX_MAX];
 	uint32_t			logtype_main;
 
+	sfc_vdpa_filter_t		filters;
+
 	int				vfio_group_fd;
 	int				vfio_dev_fd;
 	int				vfio_container_fd;

@@ -83,6 +108,11 @@ struct sfc_vdpa_ops_data *
 int
 sfc_vdpa_dma_map(struct sfc_vdpa_ops_data *vdpa_data, bool do_map);
 
+int
+sfc_vdpa_filter_remove(struct sfc_vdpa_ops_data *ops_data);
+int
+sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data);
+
 static inline struct sfc_vdpa_adapter *
 sfc_vdpa_adapter_by_dev_handle(void *dev_handle)
 {

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +55 -0

new file mode 100644
index 0000000..03b6a5d
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright(c) 2020-2021 Xilinx, Inc.
+ */
+
+#include <rte_errno.h>
+#include <rte_ether.h>
+#include <rte_kvargs.h>
+
+#include "efx.h"
+#include "efx_impl.h"
+#include "sfc_vdpa.h"
+
+static inline int
+sfc_vdpa_get_eth_addr(const char *key __rte_unused,
+		      const char *value, void *extra_args)
+{
+	struct rte_ether_addr *mac_addr = extra_args;
+
+	if (value == NULL || extra_args == NULL)
+		return -EINVAL;
+
+	/* Convert string with Ethernet address to an ether_addr */
+	rte_ether_unformat_addr(value, mac_addr);
+
+	return 0;
+}
+
+static int
+sfc_vdpa_set_mac_filter(efx_nic_t *nic, efx_filter_spec_t *spec,
+			int qid, uint8_t *eth_addr)
+{
+	int rc;
+
+	if (nic == NULL || spec == NULL)
+		return -1;
+
+	spec->efs_priority = EFX_FILTER_PRI_MANUAL;
+	spec->efs_flags = EFX_FILTER_FLAG_RX;
+	spec->efs_dmaq_id = qid;
+
+	rc = efx_filter_spec_set_eth_local(spec, EFX_FILTER_SPEC_VID_UNSPEC,
+					   eth_addr);
+	if (rc != 0)
+		return rc;
+
+	rc = efx_filter_insert(nic, spec);
+	if (rc != 0)
+		return rc;
+
+	return rc;
+}
+
+int sfc_vdpa_filter_config(struct sfc_vdpa_ops_data *ops_data)
+{

drivers/vdpa/sfc/sfc_vdpa_filter.c line 65

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +10 -0

+	int rc;
+	int qid;
+	efx_nic_t *nic;
+	struct rte_ether_addr bcast_eth_addr;
+	struct rte_ether_addr ucast_eth_addr;
+	struct sfc_vdpa_adapter *sva = ops_data->dev_handle;
+	efx_filter_spec_t *spec;
+
+	if (ops_data == NULL)
+		return -1;

You check this after you use the pointer to reference adapter. One option is to remove the check as I don't think it will be NULL.

Quoted 22 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +11 -0

+
+	sfc_vdpa_log_init(sva, "entry");
+
+	nic = sva->nic;
+
+	sfc_vdpa_log_init(sva, "process kvarg");
+
+	/* skip MAC filter configuration if mac address is not provided */
+	if (rte_kvargs_count(sva->kvargs, SFC_VDPA_MAC_ADDR) == 0) {
+		sfc_vdpa_warn(sva,
+			      "MAC address is not provided, skipping MAC Filter

Config");

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +10 -0

+		return -1;
+	}
+
+	rc = rte_kvargs_process(sva->kvargs, SFC_VDPA_MAC_ADDR,
+				&sfc_vdpa_get_eth_addr,
+				&ucast_eth_addr);
+	if (rc < 0)
+		return -1;
+
+	/* create filters on the base queue */

drivers/vdpa/sfc/sfc_vdpa_filter.c line 96

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +10 -0

+	qid = SFC_VDPA_GET_VI_INDEX(0);
+
+	sfc_vdpa_log_init(sva, "insert broadcast mac filter");
+
+	EFX_MAC_BROADCAST_ADDR_SET(bcast_eth_addr.addr_bytes);
+	spec = &sva->filters.spec[SFC_VDPA_BCAST_MAC_FILTER];
+
+	rc = sfc_vdpa_set_mac_filter(nic,
+				     spec, qid,
+				     bcast_eth_addr.addr_bytes);

Can improve to use two lines as it won't exceed 80

drivers/vdpa/sfc/sfc_vdpa_filter.c line 109

Quoted 13 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +13 -0

+	if (rc != 0)
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "broadcast MAC filter insertion failed: %s",
+			     rte_strerror(rc));
+	else
+		sva->filters.filter_cnt++;
+
+	sfc_vdpa_log_init(sva, "insert unicast mac filter");
+	spec = &sva->filters.spec[SFC_VDPA_UCAST_MAC_FILTER];
+
+	rc = sfc_vdpa_set_mac_filter(nic,
+				     spec, qid,
+				     ucast_eth_addr.addr_bytes);

Ditto.

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +10 -0

+	if (rc != 0)
+		sfc_vdpa_err(sva,
+			     "unicast MAC filter insertion failed: %s",
+			     rte_strerror(rc));
+	else
+		sva->filters.filter_cnt++;
+
+	sfc_vdpa_log_init(sva, "done");
+
+	return rc;

drivers/vdpa/sfc/sfc_vdpa_filter.c line 129

Quoted 10 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +10 -0

+}
+
+int sfc_vdpa_filter_remove(struct sfc_vdpa_ops_data *ops_data)
+{
+	int i, rc = 0;
+	struct sfc_vdpa_adapter *sva = ops_data->dev_handle;
+	efx_nic_t *nic;
+
+	if (ops_data == NULL)
+		return -1;

You check this after you use the pointer to reference adapter. One option is to remove the check as I don't think it will be NULL.

Thanks,
Chenbo

Quoted 83 lines

drivers/vdpa/sfc/sfc_vdpa_filter.c new file +15 -0

+
+	nic = sva->nic;
+
+	for (i = 0; i < sva->filters.filter_cnt; i++) {
+		rc = efx_filter_remove(nic, &(sva->filters.spec[i]));
+		if (rc != 0)
+			sfc_vdpa_err(sva,
+				     "remove HW filter failed for entry %d: %s",
+				     i, rte_strerror(rc));
+	}
+
+	sva->filters.filter_cnt = 0;
+
+	return rc;
+}

drivers/vdpa/sfc/sfc_vdpa_hw.c +10 -0

index b473708..5307b03 100644
@@ -354,10 +354,20 @@
 		goto fail_virtio_init;
 	}
 
+	sfc_vdpa_log_init(sva, "init filter");
+	rc = efx_filter_init(enp);
+	if (rc != 0) {
+		sfc_vdpa_err(sva, "filter init failed: %s", rte_strerror(rc));
+		goto fail_filter_init;
+	}
+
 	sfc_vdpa_log_init(sva, "done");
 
 	return 0;
 
+fail_filter_init:
+	efx_virtio_fini(enp);
+
 fail_virtio_init:
 	efx_nic_fini(enp);

drivers/vdpa/sfc/sfc_vdpa_ops.c +17 -1

index 774d73e..8551b65 100644
@@ -426,6 +426,8 @@
 
 	sfc_vdpa_disable_vfio_intr(ops_data);
 
+	sfc_vdpa_filter_remove(ops_data);
+
 	ops_data->state = SFC_VDPA_STATE_CONFIGURED;
 }

@@ -465,12 +467,27 @@
 			goto fail_vq_start;
 	}
 
+	ops_data->vq_count = i;
+
+	sfc_vdpa_log_init(ops_data->dev_handle,
+			  "configure MAC filters");
+	rc = sfc_vdpa_filter_config(ops_data);
+	if (rc != 0) {
+		sfc_vdpa_err(ops_data->dev_handle,
+			     "MAC filter config failed: %s",
+			     rte_strerror(rc));
+		goto fail_filter_cfg;
+	}
+
 	ops_data->state = SFC_VDPA_STATE_STARTED;
 
 	sfc_vdpa_log_init(ops_data->dev_handle, "done");
 
 	return 0;
 
+fail_filter_cfg:
+	/* remove already created filters */
+	sfc_vdpa_filter_remove(ops_data);
 fail_vq_start:
 	/* stop already started virtqueues */
 	for (j = 0; j < i; j++)
--

1.8.3.1

Vijay Srivastava

author

2021-10-29 14:49 UTC #91 lore raw

PATCH v3 9/10 vdpa/sfc: add support to set vring state

From: Vijay Kumar Srivastava <[email protected]>

Implements vDPA ops set_vring_state to configure vring state.

Signed-off-by Vijay Kumar Srivastava <[email protected]>