Skip to contentSkip to conversation

vhost: introduce rte_vdpa_get_device_num api

PATCH In review Xiaolong Ye 3 participants 6 messages Sep 6, 2018 – Sep 10, 2018 Arm/Amlogic Meson SoC support

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 vhost: introduce rte_vdpa_get_device_num api

---
 lib/librte_vhost/rte_vdpa.h | 3 +++
 lib/librte_vhost/vdpa.c     | 6 ++++++
 2 files changed, 9 insertions(+)
lib/librte_vhost/rte_vdpa.h +3 -0
index 90465ca26..b8223e337 100644
@@ -84,4 +84,7 @@ rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr);
 struct rte_vdpa_device * __rte_experimental
 rte_vdpa_get_device(int did);
 
+/* Get current available vdpa device number */
+int __rte_experimental
+rte_vdpa_get_device_num(void);
 #endif /* _RTE_VDPA_H_ */
lib/librte_vhost/vdpa.c +6 -0
index c82fd4370..c2c5dff1d 100644
@@ -113,3 +113,9 @@ rte_vdpa_get_device(int did)
 
 	return vdpa_devices[did];
 }
+
+int
+rte_vdpa_get_device_num(void)
+{
+	return vdpa_device_num;
+}
Signature · 1 line
2.18.0.rc1.1.g6f333ff2f
#1 lore raw

PATCH 2/2 examples/vdpa: add a new sample for vdpa

From: Xiao Wang <[email protected]>

This patch adds a sample which creates vhost-user socket based on vdpa driver. vdpa driver can help to set up vhost datapath so this app doesn't need to spend a dedicated worker thread on vhost enqueue/dequeue operations.

Below are setup steps for your reference (using IFC driver):

1. Make sure your kernnel vhost module and QEMU support vhost-user.
- OS: CentOS 7.4
- QEMU: 2.10.1
- Guest OS: CentOS 7.2

2. Bind VFIO-pci to VF
a) modprobe vfio-pci
d) ./usertools/dpdk-devbind.py -b vfio-pci 06:00.2 06:00.3

3. Start vdpa sample

    ./examples/vdpa/build/vdpa --log-level=9 -c 0x10 -n 4 --socket-mem
    1024,1024  -w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1   --
    --iface  /tmp/vhost-user- (or use interactive mode to create vdpa
    ports)

4. Start two VMs ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm \
<snip>
-mem-prealloc \
-chardev socket,id=char0,path=/tmp/vhost-user-0 \
-netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
-device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ee \

   ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm \
   <snip>
   -mem-prealloc \
   -chardev socket,id=char0,path=/tmp/vhost-user-1 \
   -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
   -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ff \

5. Login two VMs and configure the ip, verify the network connection via ping.

---
 examples/vdpa/Makefile    |  32 ++++
 examples/vdpa/main.c      | 327 ++++++++++++++++++++++++++++++++++++++
 examples/vdpa/meson.build |  11 ++
 3 files changed, 370 insertions(+)
 create mode 100644 examples/vdpa/Makefile
 create mode 100644 examples/vdpa/main.c
 create mode 100644 examples/vdpa/meson.build
examples/vdpa/Makefile new file +32 -0
new file mode 100644
index 000000000..42672a2bc
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+$(info This application can only operate in a linuxapp environment, \
+please change the definition of the RTE_TARGET environment variable)
+all:
+else
+
+# binary name
+APP = vdpa
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -D_GNU_SOURCE
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
examples/vdpa/main.c new file +327 -0

327 lines not on this page · raw message

examples/vdpa/meson.build new file +11 -0
new file mode 100644
index 000000000..e85d84a76
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+sources = files(
+	'main.c'
+)
Signature · 1 line
2.18.0.rc1.1.g6f333ff2f
#2 lore raw

Re: [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

Hi all,
First, thanks for the vdpa example patches.
Second, I am getting a compilation error under Ubuntu 18.04, with gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)
...
CC main.o
/work/src/dpdk/examples/vdpa/main.c: In function ‘main’:
/work/src/dpdk/examples/vdpa/main.c:321:5: error: ignoring return value of ‘scanf’, declared with attribute warn_unused_result [-Werror=unused-result]

     scanf("%c", &ch);
     ^~~~~~~~~~~~~~~~

cc1: all warnings being treated as errors /work/src/dpdk/mk/internal/rte.compile-pre.mk:114: recipe for target 'main.o' failed
make[1]: *** [main.o] Error 1

Also, it would be nice to have as part of this patch series adding the relevant info in
MAINTAINERS, doc/guides/sample_app_ug/index.rst , examples/Makefile and adding a doc/guides/sample_app_ug/vdpa.rst, like most patches for examples do.
See for example,

commit f5188211c721688bf8530d1648d623205246e1da

Author: Fan Zhang <[email protected]>
Date:   Thu Apr 5 17:01:36 2018 +0100

examples/vhost_crypto: add sample application

Regards,
Rami Rosen

#3 lore raw

Re: [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

Hi, Rosen.

Thanks a lot for your comments.

Quoted 10 lines · Rami Rosen

Hi all,
First, thanks for the vdpa example patches.
Second, I am getting a compilation error under Ubuntu 18.04, with gcc version 7.3.0 (Ubuntu 7.3.0-16ubuntu3)
...
CC main.o
/work/src/dpdk/examples/vdpa/main.c: In function ‘main’:
/work/src/dpdk/examples/vdpa/main.c:321:5: error: ignoring return value of ‘scanf’, declared with attribute warn_unused_result [-Werror=unused-result]

Quoted 7 lines
    scanf("%c", &ch);
    ^~~~~~~~~~~~~~~~

cc1: all warnings being treated as errors /work/src/dpdk/mk/internal/rte.compile-pre.mk:114: recipe for target 'main.o' failed
make[1]: *** [main.o] Error 1

I'll look into it and solve it in v2.

Quoted 7 lines

Also, it would be nice to have as part of this patch series adding the relevant info in
MAINTAINERS, doc/guides/sample_app_ug/index.rst , examples/Makefile and adding a doc/guides/sample_app_ug/vdpa.rst, like most patches for examples do.
See for example,

Quoted 5 lines

commit f5188211c721688bf8530d1648d623205246e1da

Author: Fan Zhang <[email protected]>
Date:   Thu Apr 5 17:01:36 2018 +0100

examples/vhost_crypto: add sample application

Got it, I'll add necessary documentations accordingly.

Thanks,
Xiaolong

Quoted 2 lines

Regards,
Rami Rosen

#4 lore raw

Re: [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

In reply to Xiaolong Ye

Can we add options to support 'int rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)' API calling to get the device id ? Not just hard code 0 ~ num ? Like choosing one of vDPA engines explicitly by BDF:
-w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1

BR,
Haiyue

Quoted 448 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Xiaolong Ye
Sent: Thursday, September 6, 2018 21:17
To: [email protected]; Maxime Coquelin <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>
Cc: Wang, Xiao W <[email protected]>; Ye, Xiaolong <[email protected]>
Subject: [dpdk-dev] [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

From: Xiao Wang <[email protected]>

This patch adds a sample which creates vhost-user socket based on vdpa driver. vdpa driver can help to set up vhost datapath so this app doesn't need to spend a dedicated worker thread on vhost enqueue/dequeue operations.

Below are setup steps for your reference (using IFC driver):

1. Make sure your kernnel vhost module and QEMU support vhost-user.
- OS: CentOS 7.4
- QEMU: 2.10.1
- Guest OS: CentOS 7.2

2. Bind VFIO-pci to VF
a) modprobe vfio-pci
d) ./usertools/dpdk-devbind.py -b vfio-pci 06:00.2 06:00.3

3. Start vdpa sample

    ./examples/vdpa/build/vdpa --log-level=9 -c 0x10 -n 4 --socket-mem
    1024,1024  -w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1   --
    --iface  /tmp/vhost-user- (or use interactive mode to create vdpa
    ports)

4. Start two VMs ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm

\
   <snip>
   -mem-prealloc \
   -chardev socket,id=char0,path=/tmp/vhost-user-0 \
   -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
   -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ee \

   ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm
\
   <snip>
   -mem-prealloc \
   -chardev socket,id=char0,path=/tmp/vhost-user-1 \
   -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
   -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ff \

5. Login two VMs and configure the ip, verify the network connection via ping.

---
 examples/vdpa/Makefile    |  32 ++++
 examples/vdpa/main.c      | 327 ++++++++++++++++++++++++++++++++++++++
 examples/vdpa/meson.build |  11 ++
 3 files changed, 370 insertions(+)
 create mode 100644 examples/vdpa/Makefile  create mode 100644

examples/vdpa/main.c create mode 100644 examples/vdpa/meson.build

examples/vdpa/Makefile new file mode renamed from examples/vdpa/Makefile +0 -0

100644 index 000000000..42672a2bc

examples/vdpa/Makefile new file +31 -0
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
+Corporation
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable") endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+$(info This application can only operate in a linuxapp environment, \
+please change the definition of the RTE_TARGET environment variable)
+all:
+else
+
+# binary name
+APP = vdpa
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -D_GNU_SOURCE
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
examples/vdpa/main.c new file mode renamed from examples/vdpa/main.c +0 -0

100644 index 000000000..b19d7a217

examples/vdpa/main.c new file +44 -0
@@ -0,0 +1,327 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <getopt.h>
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+#include <rte_vhost.h>
+#include <rte_vdpa.h>
+
+#include <cmdline_parse.h>
+#include <cmdline_socket.h>
+#include <cmdline_parse_string.h>
+#include <cmdline.h>
+
+#define MAX_PATH_LEN 128
+#define MAX_VDPA_SAMPLE_PORTS 1024
+
+struct vdpa_port {
+	char ifname[MAX_PATH_LEN];
+	int did;
+	int vid;
+};
+
+struct vdpa_port vports[MAX_VDPA_SAMPLE_PORTS];
+
+char iface[MAX_PATH_LEN];
+int devcnt;
+int dev_id;
+uint8_t interactive;
+
+/* display usage */
+static void
+vdpa_usage(const char *prgname)
+{
+	printf("Usage: %s [EAL options] -- "
+				 "[--interactive|-i] "
+				 "[--iface PATH]\n"
+				 "	--interactive: run in interactive

mode.\n"

examples/vdpa/main.c new file +1 -0
+				 "	--iface <path>: specify the path prefix

of the socket files, e.g. /tmp/vhost-user-\n",

examples/vdpa/main.c new file +165 -0
+				 prgname);
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+	static const char *short_option = "i";
+	static struct option long_option[] = {
+		{"iface", required_argument, NULL, 0},
+		{"interactive", 0, 0, 0},
+		{NULL, 0, 0, 0},
+	};
+	int opt, idx;
+	char *prgname = argv[0];
+
+	while ((opt = getopt_long(argc, argv, short_option, long_option, &idx))
+			!= EOF) {
+		switch (opt) {
+		case 'i':
+			printf("Interactive-mode selected\n");
+			interactive = 1;
+			break;
+		/* long options */
+		case 0:
+			if (strncmp(long_option[idx].name, "iface",
+						MAX_PATH_LEN) == 0) {
+				strncpy(iface, optarg, MAX_PATH_LEN);
+				printf("iface %s\n", iface);
+			}
+			if (!strcmp(long_option[idx].name, "interactive")) {
+				printf("Interactive-mode selected\n");
+				interactive = 1;
+			}
+			break;
+
+		default:
+			vdpa_usage(prgname);
+			return -1;
+		}
+	}
+
+	if (*iface == '\0' && interactive == 0) {
+		vdpa_usage(prgname);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+data_init(void)
+{
+	devcnt = rte_vdpa_get_device_num();
+	if (devcnt <= 0) {
+		printf("No available vdpa device found\n");
+		return -1;
+	}
+	dev_id = 0;
+	interactive = 0;
+	memset(iface, 0, MAX_PATH_LEN * sizeof(iface[0]));
+	memset(vports, 0, MAX_VDPA_SAMPLE_PORTS * sizeof(vports[0]));
+	return 0;
+}
+
+static void
+signal_handler(int signum)
+{
+	if (signum == SIGINT || signum == SIGTERM) {
+		printf("\nSignal %d received, preparing to exit...\n",
+				signum);
+		exit(0);
+	}
+}
+
+static int
+new_device(int vid)
+{
+	char ifname[MAX_PATH_LEN];
+	int i;
+
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
+		if (strncmp(ifname, vports[i].ifname, MAX_PATH_LEN) == 0) {
+			printf("\nnew port %s, did: %d\n",
+					ifname, vports[i].did);
+			vports[i].vid = vid;
+			break;
+		}
+	}
+
+	if (i >= MAX_VDPA_SAMPLE_PORTS)
+		return -1;
+
+	return 0;
+}
+
+static void
+destroy_device(int vid)
+{
+	char ifname[MAX_PATH_LEN];
+	int i;
+
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
+		if (strcmp(ifname, vports[i].ifname) == 0) {
+			printf("\ndestroy port %s, did: %d\n",
+					ifname, vports[i].did);
+			break;
+		}
+	}
+}
+
+static const struct vhost_device_ops vdpa_sample_devops = {
+	.new_device = new_device,
+	.destroy_device = destroy_device,
+	.vring_state_changed = NULL,
+	.features_changed = NULL,
+	.new_connection = NULL,
+	.destroy_connection = NULL,
+};
+
+static void
+start_vdpa(const char *socket_path, int did, uint64_t flags) {
+	int ret;
+	ret = rte_vhost_driver_register(socket_path, flags);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"register driver failed: %s\n",
+			socket_path);
+
+	ret = rte_vhost_driver_callback_register(socket_path,
+			&vdpa_sample_devops);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"register driver ops failed: %s\n",
+			socket_path);
+
+	ret = rte_vhost_driver_attach_vdpa_device(socket_path, did);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"attach vdpa device failed: %s\n",
+			socket_path);
+
+	if (rte_vhost_driver_start(socket_path) < 0)
+		rte_exit(EXIT_FAILURE,
+			"start vhost driver failed: %s\n",
+			socket_path);
+}
+
+/* interactive cmds */
+
+/* *** Help command with introduction. *** */ struct cmd_help_result {
+	cmdline_fixed_string_t help;
+};
+
+static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
+				struct cmdline *cl,
+				__attribute__((unused)) void *data) {
+	cmdline_printf(
+		cl,
+		"\n"
+		"The following commands are currently available:\n\n"
+		"Control:\n"
+		"    help                                      : Show interactive instructions.\n"
+		"    create <socket_file_path>                 : Create a new vdpa

port.\n\n"

examples/vdpa/main.c new file +39 -0
+	);
+}
+
+cmdline_parse_token_string_t cmd_help_help =
+	TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
+
+cmdline_parse_inst_t cmd_help = {
+	.f = cmd_help_parsed,
+	.data = NULL,
+	.help_str = "show help",
+	.tokens = {
+		(void *)&cmd_help_help,
+		NULL,
+	},
+};
+
+/* *** Create new vdpa port *** */
+struct cmd_create_result {
+	cmdline_fixed_string_t action;
+	cmdline_fixed_string_t socket_path;
+};
+
+static void cmd_create_vdpa_port_parsed(void *parsed_result,
+				struct cmdline *cl,
+				__attribute__((unused)) void *data) {
+	struct cmd_create_result *res = parsed_result;
+	char ifname[MAX_PATH_LEN];
+	uint64_t flags = 0;
+
+	strncpy(ifname, res->socket_path, MAX_PATH_LEN);
+	if (dev_id < devcnt)
+		start_vdpa(ifname, dev_id++, flags);
+	else
+		cmdline_printf(cl, "No available vdpa devices.\n"); }
+
+cmdline_parse_token_string_t cmd_action_create =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, action, "create");
+cmdline_parse_token_string_t cmd_socket_path =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, socket_path,

NULL);

examples/vdpa/main.c new file +51 -0
+
+cmdline_parse_inst_t cmd_create_vdpa_port = {
+	.f = cmd_create_vdpa_port_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "Create an new vdpa port",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_action_create,
+		(void *)&cmd_socket_path,
+		NULL,
+	},
+};
+
+cmdline_parse_ctx_t main_ctx[] = {
+	(cmdline_parse_inst_t *)&cmd_help,
+	(cmdline_parse_inst_t *)&cmd_create_vdpa_port,
+	NULL,
+};
+
+int
+main(int argc, char *argv[])
+{
+	char ifname[MAX_PATH_LEN];
+	char ch;
+	int i, did;
+	int ret;
+	uint64_t flags = 0;
+	struct cmdline *cl;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "eal init failed\n");
+	argc -= ret;
+	argv += ret;
+
+	signal(SIGINT, signal_handler);
+	signal(SIGTERM, signal_handler);
+
+	data_init();
+
+	ret = parse_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "invalid argument\n");
+
+	if (interactive == 1) {
+		cl = cmdline_stdin_new(main_ctx, "vdpa> ");
+		if (cl == NULL)
+			rte_panic("Cannot create cmdline instance\n");
+		cmdline_interact(cl);
+		cmdline_stdin_exit(cl);
+	} else {
+		for (i = 0; i <  RTE_MIN(MAX_VDPA_SAMPLE_PORTS, devcnt);

i++) {

examples/vdpa/main.c new file +22 -0
+			did = i;
+			snprintf(ifname, sizeof(ifname), "%s%d", iface, i);
+			vports[i].did = did;
+			strncpy(vports[i].ifname, ifname, MAX_PATH_LEN);
+
+			start_vdpa(ifname, did, flags);
+
+			did++;
+		}
+
+		printf("enter \'q\' to quit\n");
+		while (scanf("%c", &ch)) {
+			if (ch == 'q')
+				break;
+			while (ch != '\n')
+				scanf("%c", &ch);
+			printf("enter \'q\' to quit\n");
+		}
+	}
+
+	return 0;
+}
examples/vdpa/meson.build new file renamed from examples/vdpa/meson.build +0 -0

mode 100644 index 000000000..e85d84a76

examples/vdpa/meson.build new file +11 -1
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
+Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an
+already-installed # DPDK instance, use 'make'
+
+sources = files(
+	'main.c'
+)
--

2.18.0.rc1.1.g6f333ff2f

#5 lore raw

Re: [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

Quoted 4 lines · Haiyue

Can we add options to support 'int rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)' API calling to get the device id ? Not just hard code 0 ~ num ? Like choosing one of vDPA engines explicitly by BDF:
-w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1

Good suggestion, I'll adopt it in v2.

Thanks,
Xiaolong

Quoted 452 lines

BR,
Haiyue

Quoted 448 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Xiaolong Ye
Sent: Thursday, September 6, 2018 21:17
To: [email protected]; Maxime Coquelin <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>
Cc: Wang, Xiao W <[email protected]>; Ye, Xiaolong <[email protected]>
Subject: [dpdk-dev] [PATCH v1 2/2] examples/vdpa: add a new sample for vdpa

From: Xiao Wang <[email protected]>

This patch adds a sample which creates vhost-user socket based on vdpa driver. vdpa driver can help to set up vhost datapath so this app doesn't need to spend a dedicated worker thread on vhost enqueue/dequeue operations.

Below are setup steps for your reference (using IFC driver):

1. Make sure your kernnel vhost module and QEMU support vhost-user.
- OS: CentOS 7.4
- QEMU: 2.10.1
- Guest OS: CentOS 7.2

2. Bind VFIO-pci to VF
a) modprobe vfio-pci
d) ./usertools/dpdk-devbind.py -b vfio-pci 06:00.2 06:00.3

3. Start vdpa sample

    ./examples/vdpa/build/vdpa --log-level=9 -c 0x10 -n 4 --socket-mem
    1024,1024  -w 0000:06:00.2,vdpa=1 -w 0000:06:00.3,vdpa=1   --
    --iface  /tmp/vhost-user- (or use interactive mode to create vdpa
    ports)

4. Start two VMs ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm

\
   <snip>
   -mem-prealloc \
   -chardev socket,id=char0,path=/tmp/vhost-user-0 \
   -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
   -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ee \

   ./qemu-2.10.1/x86_64-softmmu/qemu-system-x86_64 -cpu host -enable-kvm
\
   <snip>
   -mem-prealloc \
   -chardev socket,id=char0,path=/tmp/vhost-user-1 \
   -netdev type=vhost-user,id=vdpa,chardev=char0,vhostforce \
   -device virtio-net-pci,netdev=vdpa,mac=00:aa:bb:cc:dd:ff \

5. Login two VMs and configure the ip, verify the network connection via ping.

---
 examples/vdpa/Makefile    |  32 ++++
 examples/vdpa/main.c      | 327 ++++++++++++++++++++++++++++++++++++++
 examples/vdpa/meson.build |  11 ++
 3 files changed, 370 insertions(+)
 create mode 100644 examples/vdpa/Makefile  create mode 100644

examples/vdpa/main.c create mode 100644 examples/vdpa/meson.build

examples/vdpa/Makefile new file mode renamed from examples/vdpa/Makefile +0 -0

100644 index 000000000..42672a2bc

examples/vdpa/Makefile new file +31 -0
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
+Corporation
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable") endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+$(info This application can only operate in a linuxapp environment, \
+please change the definition of the RTE_TARGET environment variable)
+all:
+else
+
+# binary name
+APP = vdpa
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -D_GNU_SOURCE
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
examples/vdpa/main.c new file mode renamed from examples/vdpa/main.c +0 -0

100644 index 000000000..b19d7a217

examples/vdpa/main.c new file +44 -0
@@ -0,0 +1,327 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <getopt.h>
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+#include <rte_vhost.h>
+#include <rte_vdpa.h>
+
+#include <cmdline_parse.h>
+#include <cmdline_socket.h>
+#include <cmdline_parse_string.h>
+#include <cmdline.h>
+
+#define MAX_PATH_LEN 128
+#define MAX_VDPA_SAMPLE_PORTS 1024
+
+struct vdpa_port {
+	char ifname[MAX_PATH_LEN];
+	int did;
+	int vid;
+};
+
+struct vdpa_port vports[MAX_VDPA_SAMPLE_PORTS];
+
+char iface[MAX_PATH_LEN];
+int devcnt;
+int dev_id;
+uint8_t interactive;
+
+/* display usage */
+static void
+vdpa_usage(const char *prgname)
+{
+	printf("Usage: %s [EAL options] -- "
+				 "[--interactive|-i] "
+				 "[--iface PATH]\n"
+				 "	--interactive: run in interactive

mode.\n"

examples/vdpa/main.c new file +1 -0
+				 "	--iface <path>: specify the path prefix

of the socket files, e.g. /tmp/vhost-user-\n",

examples/vdpa/main.c new file +165 -0
+				 prgname);
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+	static const char *short_option = "i";
+	static struct option long_option[] = {
+		{"iface", required_argument, NULL, 0},
+		{"interactive", 0, 0, 0},
+		{NULL, 0, 0, 0},
+	};
+	int opt, idx;
+	char *prgname = argv[0];
+
+	while ((opt = getopt_long(argc, argv, short_option, long_option, &idx))
+			!= EOF) {
+		switch (opt) {
+		case 'i':
+			printf("Interactive-mode selected\n");
+			interactive = 1;
+			break;
+		/* long options */
+		case 0:
+			if (strncmp(long_option[idx].name, "iface",
+						MAX_PATH_LEN) == 0) {
+				strncpy(iface, optarg, MAX_PATH_LEN);
+				printf("iface %s\n", iface);
+			}
+			if (!strcmp(long_option[idx].name, "interactive")) {
+				printf("Interactive-mode selected\n");
+				interactive = 1;
+			}
+			break;
+
+		default:
+			vdpa_usage(prgname);
+			return -1;
+		}
+	}
+
+	if (*iface == '\0' && interactive == 0) {
+		vdpa_usage(prgname);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+data_init(void)
+{
+	devcnt = rte_vdpa_get_device_num();
+	if (devcnt <= 0) {
+		printf("No available vdpa device found\n");
+		return -1;
+	}
+	dev_id = 0;
+	interactive = 0;
+	memset(iface, 0, MAX_PATH_LEN * sizeof(iface[0]));
+	memset(vports, 0, MAX_VDPA_SAMPLE_PORTS * sizeof(vports[0]));
+	return 0;
+}
+
+static void
+signal_handler(int signum)
+{
+	if (signum == SIGINT || signum == SIGTERM) {
+		printf("\nSignal %d received, preparing to exit...\n",
+				signum);
+		exit(0);
+	}
+}
+
+static int
+new_device(int vid)
+{
+	char ifname[MAX_PATH_LEN];
+	int i;
+
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
+		if (strncmp(ifname, vports[i].ifname, MAX_PATH_LEN) == 0) {
+			printf("\nnew port %s, did: %d\n",
+					ifname, vports[i].did);
+			vports[i].vid = vid;
+			break;
+		}
+	}
+
+	if (i >= MAX_VDPA_SAMPLE_PORTS)
+		return -1;
+
+	return 0;
+}
+
+static void
+destroy_device(int vid)
+{
+	char ifname[MAX_PATH_LEN];
+	int i;
+
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
+		if (strcmp(ifname, vports[i].ifname) == 0) {
+			printf("\ndestroy port %s, did: %d\n",
+					ifname, vports[i].did);
+			break;
+		}
+	}
+}
+
+static const struct vhost_device_ops vdpa_sample_devops = {
+	.new_device = new_device,
+	.destroy_device = destroy_device,
+	.vring_state_changed = NULL,
+	.features_changed = NULL,
+	.new_connection = NULL,
+	.destroy_connection = NULL,
+};
+
+static void
+start_vdpa(const char *socket_path, int did, uint64_t flags) {
+	int ret;
+	ret = rte_vhost_driver_register(socket_path, flags);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"register driver failed: %s\n",
+			socket_path);
+
+	ret = rte_vhost_driver_callback_register(socket_path,
+			&vdpa_sample_devops);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"register driver ops failed: %s\n",
+			socket_path);
+
+	ret = rte_vhost_driver_attach_vdpa_device(socket_path, did);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"attach vdpa device failed: %s\n",
+			socket_path);
+
+	if (rte_vhost_driver_start(socket_path) < 0)
+		rte_exit(EXIT_FAILURE,
+			"start vhost driver failed: %s\n",
+			socket_path);
+}
+
+/* interactive cmds */
+
+/* *** Help command with introduction. *** */ struct cmd_help_result {
+	cmdline_fixed_string_t help;
+};
+
+static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
+				struct cmdline *cl,
+				__attribute__((unused)) void *data) {
+	cmdline_printf(
+		cl,
+		"\n"
+		"The following commands are currently available:\n\n"
+		"Control:\n"
+		"    help                                      : Show interactive instructions.\n"
+		"    create <socket_file_path>                 : Create a new vdpa

port.\n\n"

examples/vdpa/main.c new file +39 -0
+	);
+}
+
+cmdline_parse_token_string_t cmd_help_help =
+	TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
+
+cmdline_parse_inst_t cmd_help = {
+	.f = cmd_help_parsed,
+	.data = NULL,
+	.help_str = "show help",
+	.tokens = {
+		(void *)&cmd_help_help,
+		NULL,
+	},
+};
+
+/* *** Create new vdpa port *** */
+struct cmd_create_result {
+	cmdline_fixed_string_t action;
+	cmdline_fixed_string_t socket_path;
+};
+
+static void cmd_create_vdpa_port_parsed(void *parsed_result,
+				struct cmdline *cl,
+				__attribute__((unused)) void *data) {
+	struct cmd_create_result *res = parsed_result;
+	char ifname[MAX_PATH_LEN];
+	uint64_t flags = 0;
+
+	strncpy(ifname, res->socket_path, MAX_PATH_LEN);
+	if (dev_id < devcnt)
+		start_vdpa(ifname, dev_id++, flags);
+	else
+		cmdline_printf(cl, "No available vdpa devices.\n"); }
+
+cmdline_parse_token_string_t cmd_action_create =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, action, "create");
+cmdline_parse_token_string_t cmd_socket_path =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, socket_path,

NULL);

examples/vdpa/main.c new file +51 -0
+
+cmdline_parse_inst_t cmd_create_vdpa_port = {
+	.f = cmd_create_vdpa_port_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "Create an new vdpa port",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_action_create,
+		(void *)&cmd_socket_path,
+		NULL,
+	},
+};
+
+cmdline_parse_ctx_t main_ctx[] = {
+	(cmdline_parse_inst_t *)&cmd_help,
+	(cmdline_parse_inst_t *)&cmd_create_vdpa_port,
+	NULL,
+};
+
+int
+main(int argc, char *argv[])
+{
+	char ifname[MAX_PATH_LEN];
+	char ch;
+	int i, did;
+	int ret;
+	uint64_t flags = 0;
+	struct cmdline *cl;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "eal init failed\n");
+	argc -= ret;
+	argv += ret;
+
+	signal(SIGINT, signal_handler);
+	signal(SIGTERM, signal_handler);
+
+	data_init();
+
+	ret = parse_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "invalid argument\n");
+
+	if (interactive == 1) {
+		cl = cmdline_stdin_new(main_ctx, "vdpa> ");
+		if (cl == NULL)
+			rte_panic("Cannot create cmdline instance\n");
+		cmdline_interact(cl);
+		cmdline_stdin_exit(cl);
+	} else {
+		for (i = 0; i <  RTE_MIN(MAX_VDPA_SAMPLE_PORTS, devcnt);

i++) {

examples/vdpa/main.c new file +22 -0
+			did = i;
+			snprintf(ifname, sizeof(ifname), "%s%d", iface, i);
+			vports[i].did = did;
+			strncpy(vports[i].ifname, ifname, MAX_PATH_LEN);
+
+			start_vdpa(ifname, did, flags);
+
+			did++;
+		}
+
+		printf("enter \'q\' to quit\n");
+		while (scanf("%c", &ch)) {
+			if (ch == 'q')
+				break;
+			while (ch != '\n')
+				scanf("%c", &ch);
+			printf("enter \'q\' to quit\n");
+		}
+	}
+
+	return 0;
+}
examples/vdpa/meson.build new file renamed from examples/vdpa/meson.build +0 -0

mode 100644 index 000000000..e85d84a76

examples/vdpa/meson.build new file +11 -1
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Intel
+Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an
+already-installed # DPDK instance, use 'make'
+
+sources = files(
+	'main.c'
+)
--

2.18.0.rc1.1.g6f333ff2f