Skip to contentSkip to conversation

support inflight share memory protocol feature

PATCH New JinYu 1 participant 3 messages Jul 8, 2019 Arm/Amlogic Meson SoC support

lore
Share & tools
lore

JinYu

author
#0 lore raw

cover support inflight share memory protocol feature

This patches series introduces two new messages VHOST_USER_GET_INFLIGHT_FD and VHOST_USER_SET_INFLIGHT_FD to support transferring a shared buffer between qemu and backend.

This shared buffer is used to process inflight I/O when backend reconnect.

The example show how these new APIs work.

JinYu (2):
[1/2]vhost: support inflight share memory protocol feature [2/2]vhost: Add vhost-user-blk example which support inflight

 examples/vhost_blk/Makefile            |  67 +++
 examples/vhost_blk/blk.c               | 116 +++++
 examples/vhost_blk/blk_spec.h          |  95 ++++
 examples/vhost_blk/meson.build         |  20 +
 examples/vhost_blk/vhost_blk.c         | 590 +++++++++++++++++++++++++
 examples/vhost_blk/vhost_blk.h         |  95 ++++
 examples/vhost_blk/vhost_blk_compat.c  | 196 ++++++++
 lib/librte_vhost/rte_vhost.h           | 105 +++++
 lib/librte_vhost/rte_vhost_version.map |   4 +
 lib/librte_vhost/vhost.c               | 158 ++++++-
 lib/librte_vhost/vhost.h               |  16 +
 lib/librte_vhost/vhost_user.c          | 313 +++++++++++++
 lib/librte_vhost/vhost_user.h          |  13 +-
 13 files changed, 1786 insertions(+), 2 deletions(-)
 create mode 100644 examples/vhost_blk/Makefile
 create mode 100644 examples/vhost_blk/blk.c
 create mode 100644 examples/vhost_blk/blk_spec.h
 create mode 100644 examples/vhost_blk/meson.build
 create mode 100644 examples/vhost_blk/vhost_blk.c
 create mode 100644 examples/vhost_blk/vhost_blk.h
 create mode 100644 examples/vhost_blk/vhost_blk_compat.c
Signature · 1 line
2.17.2
author
#1 lore raw

PATCH 1/2 vhost: support inflight share memory protocol feature

This patch introduces two new messages VHOST_USER_GET_INFLIGHT_FD and VHOST_USER_SET_INFLIGHT_FD to support transferring a shared buffer between qemu and backend.

Firstly, qemu uses VHOST_USER_GET_INFLIGHT_FD to get the shared buffer from backend. Then qemu should send it back through VHOST_USER_SET_INFLIGHT_FD each time we start vhost-user.

This shared buffer is used to process inflight I/O when backend reconnect.

---
v1 - specify the APIs are split-ring only
v2 - fix APIs and judge split or packed

---
 lib/librte_vhost/rte_vhost.h           | 105 +++++++++
 lib/librte_vhost/rte_vhost_version.map |   4 +
 lib/librte_vhost/vhost.c               | 158 ++++++++++++-
 lib/librte_vhost/vhost.h               |  16 ++
 lib/librte_vhost/vhost_user.c          | 313 +++++++++++++++++++++++++
 lib/librte_vhost/vhost_user.h          |  13 +-
 6 files changed, 607 insertions(+), 2 deletions(-)
lib/librte_vhost/rte_vhost.h +105 -0
index 0226b3eff..a17aff876 100644
@@ -71,6 +71,10 @@ extern "C" {
 #define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11
 #endif
 
+#ifndef VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
+#define VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD 12
+#endif
+
 /** Indicate whether protocol features negotiation is supported. */
 #ifndef VHOST_USER_F_PROTOCOL_FEATURES
 #define VHOST_USER_F_PROTOCOL_FEATURES	30
@@ -98,6 +102,41 @@ struct rte_vhost_memory {
 	struct rte_vhost_mem_region regions[];
 };
 
+struct inflight_desc_split {
+	uint8_t		inflight;
+	uint8_t		padding[5];
+	uint16_t	next;
+	uint64_t	counter;
+};
+
+struct inflight_info_split {
+	uint64_t		features;
+	uint16_t		version;
+	uint16_t		desc_num;
+	uint16_t		last_inflight_io;
+	uint16_t		used_idx;
+	struct inflight_desc_split desc[0];
+};
+
+struct resubmit_desc {
+	uint16_t index;
+	uint64_t counter;
+};
+
+struct resubmit_info {
+	struct resubmit_desc	*resubmit_list;
+	uint16_t		resubmit_num;
+};
+
+struct rte_vhost_ring_inflight_split {
+	union {
+		struct inflight_info_split *inflight_split;
+		/* TODO */
+	};
+
+	struct resubmit_info *resubmit_inflight_split;
+};
+
 struct rte_vhost_vring {
 	struct vring_desc	*desc;
 	struct vring_avail	*avail;
@@ -603,6 +642,22 @@ uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
  */
 int rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem);
 
+/**
+ * Get guest inflight vring info, including inflight ring and resubmit list.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param vring
+ *  the structure to hold the requested inflight vring info
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_get_vhost_ring_inflight_split(int vid, uint16_t vring_idx,
+			      struct rte_vhost_ring_inflight_split *vring);
+
 /**
  * Get guest vring info, including the vring address, vring size, etc.
  *
@@ -631,6 +686,56 @@ int rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
  */
 int rte_vhost_vring_call(int vid, uint16_t vring_idx);
 
+/**
+ * set inflight flag for a desc.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx,
+		uint16_t idx);
+
+/**
+ * clear inflight flag for a desc.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param last_used_idx
+ *  next free used_idx
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
+		uint16_t last_used_idx, uint16_t idx);
+
+/**
+ * set last inflight io index.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_set_last_inflight_io_split(int vid, uint16_t vring_idx,
+		uint16_t idx);
+
 /**
  * Get vhost RX queue avail count.
  *
lib/librte_vhost/rte_vhost_version.map +4 -0
index 5f1d4a75c..709593701 100644
@@ -87,4 +87,8 @@ EXPERIMENTAL {
 	rte_vdpa_relay_vring_used;
 	rte_vhost_extern_callback_register;
 	rte_vhost_driver_set_protocol_features;
+	rte_vhost_set_inflight_desc_split;
+	rte_vhost_clr_inflight_desc_split;
+	rte_vhost_set_last_inflight_io_split;
+	rte_vhost_get_vhost_vring_with_inflight_split;
 };
lib/librte_vhost/vhost.c +157 -1

183 lines not on this page · raw message

lib/librte_vhost/vhost.h +16 -0
index 884befa85..22925586d 100644
@@ -128,6 +128,14 @@ struct vhost_virtqueue {
 	/* Physical address of used ring, for logging */
 	uint64_t		log_guest_addr;
 
+	/* inflight share memory info */
+	union {
+		struct inflight_info_split *inflight_split;
+		/* TODO */
+	};
+	struct resubmit_info *resubmit_inflight_split;
+	uint64_t		global_counter;
+
 	uint16_t		nr_zmbuf;
 	uint16_t		zmbuf_size;
 	uint16_t		last_zmbuf_idx;
@@ -286,6 +294,12 @@ struct guest_page {
 	uint64_t size;
 };
 
+struct inflight_mem_info {
+	int		fd;
+	void		*addr;
+	uint64_t	size;
+};
+
 /**
  * Device structure contains all configuration information relating
  * to the device.
@@ -303,6 +317,7 @@ struct virtio_net {
 	uint32_t		nr_vring;
 	int			dequeue_zero_copy;
 	struct vhost_virtqueue	*virtqueue[VHOST_MAX_QUEUE_PAIRS * 2];
+	struct inflight_mem_info inflight_info;
 #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
 	char			ifname[IF_NAME_SZ];
 	uint64_t		log_size;
@@ -467,6 +482,7 @@ void vhost_destroy_device(int);
 void vhost_destroy_device_notify(struct virtio_net *dev);
 
 void cleanup_vq(struct vhost_virtqueue *vq, int destroy);
+void cleanup_vq_inflight_split(struct virtio_net *dev, struct vhost_virtqueue *vq);
 void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq);
 
 int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx);
lib/librte_vhost/vhost_user.c +313 -0

367 lines not on this page · raw message

lib/librte_vhost/vhost_user.h +12 -1
index 2a650fe4b..99a773910 100644
@@ -54,7 +54,9 @@ typedef enum VhostUserRequest {
 	VHOST_USER_POSTCOPY_ADVISE = 28,
 	VHOST_USER_POSTCOPY_LISTEN = 29,
 	VHOST_USER_POSTCOPY_END = 30,
-	VHOST_USER_MAX = 31
+	VHOST_USER_GET_INFLIGHT_FD = 31,
+	VHOST_USER_SET_INFLIGHT_FD = 32,
+	VHOST_USER_MAX = 33
 } VhostUserRequest;
 
 typedef enum VhostUserSlaveRequest {
@@ -112,6 +114,13 @@ typedef struct VhostUserVringArea {
 	uint64_t offset;
 } VhostUserVringArea;
 
+typedef struct VhostUserInflight {
+	uint64_t mmap_size;
+	uint64_t mmap_offset;
+	uint16_t num_queues;
+	uint16_t queue_size;
+} VhostUserInflight;
+
 typedef struct VhostUserMsg {
 	union {
 		uint32_t master; /* a VhostUserRequest value */
@@ -131,6 +140,7 @@ typedef struct VhostUserMsg {
 		struct vhost_vring_addr addr;
 		VhostUserMemory memory;
 		VhostUserLog    log;
+		VhostUserInflight inflight;
 		struct vhost_iotlb_msg iotlb;
 		VhostUserCryptoSessionParam crypto_session;
 		VhostUserVringArea area;
@@ -148,6 +158,7 @@ typedef struct VhostUserMsg {
 /* vhost_user.c */
 int vhost_user_msg_handler(int vid, int fd);
 int vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm);
+void *inflight_mem_alloc(const char *name, size_t size, int *fd);
 
 /* socket.c */
 int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int max_fds,
Signature · 1 line
2.17.2
author
#2 lore raw

PATCH 2/2 vhost: Add vhost-user-blk example which support inflight

In reply to JinYu

A vhost-user-blk example that support inflight feature. It uses the new APIs that introduced in the first patch, so It can show how there APIs work to support inflight feature.

---
V1 - add the case.

---
 examples/vhost_blk/Makefile           |  67 +++
 examples/vhost_blk/blk.c              | 116 +++++
 examples/vhost_blk/blk_spec.h         |  95 +++++
 examples/vhost_blk/meson.build        |  20 +
 examples/vhost_blk/vhost_blk.c        | 590 ++++++++++++++++++++++++++
 examples/vhost_blk/vhost_blk.h        |  95 +++++
 examples/vhost_blk/vhost_blk_compat.c | 196 +++++++++
 7 files changed, 1179 insertions(+)
 create mode 100644 examples/vhost_blk/Makefile
 create mode 100644 examples/vhost_blk/blk.c
 create mode 100644 examples/vhost_blk/blk_spec.h
 create mode 100644 examples/vhost_blk/meson.build
 create mode 100644 examples/vhost_blk/vhost_blk.c
 create mode 100644 examples/vhost_blk/vhost_blk.h
 create mode 100644 examples/vhost_blk/vhost_blk_compat.c
examples/vhost_blk/Makefile new file +67 -0
new file mode 100644
index 000000000..52e9befd8
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2017 Intel Corporation
+
+# binary name
+APP = vhost-blk
+
+# all source are stored in SRCS-y
+SRCS-y := blk.c vhost_blk.c vhost_blk_compat.c
+
+# Build using pkg-config variables if possible
+$(shell pkg-config --exists libdpdk)
+ifeq ($(.SHELLSTATUS),0)
+
+all: shared
+.PHONY: shared static
+shared: build/$(APP)-shared
+	ln -sf $(APP)-shared build/$(APP)
+static: build/$(APP)-static
+	ln -sf $(APP)-static build/$(APP)
+
+CFLAGS += -D_FILE_OFFSET_BITS=64
+LDFLAGS += -pthread
+
+PC_FILE := $(shell pkg-config --path libdpdk)
+CFLAGS += -O3 $(shell pkg-config --cflags libdpdk)
+LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk)
+LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk)
+
+build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
+
+build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
+
+build:
+	@mkdir -p $@
+
+.PHONY: clean
+clean:
+	rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
+	test -d build && rmdir -p build || true
+
+else # Build using legacy build system
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, detect a build directory, by looking for a path with a .config
+RTE_TARGET ?= $(notdir $(abspath $(dir $(firstword $(wildcard $(RTE_SDK)/*/.config)))))
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(CONFIG_RTE_EXEC_ENV_LINUX),y)
+$(info This application can only operate in a linux environment, \
+please change the definition of the RTE_TARGET environment variable)
+all:
+else
+
+CFLAGS += -D_FILE_OFFSET_BITS=64
+CFLAGS += -O2
+#CFLAGS += $(WERROR_FLAGS)
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
+endif
examples/vhost_blk/blk.c new file +116 -0

116 lines not on this page · raw message

examples/vhost_blk/blk_spec.h new file +95 -0

95 lines not on this page · raw message

examples/vhost_blk/meson.build new file +20 -0
new file mode 100644
index 000000000..028aa4f62
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2017 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+if not is_linux
+	build = false
+endif
+
+if not cc.has_header('linux/virtio_blk.h')
+	build = false
+endif
+
+deps += 'vhost'
+sources = files(
+	'blk.c', 'vhost_blk.c', 'vhost_blk_compat.c'
+)
examples/vhost_blk/vhost_blk.c new file +590 -0

590 lines not on this page · raw message

examples/vhost_blk/vhost_blk.h new file +95 -0

95 lines not on this page · raw message

examples/vhost_blk/vhost_blk_compat.c new file +196 -0

196 lines not on this page · raw message

Signature · 1 line
2.17.2