support inflight share memory protocol feature
cover support inflight share memory protocol feature
This patches series introduces two new messages VHOST_USER_GET_INFLIGHT_FD and VHOST_USER_SET_INFLIGHT_FD to support transferring a shared buffer between qemu and backend.
This shared buffer is used to process inflight I/O when backend reconnect.
The example show how these new APIs work.
JinYu (2):
[1/2]vhost: support inflight share memory protocol feature [2/2]vhost: Add vhost-user-blk example which support inflight
examples/vhost_blk/Makefile | 67 +++ examples/vhost_blk/blk.c | 116 +++++ examples/vhost_blk/blk_spec.h | 95 ++++ examples/vhost_blk/meson.build | 20 + examples/vhost_blk/vhost_blk.c | 590 +++++++++++++++++++++++++ examples/vhost_blk/vhost_blk.h | 95 ++++ examples/vhost_blk/vhost_blk_compat.c | 196 ++++++++ lib/librte_vhost/rte_vhost.h | 105 +++++ lib/librte_vhost/rte_vhost_version.map | 4 + lib/librte_vhost/vhost.c | 158 ++++++- lib/librte_vhost/vhost.h | 16 + lib/librte_vhost/vhost_user.c | 313 +++++++++++++ lib/librte_vhost/vhost_user.h | 13 +- 13 files changed, 1786 insertions(+), 2 deletions(-) create mode 100644 examples/vhost_blk/Makefile create mode 100644 examples/vhost_blk/blk.c create mode 100644 examples/vhost_blk/blk_spec.h create mode 100644 examples/vhost_blk/meson.build create mode 100644 examples/vhost_blk/vhost_blk.c create mode 100644 examples/vhost_blk/vhost_blk.h create mode 100644 examples/vhost_blk/vhost_blk_compat.c
Signature · 1 line
2.17.2
PATCH 1/2 vhost: support inflight share memory protocol feature
This patch introduces two new messages VHOST_USER_GET_INFLIGHT_FD and VHOST_USER_SET_INFLIGHT_FD to support transferring a shared buffer between qemu and backend.
Firstly, qemu uses VHOST_USER_GET_INFLIGHT_FD to get the shared buffer from backend. Then qemu should send it back through VHOST_USER_SET_INFLIGHT_FD each time we start vhost-user.
This shared buffer is used to process inflight I/O when backend reconnect.
---
v1 - specify the APIs are split-ring only
v2 - fix APIs and judge split or packed
--- lib/librte_vhost/rte_vhost.h | 105 +++++++++ lib/librte_vhost/rte_vhost_version.map | 4 + lib/librte_vhost/vhost.c | 158 ++++++++++++- lib/librte_vhost/vhost.h | 16 ++ lib/librte_vhost/vhost_user.c | 313 +++++++++++++++++++++++++ lib/librte_vhost/vhost_user.h | 13 +- 6 files changed, 607 insertions(+), 2 deletions(-)
lib/librte_vhost/rte_vhost.h +105 -0
@@ -71,6 +71,10 @@ extern "C" {#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11#endif +#ifndef VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD +#define VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD 12 +#endif +/** Indicate whether protocol features negotiation is supported. */ #ifndef VHOST_USER_F_PROTOCOL_FEATURES #define VHOST_USER_F_PROTOCOL_FEATURES 30
@@ -98,6 +102,41 @@ struct rte_vhost_memory {struct rte_vhost_mem_region regions[]; }; +struct inflight_desc_split { +uint8_t inflight; +uint8_t padding[5 ]; +uint16_t next; +uint64_t counter; +}; + +struct inflight_info_split { +uint64_t features; +uint16_t version; +uint16_t desc_num; +uint16_t last_inflight_io; +uint16_t used_idx; +struct inflight_desc_split desc[0 ]; +}; + +struct resubmit_desc { +uint16_t index; +uint64_t counter; +}; + +struct resubmit_info { +struct resubmit_desc *resubmit_list; +uint16_t resubmit_num; +}; + +struct rte_vhost_ring_inflight_split { +union { +struct inflight_info_split *inflight_split; +/* TODO */ + }; + +struct resubmit_info *resubmit_inflight_split; +}; +struct rte_vhost_vring {struct vring_desc *desc;struct vring_avail *avail;
@@ -603,6 +642,22 @@ uint16_t rte_vhost_dequeue_burst(int vid, uint16_t queue_id,*/ int rte_vhost_get_mem_table (int vid,struct rte_vhost_memory **mem); +/** +* Get guest inflight vring info, including inflight ring and resubmit list. +* +* @param vid +* vhost device ID +* @param vring_idx +* vring index +* @param vring +* the structure to hold the requested inflight vring info +* @return +* 0 on success, -1 on failure +*/ +int __rte_experimental +rte_vhost_get_vhost_ring_inflight_split (int vid,uint16_t vring_idx, +struct rte_vhost_ring_inflight_split *vring); +/** * Get guest vring info, including the vring address, vring size, etc. *
@@ -631,6 +686,56 @@ int rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,*/ int rte_vhost_vring_call (int vid,uint16_t vring_idx); +/** +* set inflight flag for a desc. +* +* @param vid +* vhost device ID +* @param vring_idx +* vring index +* @param idx +* inflight entry index +* @return +* 0 on success, -1 on failure +*/ +int __rte_experimental +rte_vhost_set_inflight_desc_split (int vid,uint16_t vring_idx, +uint16_t idx); + +/** +* clear inflight flag for a desc. +* +* @param vid +* vhost device ID +* @param vring_idx +* vring index +* @param last_used_idx +* next free used_idx +* @param idx +* inflight entry index +* @return +* 0 on success, -1 on failure +*/ +int __rte_experimental +rte_vhost_clr_inflight_desc_split (int vid,uint16_t vring_idx, +uint16_t last_used_idx,uint16_t idx); + +/** +* set last inflight io index. +* +* @param vid +* vhost device ID +* @param vring_idx +* vring index +* @param idx +* inflight entry index +* @return +* 0 on success, -1 on failure +*/ +int __rte_experimental +rte_vhost_set_last_inflight_io_split (int vid,uint16_t vring_idx, +uint16_t idx); +/** * Get vhost RX queue avail count. *
lib/librte_vhost/rte_vhost_version.map +4 -0
@@ -87,4 +87,8 @@ EXPERIMENTAL { rte_vdpa_relay_vring_used; rte_vhost_extern_callback_register; rte_vhost_driver_set_protocol_features; + rte_vhost_set_inflight_desc_split; + rte_vhost_clr_inflight_desc_split; + rte_vhost_set_last_inflight_io_split; + rte_vhost_get_vhost_vring_with_inflight_split; };
lib/librte_vhost/vhost.c +157 -1
183 lines not on this page · raw message
lib/librte_vhost/vhost.h +16 -0
@@ -128,6 +128,14 @@ struct vhost_virtqueue {/* Physical address of used ring, for logging */ uint64_t log_guest_addr; +/* inflight share memory info */ +union { +struct inflight_info_split *inflight_split; +/* TODO */ + }; +struct resubmit_info *resubmit_inflight_split; +uint64_t global_counter; +uint16_t nr_zmbuf;uint16_t zmbuf_size;uint16_t last_zmbuf_idx;
@@ -286,6 +294,12 @@ struct guest_page {uint64_t size; }; +struct inflight_mem_info { +int fd; +void *addr; +uint64_t size; +}; +/** * Device structure contains all configuration information relating * to the device.
@@ -303,6 +317,7 @@ struct virtio_net {uint32_t nr_vring;int dequeue_zero_copy;struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS *2 ]; +struct inflight_mem_info inflight_info;#define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)char ifname[IF_NAME_SZ ];uint64_t log_size;
@@ -467,6 +482,7 @@ void vhost_destroy_device(int);void vhost_destroy_device_notify (struct virtio_net *dev);void cleanup_vq (struct vhost_virtqueue *vq,int destroy); +void cleanup_vq_inflight_split (struct virtio_net *dev,struct vhost_virtqueue *vq);void free_vq (struct virtio_net *dev,struct vhost_virtqueue *vq);int alloc_vring_queue (struct virtio_net *dev,uint32_t vring_idx);
lib/librte_vhost/vhost_user.c +313 -0
367 lines not on this page · raw message
lib/librte_vhost/vhost_user.h +12 -1
@@ -54,7 +54,9 @@ typedef enum VhostUserRequest {VHOST_USER_POSTCOPY_ADVISE =28 ,VHOST_USER_POSTCOPY_LISTEN =29 ,VHOST_USER_POSTCOPY_END =30 , - VHOST_USER_MAX = 31 +VHOST_USER_GET_INFLIGHT_FD =31 , +VHOST_USER_SET_INFLIGHT_FD =32 , + VHOST_USER_MAX = 33 } VhostUserRequest; typedef enum VhostUserSlaveRequest {
@@ -112,6 +114,13 @@ typedef struct VhostUserVringArea {uint64_t offset; } VhostUserVringArea; +typedef struct VhostUserInflight { +uint64_t mmap_size; +uint64_t mmap_offset; +uint16_t num_queues; +uint16_t queue_size; +}VhostUserInflight ; + typedef struct VhostUserMsg { union { uint32_t master; /* a VhostUserRequest value */
@@ -131,6 +140,7 @@ typedef struct VhostUserMsg {struct vhost_vring_addr addr;VhostUserMemory memory;VhostUserLog log; +VhostUserInflight inflight;struct vhost_iotlb_msg iotlb;VhostUserCryptoSessionParam crypto_session;VhostUserVringArea area;
@@ -148,6 +158,7 @@ typedef struct VhostUserMsg {/* vhost_user.c */ int vhost_user_msg_handler (int vid,int fd);int vhost_user_iotlb_miss (struct virtio_net *dev,uint64_t iova,uint8_t perm); +void *inflight_mem_alloc (const char *name,size_t size,int *fd);/* socket.c */ int read_fd_message(int sockfd, char *buf, int buflen, int *fds, int max_fds,
Signature · 1 line
2.17.2
PATCH 2/2 vhost: Add vhost-user-blk example which support inflight
In reply to JinYuA vhost-user-blk example that support inflight feature. It uses the new APIs that introduced in the first patch, so It can show how there APIs work to support inflight feature.
---
V1 - add the case.
--- examples/vhost_blk/Makefile | 67 +++ examples/vhost_blk/blk.c | 116 +++++ examples/vhost_blk/blk_spec.h | 95 +++++ examples/vhost_blk/meson.build | 20 + examples/vhost_blk/vhost_blk.c | 590 ++++++++++++++++++++++++++ examples/vhost_blk/vhost_blk.h | 95 +++++ examples/vhost_blk/vhost_blk_compat.c | 196 +++++++++ 7 files changed, 1179 insertions(+) create mode 100644 examples/vhost_blk/Makefile create mode 100644 examples/vhost_blk/blk.c create mode 100644 examples/vhost_blk/blk_spec.h create mode 100644 examples/vhost_blk/meson.build create mode 100644 examples/vhost_blk/vhost_blk.c create mode 100644 examples/vhost_blk/vhost_blk.h create mode 100644 examples/vhost_blk/vhost_blk_compat.c
examples/vhost_blk/Makefile new file +67 -0
@@ -0,0 +1,67 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2017 Intel Corporation + +# binary name +APP = vhost-blk + +# all source are stored in SRCS-y +SRCS-y := blk.c vhost_blk.c vhost_blk_compat.c + +# Build using pkg-config variables if possible +$(shell pkg-config --exists libdpdk) +ifeq ($(.SHELLSTATUS),0) + +all: shared +.PHONY: shared static +shared: build/$(APP)-shared + ln -sf $(APP)-shared build/$(APP) +static: build/$(APP)-static + ln -sf $(APP)-static build/$(APP) + +CFLAGS += -D_FILE_OFFSET_BITS=64 +LDFLAGS += -pthread + +PC_FILE := $(shell pkg-config --path libdpdk) +CFLAGS += -O3 $(shell pkg-config --cflags libdpdk) +LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk) +LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk) + +build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build + $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED) + +build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build + $(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC) + +build: + @mkdir -p $@ + +.PHONY: clean +clean: + rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared + test -d build && rmdir -p build || true + +else # Build using legacy build system + +ifeq ($(RTE_SDK),) +$(error "Please define RTE_SDK environment variable") +endif + +# Default target, detect a build directory, by looking for a path with a .config +RTE_TARGET ?= $(notdir $(abspath $(dir $(firstword $(wildcard $(RTE_SDK)/*/.config))))) + +include $(RTE_SDK)/mk/rte.vars.mk + +ifneq ($(CONFIG_RTE_EXEC_ENV_LINUX),y) +$(info This application can only operate in a linux environment, \ +please change the definition of the RTE_TARGET environment variable) +all: +else + +CFLAGS += -D_FILE_OFFSET_BITS=64 +CFLAGS += -O2 +#CFLAGS += $(WERROR_FLAGS) + +include $(RTE_SDK)/mk/rte.extapp.mk + +endif +endif
examples/vhost_blk/blk.c new file +116 -0
116 lines not on this page · raw message
examples/vhost_blk/blk_spec.h new file +95 -0
95 lines not on this page · raw message
examples/vhost_blk/meson.build new file +20 -0
@@ -0,0 +1,20 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +# meson file, for building this example as part of a main DPDK build. +# +# To build this example as a standalone application with an already-installed +# DPDK instance, use 'make' + +if not is_linux + build = false +endif + +if not cc.has_header('linux/virtio_blk.h') + build = false +endif + +deps += 'vhost' +sources = files( + 'blk.c', 'vhost_blk.c', 'vhost_blk_compat.c' +)
examples/vhost_blk/vhost_blk.c new file +590 -0
590 lines not on this page · raw message
examples/vhost_blk/vhost_blk.h new file +95 -0
95 lines not on this page · raw message
examples/vhost_blk/vhost_blk_compat.c new file +196 -0
196 lines not on this page · raw message
Signature · 1 line
2.17.2