Initial support for multi-process Qemu
Subsystems (2)
PATCH v19 3/20 memory: alloc RAM from file at offset
In reply to Jagannathan RamanAllow RAM MemoryRegion to be created from an offset in a file, instead of allocating at offset of 0 by default. This is needed to synchronize RAM between QEMU & remote process.
--- include/exec/memory.h | 2 ++ include/exec/ram_addr.h | 2 +- include/qemu/mmap-alloc.h | 4 +++- backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 ++- softmmu/memory.c | 3 ++- softmmu/physmem.c | 11 +++++++---- util/mmap-alloc.c | 7 ++++--- util/oslib-posix.c | 2 +- 9 files changed, 23 insertions(+), 13 deletions(-)
include/exec/memory.h +2 -0
@@ -990,6 +990,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr,* @size: size of the region. * @share: %true if memory must be mmaped with the MAP_SHARED flag * @fd: the fd to mmap. +* @offset: offset within the file referenced by fd * @errp: pointer to Error*, to store an error if it happens. * * Note that this function does not do anything to cause the data in the
@@ -1001,6 +1002,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp);#endif
include/exec/ram_addr.h +1 -1
@@ -121,7 +121,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, Error **errp);RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp); +off_t offset,Error **errp);RAMBlock *qemu_ram_alloc_from_ptr (ram_addr_t size,void *host,MemoryRegion *mr,Error **errp);
include/qemu/mmap-alloc.h +3 -1
@@ -16,6 +16,7 @@ size_t qemu_mempath_getpagesize(const char *mem_path);* otherwise, the alignment in use will be determined by QEMU. * @shared: map has RAM_SHARED flag. * @is_pmem: map has RAM_PMEM flag. +* @map_offset: map starts at offset of map_offset from the start of fd * * Return: * On success, return a pointer to the mapped area.
@@ -25,7 +26,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem); + bool is_pmem, + off_t map_offset);void qemu_ram_munmap (int fd,void *ptr,size_t size);
backends/hostmem-memfd.c +1 -1
@@ -55,7 +55,7 @@ memfd_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) name =host_memory_backend_get_name (backend);memory_region_init_ram_from_fd (&backend->mr,OBJECT (backend), name, backend->size, - backend->share, fd, errp); + backend->share, fd,0 , errp);g_free (name); }
hw/misc/ivshmem.c +2 -1
@@ -495,7 +495,8 @@ static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)/* mmap the region and map into the BAR2 */ memory_region_init_ram_from_fd (&s->server_bar2,OBJECT (s), -"ivshmem.bar2" , size, true, fd, &local_err); +"ivshmem.bar2" , size, true, fd,0 , + &local_err);if (local_err) {error_propagate (errp, local_err);return ;
softmmu/memory.c +2 -1
@@ -1609,6 +1609,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp) { Error *err = NULL;
@@ -1618,7 +1619,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->ram_block =qemu_ram_alloc_from_fd (size, mr, share ?RAM_SHARED :0 , - fd, &err); + fd, offset, &err);if (err) { mr->size =int128_zero ();object_unparent (OBJECT (mr));
softmmu/physmem.c +7 -4
@@ -1461,6 +1461,7 @@ static void *file_ram_alloc(RAMBlock *block,ram_addr_t memory, int fd, bool truncate, + off_t offset, Error **errp) { void *area;
@@ -1511,7 +1512,8 @@ static void *file_ram_alloc(RAMBlock *block, } area =qemu_ram_mmap (fd, memory, block->mr->align, - block->flags &RAM_SHARED , block->flags &RAM_PMEM ); + block->flags &RAM_SHARED , block->flags &RAM_PMEM , + offset);if (area ==MAP_FAILED ) {error_setg_errno (errp, errno,"unable to map backing store for guest RAM" );
@@ -1943,7 +1945,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)#ifdef CONFIG_POSIX RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp) +off_t offset,Error **errp) {RAMBlock *new_block;Error *local_err =NULL ;
@@ -1996,7 +1998,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, new_block->used_length = size; new_block->max_length = size; new_block->flags = ram_flags; - new_block->host =file_ram_alloc (new_block, size, fd, !file_size, errp); + new_block->host =file_ram_alloc (new_block, size, fd, !file_size, offset, + errp);if (!new_block->host) {g_free (new_block);return NULL ;
@@ -2026,7 +2029,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,return NULL ; } - block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd, errp); + block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd,0 , errp);if (!block) {if (created) {unlink (mem_path);
util/mmap-alloc.c +4 -3
@@ -86,7 +86,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem) + bool is_pmem, + off_t map_offset) { int flags;int map_sync_flags =0 ;
@@ -147,7 +148,7 @@ void *qemu_ram_mmap(int fd, offset =QEMU_ALIGN_UP ((uintptr_t )guardptr, align) - (uintptr_t )guardptr; ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags | map_sync_flags, fd,0 ); + flags | map_sync_flags, fd, map_offset);if (ptr ==MAP_FAILED && map_sync_flags) {if (errno ==ENOTSUP ) {
@@ -172,7 +173,7 @@ void *qemu_ram_mmap(int fd,* we will remove these flags to handle compatibility. */ ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags, fd,0 ); + flags, fd, map_offset); }if (ptr ==MAP_FAILED ) {
util/oslib-posix.c +1 -1
@@ -230,7 +230,7 @@ void *qemu_memalign(size_t alignment, size_t size)void *qemu_anon_ram_alloc (size_t size,uint64_t *alignment,bool shared) {size_t align =QEMU_VMALLOC_ALIGN ; -void *ptr =qemu_ram_mmap (-1 , size, align, shared, false); +void *ptr =qemu_ram_mmap (-1 , size, align, shared, false,0 );if (ptr ==MAP_FAILED ) {return NULL ;
Signature · 1 line
1.8.3.1
PATCH v19 9/20 multi-process: define MPQemuMsg format and transmission functions
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks. Define transmission functions used by proxy and by remote.
--- meson.build | 1 + hw/remote/trace.h | 1 + include/hw/remote/mpqemu-link.h | 63 ++++++++++++ include/sysemu/iothread.h | 6 ++ hw/remote/mpqemu-link.c | 205 ++++++++++++++++++++++++++++++++++++++++ iothread.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + hw/remote/trace-events | 4 + 9 files changed, 289 insertions(+) create mode 100644 hw/remote/trace.h create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.c create mode 100644 hw/remote/trace-events
meson.build +1 -0
@@ -1754,6 +1754,7 @@ if have_system 'net', 'softmmu', 'ui', + 'hw/remote', ] endif trace_events_subdirs += [
hw/remote/trace.h new file +1 -0
@@ -0,0 +1 @@ +#include "trace/trace-hw_remote.h"
include/hw/remote/mpqemu-link.h new file +63 -0
63 lines not on this page · raw message
include/sysemu/iothread.h +6 -0
@@ -57,4 +57,10 @@ IOThread *iothread_create(const char *id, Error **errp);void iothread_stop (IOThread *iothread);void iothread_destroy (IOThread *iothread); +/* +* Returns true if executing withing IOThread context, +* false otherwise. +*/ +bool qemu_in_iothread (void ); +#endif /* IOTHREAD_H */
hw/remote/mpqemu-link.c new file +205 -0
205 lines not on this page · raw message
iothread.c +6 -0
@@ -375,3 +375,9 @@ IOThread *iothread_by_id(const char *id) {return IOTHREAD (object_resolve_path_type (id,TYPE_IOTHREAD ,NULL )); } + +bool qemu_in_iothread (void ) +{ +return qemu_get_current_aio_context () ==qemu_get_aio_context () ? + false : true; +}
MAINTAINERS +2 -0
@@ -3191,6 +3191,8 @@ F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h F: hw/remote/machine.c F: include/hw/remote/machine.h +F: hw/remote/mpqemu-link.c +F: include/hw/remote/mpqemu-link.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -1,5 +1,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
hw/remote/trace-events new file +4 -0
@@ -0,0 +1,4 @@ +# multi-process trace events + +mpqemu_send_io_error(int cmd, int size, int nfds) "send command %d size %d, %d file descriptors to remote process" +mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, %d file descriptors to remote process"
Signature · 1 line
1.8.3.1
PATCH v19 14/20 multi-process: add proxy communication functions
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
--- include/hw/remote/mpqemu-link.h | 4 ++++ hw/remote/mpqemu-link.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+)
include/hw/remote/mpqemu-link.h +4 -0
@@ -15,6 +15,8 @@#include "qemu/thread.h" #include "io/channel.h" #include "exec/hwaddr.h" +#include "io/channel-socket.h" +#include "hw/remote/proxy.h" #define REMOTE_MAX_FDS 8
@@ -68,6 +70,8 @@ typedef struct {bool mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp);bool mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp);bool mpqemu_msg_valid (MPQemuMsg *msg);#endif
hw/remote/mpqemu-link.c +34 -0
@@ -182,6 +182,40 @@ fail:return ret; } +/* +* Send msg and wait for a reply with command code RET_MSG. +* Returns the message received of size u64 or UINT64_MAX +* on error. +* Called from VCPU thread in non-coroutine context. +* Used by the Proxy object to communicate to remote processes. +*/ +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp) +{ +ERRP_GUARD (); +MPQemuMsg msg_reply = {0 }; +uint64_t ret =UINT64_MAX ; + +assert (!qemu_in_coroutine ()); + +QEMU_LOCK_GUARD (&pdev->io_mutex); +if (!mpqemu_msg_send (msg, pdev->ioc, errp)) { +return ret; + } + +if (!mpqemu_msg_recv (&msg_reply, pdev->ioc, errp)) { +return ret; + } + +if (!mpqemu_msg_valid (&msg_reply)) { +error_setg (errp,"ERROR: Invalid reply received for command %d" , + msg->cmd); +return ret; + } + +return msg_reply.data.u64; +} +bool mpqemu_msg_valid (MPQemuMsg *msg) {if (msg->cmd >=MPQEMU_CMD_MAX && msg->cmd <0 ) {
Signature · 1 line
1.8.3.1
PATCH v19 7/20 io: add qio_channel_writev_full_all helper
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Adds qio_channel_writev_full_all() to transmit both data and FDs. Refactors existing code to use this helper.
--- include/io/channel.h | 25 +++++++++++++++++++++++++ io/channel.c | 15 ++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-)
include/io/channel.h +25 -0
@@ -777,4 +777,29 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_write, void *opaque); +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp); +#endif /* QIO_CHANNEL_H */
io/channel.c +14 -1
@@ -157,6 +157,15 @@ int qio_channel_writev_all(QIOChannel *ioc,size_t niov, Error **errp) { + return qio_channel_writev_full_all(ioc, iov, niov, NULL, 0, errp); +} + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp) +{int ret =-1 ; struct iovec *local_iov = g_new(struct iovec, niov);struct iovec *local_iov_head = local_iov;
@@ -168,7 +177,8 @@ int qio_channel_writev_all(QIOChannel *ioc,while (nlocal_iov >0 ) {ssize_t len; - len =qio_channel_writev (ioc, local_iov, nlocal_iov, errp); + len =qio_channel_writev_full (ioc, local_iov, nlocal_iov, fds, nfds, + errp);if (len ==QIO_CHANNEL_ERR_BLOCK ) {if (qemu_in_coroutine ()) {qio_channel_yield (ioc,G_IO_OUT );
@@ -182,6 +192,9 @@ int qio_channel_writev_all(QIOChannel *ioc, }iov_discard_front (&local_iov, &nlocal_iov, len); + + fds =NULL ; + nfds =0 ; } ret =0 ;
Signature · 1 line
1.8.3.1
PATCH v19 5/20 multi-process: setup PCI host bridge for remote device
In reply to Jagannathan RamanPCI host bridge is setup for the remote device process. It is implemented using remote-pcihost object. It is an extension of the PCI host bridge setup by QEMU.
Remote-pcihost configures a PCI bus which could be used by the remote PCI device to latch on to.
--- include/hw/pci-host/remote.h | 29 +++++++++++++++++ hw/pci-host/remote.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/pci-host/Kconfig | 3 ++ hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 1 + 6 files changed, 111 insertions(+) create mode 100644 include/hw/pci-host/remote.h create mode 100644 hw/pci-host/remote.c
include/hw/pci-host/remote.h new file +29 -0
@@ -0,0 +1,29 @@ +/* +* PCI Host for remote device +* +* Copyright © 2018, 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_PCIHOST_H +#define REMOTE_PCIHOST_H + +#include "exec/memory.h" +#include "hw/pci/pcie_host.h" + +#define TYPE_REMOTE_PCIHOST "remote-pcihost" +OBJECT_DECLARE_SIMPLE_TYPE(RemotePCIHost, REMOTE_PCIHOST) + +struct RemotePCIHost { +/*< private >*/ +PCIExpressHost parent_obj; +/*< public >*/ + +MemoryRegion *mr_pci_mem; +MemoryRegion *mr_sys_io; +}; + +#endif
hw/pci-host/remote.c new file +75 -0
75 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3187,6 +3187,8 @@ M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst F: docs/multi-process.rst +F: hw/pci-host/remote.c +F: include/hw/pci-host/remote.h Build and test automation -------------------------
hw/pci-host/Kconfig +3 -0
@@ -65,3 +65,6 @@ config PCI_POWERNV select PCI_EXPRESS select MSI_NONBROKEN select PCIE_PORT + +config REMOTE_PCIHOST + bool
hw/pci-host/meson.build +1 -0
@@ -9,6 +9,7 @@ pci_ss.add(when: 'CONFIG_PCI_EXPRESS_XILINX', if_true: files('xilinx-pcie.c')) pci_ss.add(when: 'CONFIG_PCI_I440FX', if_true: files('i440fx.c')) pci_ss.add(when: 'CONFIG_PCI_SABRE', if_true: files('sabre.c')) pci_ss.add(when: 'CONFIG_XEN_IGD_PASSTHROUGH', if_true: files('xen_igd_pt.c')) +pci_ss.add(when: 'CONFIG_REMOTE_PCIHOST', if_true: files('remote.c')) # PPC devices pci_ss.add(when: 'CONFIG_PREP_PCI', if_true: files('prep.c'))
hw/remote/Kconfig +1 -0
@@ -1,3 +1,4 @@ config MULTIPROCESS bool depends on PCI && KVM + select REMOTE_PCIHOST
Signature · 1 line
1.8.3.1
PATCH v19 1/20 multi-process: add the concept description to docs/devel/qemu-multiprocess
In reply to Jagannathan RamanFrom: John G Johnson <[email protected]>
--- docs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 +++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 7 + 3 files changed, 974 insertions(+) create mode 100644 docs/devel/multi-process.rst
docs/devel/index.rst +1 -0
@@ -36,3 +36,4 @@ Contents: clocks qom block-coroutine-wrapper + multi-process
docs/devel/multi-process.rst new file +966 -0
966 lines not on this page · raw message
MAINTAINERS +7 -0
@@ -3180,6 +3180,13 @@ S: Maintained F: hw/semihosting/ F: include/hw/semihosting/ +Multi-process QEMU +M: Elena Ufimtseva <[email protected]> +M: Jagannathan Raman <[email protected]> +M: John G Johnson <[email protected]> +S: Maintained +F: docs/devel/multi-process.rst + Build and test automation ------------------------- Build and test automation
Signature · 1 line
1.8.3.1
PATCH v19 8/20 io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Adds qio_channel_readv_full_all_eof() and qio_channel_readv_full_all() to read both data and FDs. Refactors existing code to use these helpers.
--- include/io/channel.h | 51 +++++++++++++++++++++++++++++++ io/channel.c | 85 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 120 insertions(+), 16 deletions(-)
include/io/channel.h +51 -0
@@ -778,6 +778,57 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, void *opaque);/** +* qio_channel_readv_full_all_eof: +* @ioc: the channel object +* @iov: the array of memory regions to read data to +* @niov: the length of the @iov array +* @fds: an array of file handles to read +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Performs same function as qio_channel_readv_all_eof. +* Additionally, attempts to read file descriptors shared +* over the channel. The function will wait for all +* requested data to be read, yielding from the current +* coroutine if required. +* +* Returns: 1 if all bytes were read, 0 if end-of-file +* occurs without data, or -1 on error +*/ + +int qio_channel_readv_full_all_eof (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp); + +/** +* qio_channel_readv_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to read data to +* @niov: the length of the @iov array +* @fds: an array of file handles to read +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Performs same function as qio_channel_readv_all_eof. +* Additionally, attempts to read file descriptors shared +* over the channel. The function will wait for all +* requested data to be read, yielding from the current +* coroutine if required. +* +* Returns: 0 if all bytes were read, or -1 on error +*/ + +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp); + +/** * qio_channel_writev_full_all: * @ioc: the channel object * @iov: the array of memory regions to write data from
io/channel.c +69 -16
128 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v19 4/20 multi-process: Add config option for multi-process QEMU
In reply to Jagannathan RamanAdd configuration options to enable or disable multiprocess QEMU code
--- configure | 10 ++++++++++ meson.build | 4 +++- Kconfig.host | 4 ++++ hw/Kconfig | 1 + hw/remote/Kconfig | 3 +++ 5 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 hw/remote/Kconfig
configure +10 -0
@@ -461,6 +461,7 @@ skip_meson=no gettext="auto" fuse="auto" fuse_lseek="auto" +multiprocess="no" malloc_trim="auto"
@@ -806,6 +807,7 @@ Linux) linux="yes" linux_user="yes" vhost_user=${default_feature:-yes} + multiprocess=${default_feature:-yes} ;; esac
@@ -1558,6 +1560,10 @@ for opt do ;; --disable-fuse-lseek) fuse_lseek="disabled" ;; + --enable-multiprocess) multiprocess="yes" + ;; + --disable-multiprocess) multiprocess="no" + ;; *) echo "ERROR: unknown option $opt" echo "Try '$0 --help' for more information"
@@ -1897,6 +1903,7 @@ disabled with --disable-FEATURE, default is enabled if available libdaxctl libdaxctl support fuse FUSE block device export fuse-lseek SEEK_HOLE/SEEK_DATA support for FUSE exports + multiprocess Multiprocess QEMU support NOTE: The object files are built at the place where configure is launched EOF
@@ -6135,6 +6142,9 @@ fi if test "$have_mlockall" = "yes" ; then echo "HAVE_MLOCKALL=y" >> $config_host_mak fi +if test "$multiprocess" = "yes" ; then + echo "CONFIG_MULTIPROCESS_ALLOWED=y" >> $config_host_mak +fi if test "$fuzzing" = "yes" ; then # If LIB_FUZZING_ENGINE is set, assume we are running on OSS-Fuzz, and the # needed CFLAGS have already been provided
meson.build +3 -1
@@ -1194,7 +1194,8 @@ host_kconfig = \ ('CONFIG_VHOST_KERNEL' in config_host ? ['CONFIG_VHOST_KERNEL=y'] : []) + \ (have_virtfs ? ['CONFIG_VIRTFS=y'] : []) + \ ('CONFIG_LINUX' in config_host ? ['CONFIG_LINUX=y'] : []) + \ - ('CONFIG_PVRDMA' in config_host ? ['CONFIG_PVRDMA=y'] : []) + ('CONFIG_PVRDMA' in config_host ? ['CONFIG_PVRDMA=y'] : []) + \ + ('CONFIG_MULTIPROCESS_ALLOWED' in config_host ? ['CONFIG_MULTIPROCESS_ALLOWED=y'] : []) ignored = [ 'TARGET_XML_FILES', 'TARGET_ABI_DIR', 'TARGET_ARCH' ]
@@ -2508,6 +2509,7 @@ summary_info += {'rng-none': config_host.has_key('CONFIG_RNG_NONE')} summary_info += {'Linux keyring': config_host.has_key('CONFIG_SECRET_KEYRING')} summary_info += {'FUSE exports': fuse.found()} summary_info += {'FUSE lseek': fuse_lseek.found()} +summary_info += {'Multiprocess QEMU': config_host.has_key('CONFIG_MULTIPROCESS_ALLOWED')} summary(summary_info, bool_yn: true) if not supported_cpus.contains(cpu)
Kconfig.host +4 -0
@@ -37,3 +37,7 @@ config VIRTFS config PVRDMA bool + +config MULTIPROCESS_ALLOWED + bool + imply MULTIPROCESS
hw/Kconfig +1 -0
@@ -27,6 +27,7 @@ source pci-host/Kconfig source pcmcia/Kconfig source pci/Kconfig source rdma/Kconfig +source remote/Kconfig source rtc/Kconfig source scsi/Kconfig source sd/Kconfig
hw/remote/Kconfig new file +3 -0
@@ -0,0 +1,3 @@ +config MULTIPROCESS + bool + depends on PCI && KVM
Signature · 1 line
1.8.3.1
PATCH v19 13/20 multi-process: introduce proxy object
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Defines a PCI Device proxy object as a child of TYPE_PCI_DEVICE.
--- include/hw/remote/proxy.h | 33 ++++++++++++++++ hw/remote/proxy.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 135 insertions(+) create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/remote/proxy.c
include/hw/remote/proxy.h new file +33 -0
@@ -0,0 +1,33 @@ +/* +* Copyright © 2018, 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef PROXY_H +#define PROXY_H + +#include "hw/pci/pci.h" +#include "io/channel.h" + +#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev" +OBJECT_DECLARE_SIMPLE_TYPE(PCIProxyDev, PCI_PROXY_DEV) + +struct PCIProxyDev { +PCIDevice parent_dev; +char *fd; + +/* +* Mutex used to protect the QIOChannel fd from +* the concurrent access by the VCPUs since proxy +* blocks while awaiting for the replies from the +* process remote. +*/ +QemuMutex io_mutex; +QIOChannel *ioc; +Error *migration_blocker; +}; + +#endif /* PROXY_H */
hw/remote/proxy.c new file +99 -0
99 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3197,6 +3197,8 @@ F: hw/remote/message.c F: hw/remote/remote-obj.c F: include/hw/remote/memory.h F: hw/remote/memory.c +F: hw/remote/proxy.c +F: include/hw/remote/proxy.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -4,6 +4,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c'))
Signature · 1 line
1.8.3.1
PATCH v19 17/20 multi-process: Synchronize remote memory
In reply to Jagannathan RamanAdd ProxyMemoryListener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sends SYNC_SYSMEM message to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler for SYNC_SYSMEM message.
--- include/hw/remote/proxy-memory-listener.h | 28 ++++ include/hw/remote/proxy.h | 2 + hw/remote/message.c | 4 + hw/remote/proxy-memory-listener.c | 227 ++++++++++++++++++++++++++++++ hw/remote/proxy.c | 6 + MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 270 insertions(+) create mode 100644 include/hw/remote/proxy-memory-listener.h create mode 100644 hw/remote/proxy-memory-listener.c
include/hw/remote/proxy-memory-listener.h new file +28 -0
@@ -0,0 +1,28 @@ +/* +* Copyright © 2018, 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef PROXY_MEMORY_LISTENER_H +#define PROXY_MEMORY_LISTENER_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct ProxyMemoryListener { +MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +}ProxyMemoryListener ; + +void proxy_memory_listener_configure (ProxyMemoryListener *proxy_listener, +QIOChannel *ioc); +void proxy_memory_listener_deconfigure (ProxyMemoryListener *proxy_listener); + +#endif
include/hw/remote/proxy.h +2 -0
@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/proxy-memory-listener.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"OBJECT_DECLARE_SIMPLE_TYPE (PCIProxyDev,PCI_PROXY_DEV )
@@ -36,6 +37,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +ProxyMemoryListener proxy_listener;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };
hw/remote/message.c +4 -0
@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg,Error **errp);
@@ -61,6 +62,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case MPQEMU_CMD_BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case MPQEMU_CMD_SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ;default :error_setg (&local_err, "Unknown command (%d) received for device %s"
hw/remote/proxy-memory-listener.c new file +227 -0
227 lines not on this page · raw message
hw/remote/proxy.c +6 -0
@@ -19,6 +19,8 @@#include "qemu/sockets.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/proxy-memory-listener.h" +#include "qom/object.h" static void pci_proxy_dev_realize (PCIDevice *device,Error **errp) {
@@ -52,6 +54,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +proxy_memory_listener_configure (&dev->proxy_listener, dev->ioc); }static void pci_proxy_dev_exit (PCIDevice *pdev)
@@ -65,6 +69,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +proxy_memory_listener_deconfigure (&dev->proxy_listener); } static void config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,
MAINTAINERS +2 -0
@@ -3199,6 +3199,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/proxy-memory-listener.c +F: include/hw/remote/proxy-memory-listener.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy-memory-listener.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v19 10/20 multi-process: Initialize message handler in remote device
In reply to Jagannathan RamanInitializes the message handler function in the remote process. It is called whenever there's an event pending on QIOChannel that registers this function.
--- include/hw/remote/machine.h | 9 +++++++ hw/remote/message.c | 57 +++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 4 files changed, 68 insertions(+) create mode 100644 hw/remote/message.c
include/hw/remote/machine.h +9 -0
@@ -14,6 +14,7 @@#include "qom/object.h" #include "hw/boards.h" #include "hw/pci-host/remote.h" +#include "io/channel.h" struct RemoteMachineState {MachineState parent_obj;
@@ -21,7 +22,15 @@ struct RemoteMachineState {RemotePCIHost *host; }; +/* Used to pass to co-routine device and ioc. */ +typedef struct RemoteCommDev { +PCIDevice *dev; +QIOChannel *ioc; +}RemoteCommDev ; +#define TYPE_REMOTE_MACHINE "x-remote-machine" OBJECT_DECLARE_SIMPLE_TYPE(RemoteMachineState, REMOTE_MACHINE) +void coroutine_fn mpqemu_remote_msg_loop_co(void *data); +#endif
hw/remote/message.c new file +57 -0
57 lines not on this page · raw message
MAINTAINERS +1 -0
@@ -3193,6 +3193,7 @@ F: hw/remote/machine.c F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h +F: hw/remote/message.c Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -2,5 +2,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v19 6/20 multi-process: setup a machine object for remote device process
In reply to Jagannathan Ramanx-remote-machine object sets up various subsystems of the remote device process. Instantiate PCI host bridge object and initialize RAM, IO & PCI memory regions.
--- include/hw/pci-host/remote.h | 1 + include/hw/remote/machine.h | 27 +++++++++++++++++ hw/remote/machine.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/meson.build | 1 + hw/remote/meson.build | 5 ++++ 6 files changed, 106 insertions(+) create mode 100644 include/hw/remote/machine.h create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/meson.build
include/hw/pci-host/remote.h +1 -0
@@ -24,6 +24,7 @@ struct RemotePCIHost {MemoryRegion *mr_pci_mem;MemoryRegion *mr_sys_io; + MemoryRegion *mr_sys_mem; };#endif
include/hw/remote/machine.h new file +27 -0
@@ -0,0 +1,27 @@ +/* +* Remote machine configuration +* +* Copyright © 2018, 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MACHINE_H +#define REMOTE_MACHINE_H + +#include "qom/object.h" +#include "hw/boards.h" +#include "hw/pci-host/remote.h" + +struct RemoteMachineState { +MachineState parent_obj; + +RemotePCIHost *host; +}; + +#define TYPE_REMOTE_MACHINE "x-remote-machine" +OBJECT_DECLARE_SIMPLE_TYPE(RemoteMachineState, REMOTE_MACHINE) + +#endif
hw/remote/machine.c new file +70 -0
70 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3189,6 +3189,8 @@ F: docs/devel/multi-process.rst F: docs/multi-process.rst F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h +F: hw/remote/machine.c +F: include/hw/remote/machine.h Build and test automation -------------------------
hw/meson.build +1 -0
@@ -56,6 +56,7 @@ subdir('moxie') subdir('nios2') subdir('openrisc') subdir('ppc') +subdir('remote') subdir('riscv') subdir('rx') subdir('s390x')
hw/remote/meson.build new file +5 -0
@@ -0,0 +1,5 @@ +remote_ss = ss.source_set() + +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) + +softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v19 18/20 multi-process: create IOHUB object to handle irq
In reply to Jagannathan RamanIOHUB object is added to manage PCI IRQs. It uses KVM_IRQFD ioctl to create irqfd to injecting PCI interrupts to the guest. IOHUB object forwards the irqfd to the remote process. Remote process uses this fd to directly send interrupts to the guest, bypassing QEMU.
--- include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++++++++++++++ include/hw/remote/machine.h | 2 + include/hw/remote/mpqemu-link.h | 1 + include/hw/remote/proxy.h | 4 ++ hw/remote/iohub.c | 119 ++++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 10 ++++ hw/remote/message.c | 4 ++ hw/remote/mpqemu-link.c | 5 ++ hw/remote/proxy.c | 56 +++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 12 files changed, 249 insertions(+) create mode 100644 include/hw/remote/iohub.h create mode 100644 hw/remote/iohub.c
include/hw/pci/pci_ids.h +3 -0
@@ -192,6 +192,9 @@#define PCI_DEVICE_ID_SUN_SIMBA 0x5000#define PCI_DEVICE_ID_SUN_SABRE 0xa000 +#define PCI_VENDOR_ID_ORACLE 0x108e +#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000 +#define PCI_VENDOR_ID_CMD 0x1095#define PCI_DEVICE_ID_CMD_646 0x0646
include/hw/remote/iohub.h new file +42 -0
42 lines not on this page · raw message
include/hw/remote/machine.h +2 -0
@@ -15,11 +15,13 @@#include "hw/boards.h" #include "hw/pci-host/remote.h" #include "io/channel.h" +#include "hw/remote/iohub.h" struct RemoteMachineState {MachineState parent_obj;RemotePCIHost *host; +RemoteIOHubState iohub; };/* Used to pass to co-routine device and ioc. */
include/hw/remote/mpqemu-link.h +1 -0
@@ -39,6 +39,7 @@ typedef enum {MPQEMU_CMD_PCI_CFGREAD ,MPQEMU_CMD_BAR_WRITE ,MPQEMU_CMD_BAR_READ , +MPQEMU_CMD_SET_IRQFD ,MPQEMU_CMD_MAX , } MPQemuCmd;
include/hw/remote/proxy.h +4 -0
@@ -12,6 +12,7 @@#include "hw/pci/pci.h" #include "io/channel.h" #include "hw/remote/proxy-memory-listener.h" +#include "qemu/event_notifier.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"OBJECT_DECLARE_SIMPLE_TYPE (PCIProxyDev,PCI_PROXY_DEV )
@@ -38,6 +39,9 @@ struct PCIProxyDev {QIOChannel *ioc;Error *migration_blocker;ProxyMemoryListener proxy_listener; +int virq; +EventNotifier intr; +EventNotifier resample;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };
hw/remote/iohub.c new file +119 -0
119 lines not on this page · raw message
hw/remote/machine.c +10 -0
@@ -20,12 +20,15 @@#include "exec/address-spaces.h" #include "exec/memory.h" #include "qapi/error.h" +#include "hw/pci/pci_host.h" +#include "hw/remote/iohub.h" static void remote_machine_init (MachineState *machine) {MemoryRegion *system_memory, *system_io, *pci_memory;RemoteMachineState *s =REMOTE_MACHINE (machine);RemotePCIHost *rem_host; +PCIHostState *pci_host; system_memory =get_system_memory (); system_io =get_system_io ();
@@ -45,6 +48,13 @@ static void remote_machine_init(MachineState *machine)memory_region_add_subregion_overlap (system_memory,0x0 , pci_memory,-1 );qdev_realize (DEVICE (rem_host),sysbus_get_default (), &error_fatal); + + pci_host =PCI_HOST_BRIDGE (rem_host); + +remote_iohub_init (&s->iohub); + +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, + &s->iohub,REMOTE_IOHUB_NB_PIRQS ); }static void remote_machine_class_init (ObjectClass *oc,void *data)
hw/remote/message.c +4 -0
@@ -18,6 +18,7 @@#include "hw/pci/pci.h" #include "exec/memattrs.h" #include "hw/remote/memory.h" +#include "hw/remote/iohub.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg,Error **errp);
@@ -65,6 +66,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case MPQEMU_CMD_SYNC_SYSMEM :remote_sysmem_reconfig (&msg, &local_err);break ; +case MPQEMU_CMD_SET_IRQFD : +process_set_irqfd_msg (pci_dev, &msg); +break ;default :error_setg (&local_err, "Unknown command (%d) received for device %s"
hw/remote/mpqemu-link.c +5 -0
@@ -254,6 +254,11 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case MPQEMU_CMD_SET_IRQFD : +if (msg->size || (msg->num_fds !=2 )) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +56 -0
74 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3201,6 +3201,8 @@ F: hw/remote/proxy.c F: include/hw/remote/proxy.h F: hw/remote/proxy-memory-listener.c F: include/hw/remote/proxy-memory-listener.h +F: hw/remote/iohub.c +F: include/hw/remote/iohub.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -5,6 +5,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy-memory-listener.c'))
Signature · 1 line
1.8.3.1
PATCH v19 20/20 multi-process: perform device reset in the remote process
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Perform device reset in the remote process when QEMU performs device reset. This is required to reset the internal state (like registers, etc...) of emulated devices
--- include/hw/remote/mpqemu-link.h | 1 + hw/remote/message.c | 22 ++++++++++++++++++++++ hw/remote/proxy.c | 19 +++++++++++++++++++ 3 files changed, 42 insertions(+)
include/hw/remote/mpqemu-link.h +1 -0
@@ -40,6 +40,7 @@ typedef enum {MPQEMU_CMD_BAR_WRITE ,MPQEMU_CMD_BAR_READ ,MPQEMU_CMD_SET_IRQFD , +MPQEMU_CMD_DEVICE_RESET ,MPQEMU_CMD_MAX , } MPQemuCmd;
hw/remote/message.c +22 -0
@@ -19,6 +19,7 @@#include "exec/memattrs.h" #include "hw/remote/memory.h" #include "hw/remote/iohub.h" +#include "sysemu/reset.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg,Error **errp);
@@ -26,6 +27,8 @@ static void process_config_read(QIOChannel *ioc, PCIDevice *dev, MPQemuMsg *msg, Error **errp);static void process_bar_write (QIOChannel *ioc,MPQemuMsg *msg,Error **errp);static void process_bar_read (QIOChannel *ioc,MPQemuMsg *msg,Error **errp); +static void process_device_reset_msg (QIOChannel *ioc,PCIDevice *dev, +Error **errp); void coroutine_fn mpqemu_remote_msg_loop_co(void *data) {
@@ -69,6 +72,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case MPQEMU_CMD_SET_IRQFD :process_set_irqfd_msg (pci_dev, &msg);break ; +case MPQEMU_CMD_DEVICE_RESET : +process_device_reset_msg (com->ioc, pci_dev, &local_err); +break ;default :error_setg (&local_err, "Unknown command (%d) received for device %s"
@@ -206,3 +212,19 @@ fail: getpid()); } } + +static void process_device_reset_msg (QIOChannel *ioc,PCIDevice *dev, +Error **errp) +{ +DeviceClass *dc =DEVICE_GET_CLASS (dev); +DeviceState *s =DEVICE (dev); +MPQemuMsg ret = {0 }; + +if (dc->reset) { + dc->reset (s); + } + + ret.cmd =MPQEMU_CMD_RET ; + +mpqemu_msg_send (&ret, ioc, errp); +}
hw/remote/proxy.c +19 -0
@@ -26,6 +26,7 @@#include "util/event_notifier-posix.c" static void probe_pci_info (PCIDevice *dev,Error **errp); +static void proxy_device_reset (DeviceState *dev);static void proxy_intx_update (PCIDevice *pci_dev) {
@@ -202,6 +203,8 @@ static void pci_proxy_dev_class_init(ObjectClass *klass, void *data) k->config_read = pci_proxy_read_config; k->config_write = pci_proxy_write_config; + dc->reset = proxy_device_reset; +device_class_set_props (dc, proxy_properties); }
@@ -358,3 +361,19 @@ static void probe_pci_info(PCIDevice *dev, Error **errp) } } } + +static void proxy_device_reset (DeviceState *dev) +{ +PCIProxyDev *pdev =PCI_PROXY_DEV (dev); +MPQemuMsg msg = {0 }; +Error *local_err =NULL ; + + msg.cmd =MPQEMU_CMD_DEVICE_RESET ; + msg.size =0 ; + +mpqemu_msg_send_and_await_reply (&msg, pdev, &local_err); +if (local_err) { +error_report_err (local_err); + } + +}
Signature · 1 line
1.8.3.1
PATCH v19 2/20 multi-process: add configure and usage information
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Adds documentation explaining the command-line arguments needed to use multi-process.
--- docs/multi-process.rst | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 65 insertions(+) create mode 100644 docs/multi-process.rst
docs/multi-process.rst new file +64 -0
@@ -0,0 +1,64 @@ +Multi-process QEMU +================== + +This document describes how to configure and use multi-process qemu. +For the design document refer to docs/devel/qemu-multiprocess. + +1) Configuration +---------------- + +multi-process is enabled by default for targets that enable KVM + + +2) Usage +-------- + +Multi-process QEMU requires an orchestrator to launch. + +Following is a description of command-line used to launch mpqemu. + +* Orchestrator: + + - The Orchestrator creates a unix socketpair + + - It launches the remote process and passes one of the + sockets to it via command-line. + + - It then launches QEMU and specifies the other socket as an option + to the Proxy device object + +* Remote Process: + + - QEMU can enter remote process mode by using the "remote" machine + option. + + - The orchestrator creates a "remote-object" with details about + the device and the file descriptor for the device + + - The remaining options are no different from how one launches QEMU with + devices. + + - Example command-line for the remote process is as follows: + + /usr/bin/qemu-system-x86_64 \ + -machine x-remote \ + -device lsi53c895a,id=lsi0 \ + -drive id=drive_image2,file=/build/ol7-nvme-test-1.qcow2 \ + -device scsi-hd,id=drive2,drive=drive_image2,bus=lsi0.0,scsi-id=0 \ + -object x-remote-object,id=robj1,devid=lsi1,fd=4, + +* QEMU: + + - Since parts of the RAM are shared between QEMU & remote process, a + memory-backend-memfd is required to facilitate this, as follows: + + -object memory-backend-memfd,id=mem,size=2G + + - A "x-pci-proxy-dev" device is created for each of the PCI devices emulated + in the remote process. A "socket" sub-option specifies the other end of + unix channel created by orchestrator. The "id" sub-option must be specified + and should be the same as the "id" specified for the remote PCI device + + - Example commandline for QEMU is as follows: + + -device x-pci-proxy-dev,id=lsi0,socket=3
MAINTAINERS +1 -0
@@ -3186,6 +3186,7 @@ M: Jagannathan Raman <[email protected]> M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst +F: docs/multi-process.rst Build and test automation -------------------------
Signature · 1 line
1.8.3.1
PATCH v19 11/20 multi-process: Associate fd of a PCIDevice with its object
In reply to Jagannathan RamanAssociate the file descriptor for a PCIDevice in remote process with DeviceState object.
--- hw/remote/remote-obj.c | 203 +++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 3 files changed, 205 insertions(+) create mode 100644 hw/remote/remote-obj.c
hw/remote/remote-obj.c new file +203 -0
@@ -0,0 +1,203 @@ +/* +* Copyright © 2020, 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later. +* +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/error-report.h" +#include "qemu/notify.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" +#include "qemu/sockets.h" +#include "monitor/monitor.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +OBJECT_DECLARE_TYPE(RemoteObject, RemoteObjectClass, REMOTE_OBJECT) + +struct RemoteObjectClass { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}; + +struct RemoteObject { +/* private */ +Object parent; + +Notifier machine_done; + +int32_t fd; +char *devid; + +QIOChannel *ioc; + +DeviceState *dev; +DeviceListener listener; +}; + +static void remote_object_set_fd (Object *obj,const char *str,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); +int fd =-1 ; + + fd =monitor_fd_param (monitor_cur (), str, errp); +if (fd ==-1 ) { +error_prepend (errp,"Could not parse remote object fd %s:" , str); +return ; + } + +if (!fd_is_socket (fd)) { +error_setg (errp,"File descriptor '%s' is not a socket" , str); +close (fd); +return ; + } + + o->fd = fd; +} + +static void remote_object_set_devid (Object *obj,const char *str,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); + + o->devid =g_strdup (str); +} + +static void remote_object_unrealize_listener (DeviceListener *listener, +DeviceState *dev) +{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_unrealize_listener; +device_listener_register (&o->listener); + +/* co-routine should free this. */ + comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + }; + + co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev); +qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { +error_report ("Reached maximum number of devices: %u" , k->max_devs); +return ; + } + + o->ioc =NULL ; + o->fd =-1 ; + o->devid =NULL ; + + k->nr_devs++; + + o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +device_listener_unregister (&o->listener); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc)); + + k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + +/* +* Limit number of supported devices to 1. This is done to avoid devices +* from one VM accessing the RAM of another VM. This is done until we +* start using separate address spaces for individual devices. +*/ + k->max_devs =1 ; + k->nr_devs =0 ; + +object_class_property_add_str (klass,"fd" ,NULL , remote_object_set_fd); +object_class_property_add_str (klass,"devid" ,NULL , + remote_object_set_devid); +} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);
MAINTAINERS +1 -0
@@ -3194,6 +3194,7 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: hw/remote/remote-obj.c Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v19 12/20 multi-process: setup memory manager for remote device
In reply to Jagannathan RamanSyncSysMemMsg message format is defined. It is used to send file descriptors of the RAM regions to remote device.
RAM on the remote device is configured with a set of file descriptors. Old RAM regions are deleted and new regions, each with an fd, is added to the RAM.
--- include/hw/remote/memory.h | 19 ++++++++++++ include/hw/remote/mpqemu-link.h | 10 +++++++ hw/remote/memory.c | 65 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 11 +++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 2 ++ 6 files changed, 109 insertions(+) create mode 100644 include/hw/remote/memory.h create mode 100644 hw/remote/memory.c
include/hw/remote/memory.h new file +19 -0
@@ -0,0 +1,19 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2021 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MEMORY_H +#define REMOTE_MEMORY_H + +#include "exec/hwaddr.h" +#include "hw/remote/mpqemu-link.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp); + +#endif
include/hw/remote/mpqemu-link.h +10 -0
@@ -14,6 +14,7 @@#include "qom/object.h" #include "qemu/thread.h" #include "io/channel.h" +#include "exec/hwaddr.h" #define REMOTE_MAX_FDS 8
@@ -30,9 +31,16 @@* */ typedef enum { +MPQEMU_CMD_SYNC_SYSMEM ,MPQEMU_CMD_MAX , }MPQemuCmd ; +typedef struct { +hwaddr gpas[REMOTE_MAX_FDS ]; +uint64_t sizes[REMOTE_MAX_FDS ]; +off_t offsets[REMOTE_MAX_FDS ]; +}SyncSysmemMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -43,12 +51,14 @@ typedef enum {* MPQemuMsg Format of the message sent to the remote device from QEMU. * */ + typedef struct { int cmd; size_t size; union { uint64_t u64; + SyncSysmemMsg sync_sysmem; } data; int fds[REMOTE_MAX_FDS];
hw/remote/memory.c new file +65 -0
65 lines not on this page · raw message
hw/remote/mpqemu-link.c +11 -0
@@ -201,5 +201,16 @@ bool mpqemu_msg_valid(MPQemuMsg *msg) } } +/* Verify message specific fields. */ +switch (msg->cmd) { +case MPQEMU_CMD_SYNC_SYSMEM : +if (msg->num_fds ==0 || msg->size !=sizeof (SyncSysmemMsg)) { +return false; + } +break ; +default : +break ; + } +return true; }
MAINTAINERS +2 -0
@@ -3195,6 +3195,8 @@ F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c F: hw/remote/remote-obj.c +F: include/hw/remote/memory.h +F: hw/remote/memory.c Build and test automation -------------------------
hw/remote/meson.build +2 -0
@@ -5,4 +5,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) + softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
v19 cover Initial support for multi-process Qemu
Hi,
This is the v19 of the patchset. This version has the following changes:
[PATCH v18 08/20] io: add qio_channel_readv_full_all_eof &
qio_channel_readv_full_all helpers
- Preserved fds/nfds if the read was exclusively for fds. Clears then in all other cases.
To touch upon the history of this project, we posted the Proof Of Concept patches before the BoF session in 2018. Subsequently, we have posted 18 versions on the qemu-devel mailing list. You can find them by following the links below ([1] - [18]). Following people contributed to the design and implementation of this project:
Jagannathan Raman <[email protected]>
Elena Ufimtseva <[email protected]>
John G Johnson <[email protected]>
Stefan Hajnoczi <[email protected]>
Konrad Wilk <[email protected]>
Kanth Ghatraju <[email protected]>
We would like to thank the QEMU community for your feedback in the design and implementation of this project. Qemu wiki page:
https://wiki.qemu.org/Features/MultiProcessQEMU
For the full concept writeup about QEMU multi-process, please refer to docs/devel/qemu-multiprocess.rst. Also, see docs/qemu-multiprocess.txt for usage information.
Thank you for reviewing this series!
[POC]: https://www.mail-archive.com/[email protected]/msg566538.html
[1]: https://www.mail-archive.com/[email protected]/msg602285.html
[2]: https://www.mail-archive.com/[email protected]/msg624877.html
[3]: https://www.mail-archive.com/[email protected]/msg642000.html
[4]: https://www.mail-archive.com/[email protected]/msg655118.html
[5]: https://www.mail-archive.com/[email protected]/msg682429.html
[6]: https://www.mail-archive.com/[email protected]/msg697484.html
[7]: https://patchew.org/QEMU/[email protected]/
[8]: https://www.mail-archive.com/[email protected]/msg727007.html
[9]: https://www.mail-archive.com/[email protected]/msg734275.html
[10]: https://www.mail-archive.com/[email protected]/msg747638.html
[11]: https://www.mail-archive.com/[email protected]/msg750972.html
[12]: https://patchew.org/QEMU/[email protected]/
[13]: https://www.mail-archive.com/[email protected]/msg766825.html
[14]: https://www.mail-archive.com/[email protected]/msg768376.html
[15]: https://www.mail-archive.com/[email protected]/msg769178.html
[16]: https://www.mail-archive.com/[email protected]/msg771455.html
Elena Ufimtseva (8):
multi-process: add configure and usage information
io: add qio_channel_writev_full_all helper
io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all
helpers
multi-process: define MPQemuMsg format and transmission functions
multi-process: introduce proxy object
multi-process: add proxy communication functions
multi-process: Forward PCI config space acceses to the remote process
multi-process: perform device reset in the remote process
Jagannathan Raman (11):
memory: alloc RAM from file at offset
multi-process: Add config option for multi-process QEMU
multi-process: setup PCI host bridge for remote device
multi-process: setup a machine object for remote device process
multi-process: Initialize message handler in remote device
multi-process: Associate fd of a PCIDevice with its object
multi-process: setup memory manager for remote device
multi-process: PCI BAR read/write handling for proxy & remote
endpoints
multi-process: Synchronize remote memory
multi-process: create IOHUB object to handle irq
multi-process: Retrieve PCI info from remote process
John G Johnson (1):
multi-process: add the concept description to
docs/devel/qemu-multiprocess
docs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 ++++++++++++++++++++++++++++++ docs/multi-process.rst | 64 ++ configure | 10 + meson.build | 5 +- hw/remote/trace.h | 1 + include/exec/memory.h | 2 + include/exec/ram_addr.h | 2 +- include/hw/pci-host/remote.h | 30 + include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++ include/hw/remote/machine.h | 38 ++ include/hw/remote/memory.h | 19 + include/hw/remote/mpqemu-link.h | 99 +++ include/hw/remote/proxy-memory-listener.h | 28 + include/hw/remote/proxy.h | 48 ++ include/io/channel.h | 76 +++ include/qemu/mmap-alloc.h | 4 +- include/sysemu/iothread.h | 6 + backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 +- hw/pci-host/remote.c | 75 +++ hw/remote/iohub.c | 119 ++++ hw/remote/machine.c | 80 +++ hw/remote/memory.c | 65 ++ hw/remote/message.c | 230 +++++++ hw/remote/mpqemu-link.c | 267 +++++++++ hw/remote/proxy-memory-listener.c | 227 +++++++ hw/remote/proxy.c | 379 ++++++++++++ hw/remote/remote-obj.c | 203 +++++++ io/channel.c | 100 +++- iothread.c | 6 + softmmu/memory.c | 3 +- softmmu/physmem.c | 11 +- util/mmap-alloc.c | 7 +- util/oslib-posix.c | 2 +- Kconfig.host | 4 + MAINTAINERS | 24 + hw/Kconfig | 1 + hw/meson.build | 1 + hw/pci-host/Kconfig | 3 + hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 4 + hw/remote/meson.build | 13 + hw/remote/trace-events | 4 + 45 files changed, 3247 insertions(+), 31 deletions(-) create mode 100644 docs/devel/multi-process.rst create mode 100644 docs/multi-process.rst create mode 100644 hw/remote/trace.h create mode 100644 include/hw/pci-host/remote.h create mode 100644 include/hw/remote/iohub.h create mode 100644 include/hw/remote/machine.h create mode 100644 include/hw/remote/memory.h create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 include/hw/remote/proxy-memory-listener.h create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/pci-host/remote.c create mode 100644 hw/remote/iohub.c create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/memory.c create mode 100644 hw/remote/message.c create mode 100644 hw/remote/mpqemu-link.c create mode 100644 hw/remote/proxy-memory-listener.c create mode 100644 hw/remote/proxy.c create mode 100644 hw/remote/remote-obj.c create mode 100644 hw/remote/Kconfig create mode 100644 hw/remote/meson.build create mode 100644 hw/remote/trace-events
Signature · 1 line
1.8.3.1
PATCH v19 15/20 multi-process: Forward PCI config space acceses to the remote process
From: Elena Ufimtseva <[email protected]>
The Proxy Object sends the PCI config space accesses as messages to the remote process over the communication channel
--- include/hw/remote/mpqemu-link.h | 10 +++++++ hw/remote/message.c | 60 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 8 +++++- hw/remote/proxy.c | 55 +++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-)
include/hw/remote/mpqemu-link.h +10 -0
@@ -34,6 +34,9 @@*/ typedef enum {MPQEMU_CMD_SYNC_SYSMEM , +MPQEMU_CMD_RET , +MPQEMU_CMD_PCI_CFGWRITE , +MPQEMU_CMD_PCI_CFGREAD ,MPQEMU_CMD_MAX , }MPQemuCmd ;
@@ -43,6 +46,12 @@ typedef struct {off_t offsets[REMOTE_MAX_FDS ]; } SyncSysmemMsg; +typedef struct { +uint32_t addr; +uint32_t val; +int len; +}PciConfDataMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -60,6 +69,7 @@ typedef struct {union {uint64_t u64; +PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; } data;
hw/remote/message.c +60 -0
75 lines not on this page · raw message
hw/remote/mpqemu-link.c +7 -1
@@ -207,7 +207,7 @@ uint64_t mpqemu_msg_send_and_await_reply(MPQemuMsg *msg, PCIProxyDev *pdev,return ret; } -if (!mpqemu_msg_valid (&msg_reply)) { +if (!mpqemu_msg_valid (&msg_reply) || msg_reply.cmd !=MPQEMU_CMD_RET ) {error_setg (errp,"ERROR: Invalid reply received for command %d" , msg->cmd);return ret;
@@ -242,6 +242,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case MPQEMU_CMD_PCI_CFGWRITE : +case MPQEMU_CMD_PCI_CFGREAD : +if (msg->size !=sizeof (PciConfDataMsg)) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +55 -0
73 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v19 19/20 multi-process: Retrieve PCI info from remote process
In reply to Jagannathan RamanRetrieve PCI configuration info about the remote device and configure the Proxy PCI object based on the returned information
--- hw/remote/proxy.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+)
hw/remote/proxy.c +84 -0
@@ -25,6 +25,8 @@#include "sysemu/kvm.h" #include "util/event_notifier-posix.c" +static void probe_pci_info (PCIDevice *dev,Error **errp); +static void proxy_intx_update (PCIDevice *pci_dev) {PCIProxyDev *dev =PCI_PROXY_DEV (pci_dev);
@@ -77,6 +79,7 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp) {ERRP_GUARD ();PCIProxyDev *dev =PCI_PROXY_DEV (device); +uint8_t *pci_conf = device->config;int fd;if (!dev->fd) {
@@ -106,9 +109,14 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + pci_conf[PCI_LATENCY_TIMER ] =0xff ; + pci_conf[PCI_INTERRUPT_PIN ] =0x01 ; +proxy_memory_listener_configure (&dev->proxy_listener, dev->ioc);setup_irqfd (dev); + +probe_pci_info (PCI_DEVICE (dev), errp); }static void pci_proxy_dev_exit (PCIDevice *pdev)
@@ -274,3 +282,79 @@ const MemoryRegionOps proxy_mr_ops = { .max_access_size = 8, }, }; + +static void probe_pci_info (PCIDevice *dev,Error **errp) +{ +PCIDeviceClass *pc =PCI_DEVICE_GET_CLASS (dev); +uint32_t orig_val, new_val, base_class, val; +PCIProxyDev *pdev =PCI_PROXY_DEV (dev); +DeviceClass *dc =DEVICE_CLASS (pc); +uint8_t type; +int i, size; + +config_op_send (pdev,PCI_VENDOR_ID , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->vendor_id = (uint16_t )val; + +config_op_send (pdev,PCI_DEVICE_ID , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->device_id = (uint16_t )val; + +config_op_send (pdev,PCI_CLASS_DEVICE , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->class_id = (uint16_t )val; + +config_op_send (pdev,PCI_SUBSYSTEM_ID , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->subsystem_id = (uint16_t )val; + + base_class = pc->class_id >>4 ; +switch (base_class) { +case PCI_BASE_CLASS_BRIDGE : +set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); +break ; +case PCI_BASE_CLASS_STORAGE : +set_bit (DEVICE_CATEGORY_STORAGE , dc->categories); +break ; +case PCI_BASE_CLASS_NETWORK : +set_bit (DEVICE_CATEGORY_NETWORK , dc->categories); +break ; +case PCI_BASE_CLASS_INPUT : +set_bit (DEVICE_CATEGORY_INPUT , dc->categories); +break ; +case PCI_BASE_CLASS_DISPLAY : +set_bit (DEVICE_CATEGORY_DISPLAY , dc->categories); +break ; +case PCI_BASE_CLASS_PROCESSOR : +set_bit (DEVICE_CATEGORY_CPU , dc->categories); +break ; +default : +set_bit (DEVICE_CATEGORY_MISC , dc->categories); +break ; + } + +for (i =0 ; i <PCI_NUM_REGIONS ; i++) { +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &orig_val,4 , +MPQEMU_CMD_PCI_CFGREAD ); + new_val =0xffffffff ; +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &new_val,4 , +MPQEMU_CMD_PCI_CFGWRITE ); +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &new_val,4 , +MPQEMU_CMD_PCI_CFGREAD ); + size = (~(new_val &0xFFFFFFF0 )) +1 ; +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &orig_val,4 , +MPQEMU_CMD_PCI_CFGWRITE ); + type = (new_val &0x1 ) ? +PCI_BASE_ADDRESS_SPACE_IO :PCI_BASE_ADDRESS_SPACE_MEMORY ; + +if (size) { + g_autofree char *name; + pdev->region[i].dev = pdev; + pdev->region[i].present = true; +if (type ==PCI_BASE_ADDRESS_SPACE_MEMORY ) { + pdev->region[i].memory = true; + } + name =g_strdup_printf ("bar-region-%d" , i); +memory_region_init_io (&pdev->region[i].mr,OBJECT (pdev), + &proxy_mr_ops, &pdev->region[i], + name, size); +pci_register_bar (dev, i, type, &pdev->region[i].mr); + } + } +}
Signature · 1 line
1.8.3.1
PATCH v19 16/20 multi-process: PCI BAR read/write handling for proxy & remote endpoints
In reply to Jagannathan RamanProxy device object implements handler for PCI BAR writes and reads. The handler uses BAR_WRITE/BAR_READ message to communicate to the remote process with the BAR address and value to be written/read. The remote process implements handler for BAR_WRITE/BAR_READ message.
--- include/hw/remote/mpqemu-link.h | 10 +++++ include/hw/remote/proxy.h | 9 +++++ hw/remote/message.c | 83 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 6 +++ hw/remote/proxy.c | 60 +++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+)
include/hw/remote/mpqemu-link.h +10 -0
@@ -37,6 +37,8 @@ typedef enum {MPQEMU_CMD_RET ,MPQEMU_CMD_PCI_CFGWRITE ,MPQEMU_CMD_PCI_CFGREAD , +MPQEMU_CMD_BAR_WRITE , +MPQEMU_CMD_BAR_READ ,MPQEMU_CMD_MAX , } MPQemuCmd;
@@ -52,6 +54,13 @@ typedef struct {int len; } PciConfDataMsg; +typedef struct { +hwaddr addr; +uint64_t val; +unsigned size; +bool memory; +}BarAccessMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -71,6 +80,7 @@ typedef struct {uint64_t u64;PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; +BarAccessMsg bar_access; } data;int fds[REMOTE_MAX_FDS ];
include/hw/remote/proxy.h +9 -0
@@ -15,6 +15,14 @@#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev" OBJECT_DECLARE_SIMPLE_TYPE(PCIProxyDev, PCI_PROXY_DEV) +typedef struct ProxyMemoryRegion { +PCIProxyDev *dev; +MemoryRegion mr; +bool memory; +bool present; +uint8_t type; +}ProxyMemoryRegion ; +struct PCIProxyDev {PCIDevice parent_dev;char *fd;
@@ -28,6 +36,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; + ProxyMemoryRegion region[PCI_NUM_REGIONS]; };#endif /* PROXY_H */
hw/remote/message.c +83 -0
103 lines not on this page · raw message
hw/remote/mpqemu-link.c +6 -0
@@ -248,6 +248,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case MPQEMU_CMD_BAR_WRITE : +case MPQEMU_CMD_BAR_READ : +if ((msg->size !=sizeof (BarAccessMsg)) || (msg->num_fds !=0 )) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +60 -0
63 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
In reply to Jagannathan Ramanio/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +6
-2+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp) { -int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );
qio_channel_readv_full_all_eof() can read file descriptors but no data and return 0.
Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
Quoted 19 lines · Stefan Hajnoczi
io/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +5
-0+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp){
io/channel.c +1
-2-int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );
Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
That’s a valid point. I’m wondering if the fix for this case should be in qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof() should probably return error (-1) if the amount of data read does not match iov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?
Thanks!
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
In reply to Daniel P. BerrangéQuoted 51 lines · Daniel P. Berrangé
Quoted 25 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
io/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +5
-0+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp){
io/channel.c +1
-2-int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Quoted 8 lines
That’s a valid point. I’m wondering if the fix for this case should be in
qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof()should probably return error (-1) if the amount of data read does not matchiov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?The API spec for the existing
_eof()methods says:* The function will wait for all requested data
* to be read, yielding from the current coroutine
* if required.
*
* If end-of-file occurs before any data is read,
* no error is reported; otherwise, if it occurs
* before all requested data has been read, an error
* will be reported.
Quoted 6 lines
IOW, return '0' is *only* valid if we've not read anything. I consider file descriptors to be something.
IOW,
qio_channel_readv_full_all_eofmust only return 0, if it didn't read any data and also didn't receive any file descriptors. So yeah, we must return -1 in the scenario Stefan describes
That makes sense to me. Reading “fds" is something, which is different from our previous understanding. I thought data only meant iov, and not fds.
So the return values for qio_channel_readv_full_all_eof() would be:
- ‘0’ only if EOF is reached without reading any fds and data.
- ‘1’ if all data that the caller expects are read (even if the caller reads fds exclusively, without any iovs)
- ‘-1’ otherwise, considered as error
qio_channel_readv_full_all() would return:
- ‘0’ if all the data that caller expects are read
- ‘-1’ otherwise, considered as error
Hey Stefan,
Does this sound good to you?
Thank you!
--
Jag
Quoted 8 lines
Regards,
DanielSignature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
In reply to Jag RamanQuoted 25 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
io/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +5
-0+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp){
io/channel.c +1
-2-int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Quoted 8 lines
That’s a valid point. I’m wondering if the fix for this case should be in
qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof()should probably return error (-1) if the amount of data read does not matchiov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?
The API spec for the existing _eof() methods says:
* The function will wait for all requested data
* to be read, yielding from the current coroutine
* if required.
*
* If end-of-file occurs before any data is read,
* no error is reported; otherwise, if it occurs
* before all requested data has been read, an error
* will be reported.
IOW, return '0' is *only* valid if we've not read anything. I consider file descriptors to be something.
IOW, qio_channel_readv_full_all_eof must only return 0, if it didn't read any data and also didn't receive any file descriptors. So yeah, we must return -1 in the scenario Stefan describes
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
In reply to Jag RamanQuoted 67 lines · Jag Raman
Quoted 51 lines · Daniel P. Berrangé
Quoted 25 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
io/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +5
-0+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp){
io/channel.c +1
-2-int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Quoted 8 lines
That’s a valid point. I’m wondering if the fix for this case should be in
qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof()should probably return error (-1) if the amount of data read does not matchiov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?The API spec for the existing
_eof()methods says:* The function will wait for all requested data
* to be read, yielding from the current coroutine
* if required.
*
* If end-of-file occurs before any data is read,
* no error is reported; otherwise, if it occurs
* before all requested data has been read, an error
* will be reported.Quoted 6 lines
IOW, return '0' is *only* valid if we've not read anything. I consider file descriptors to be something.
IOW,
qio_channel_readv_full_all_eofmust only return 0, if it didn't read any data and also didn't receive any file descriptors. So yeah, we must return -1 in the scenario Stefan describesThat makes sense to me. Reading “fds" is something, which is different from our previous understanding. I thought data only meant iov, and not fds.
So the return values for
qio_channel_readv_full_all_eof()would be:
- ‘0’ only if EOF is reached without reading any fds and data.
- ‘1’ if all data that the caller expects are read (even if the caller reads fds exclusively, without any iovs)
- ‘-1’ otherwise, considered as error
Quoted 7 lines
qio_channel_readv_full_all()would return:
- ‘0’ if all the data that caller expects are read
- ‘-1’ otherwise, considered as errorHey Stefan,
Does this sound good to you?
The while (nlocal_iov > 0) loop only runs if the caller has requested to read at least some data, so the fds-only case doesn't work yet.
This suggests that no current QEMU code relies on the fds-only case. Therefore you could change the doc comment to clarify this instead of adding support for this case to the code.
But if you would to fully support the fds-only case that would be even better.
Stefan
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
Quoted 83 lines · Stefan Hajnoczi
Quoted 67 lines · Jag Raman
Quoted 51 lines · Daniel P. Berrangé
Quoted 25 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
io/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +5
-0+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp){
io/channel.c +1
-2-int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Quoted 8 lines
That’s a valid point. I’m wondering if the fix for this case should be in
qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof()should probably return error (-1) if the amount of data read does not matchiov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?The API spec for the existing
_eof()methods says:* The function will wait for all requested data
* to be read, yielding from the current coroutine
* if required.
*
* If end-of-file occurs before any data is read,
* no error is reported; otherwise, if it occurs
* before all requested data has been read, an error
* will be reported.Quoted 6 lines
IOW, return '0' is *only* valid if we've not read anything. I consider file descriptors to be something.
IOW,
qio_channel_readv_full_all_eofmust only return 0, if it didn't read any data and also didn't receive any file descriptors. So yeah, we must return -1 in the scenario Stefan describesThat makes sense to me. Reading “fds" is something, which is different from our previous understanding. I thought data only meant iov, and not fds.
So the return values for
qio_channel_readv_full_all_eof()would be:
- ‘0’ only if EOF is reached without reading any fds and data.
- ‘1’ if all data that the caller expects are read (even if the caller reads fds exclusively, without any iovs)
- ‘-1’ otherwise, considered as errorQuoted 7 lines
qio_channel_readv_full_all()would return:
- ‘0’ if all the data that caller expects are read
- ‘-1’ otherwise, considered as errorHey Stefan,
Does this sound good to you?
The while (
nlocal_iov> 0) loop only runs if the caller has requested to read at least some data, so the fds-only case doesn't work yet.
Quoted 6 lines
This suggests that no current QEMU code relies on the fds-only case. Therefore you could change the doc comment to clarify this instead of adding support for this case to the code.
But if you would to fully support the fds-only case that would be even better.
Thanks for confirming, Stefan!
Since we are adding fds support in this patch, I suppose we could add the fds-only case since you feel that’s a better approach.
I believe it’s reasonable to assume that the caller wants to read file descriptors if it sets the fds double-pointer. I think it would be reasonable to wait for data if the caller expects to read either data or fds, as follows:
while ((nlocal_iov > 0) || local_fds)
Thank you!
Quoted 2 lines
Stefan
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
In reply to Stefan HajnocziQuoted 83 lines · Stefan Hajnoczi
Quoted 67 lines · Jag Raman
Quoted 51 lines · Daniel P. Berrangé
Quoted 25 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
io/channel.c int qio_channel_readv_all_eof(QIOChannel *ioc, · line 198
Quoted 13 lines · Jagannathan Raman
io/channel.c +5
-0+int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp){
io/channel.c +1
-2-int ret =qio_channel_readv_all_eof (ioc, iov, niov, errp); +int ret =qio_channel_readv_full_all_eof (ioc, iov, niov, fds, nfds, errp);if (ret ==0 ) { - ret =-1 ;error_setg (errp,"Unexpected end-of-file before all bytes were read" );Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Quoted 8 lines
That’s a valid point. I’m wondering if the fix for this case should be in
qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof()should probably return error (-1) if the amount of data read does not matchiov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?The API spec for the existing
_eof()methods says:* The function will wait for all requested data
* to be read, yielding from the current coroutine
* if required.
*
* If end-of-file occurs before any data is read,
* no error is reported; otherwise, if it occurs
* before all requested data has been read, an error
* will be reported.Quoted 6 lines
IOW, return '0' is *only* valid if we've not read anything. I consider file descriptors to be something.
IOW,
qio_channel_readv_full_all_eofmust only return 0, if it didn't read any data and also didn't receive any file descriptors. So yeah, we must return -1 in the scenario Stefan describesThat makes sense to me. Reading “fds" is something, which is different from our previous understanding. I thought data only meant iov, and not fds.
So the return values for
qio_channel_readv_full_all_eof()would be:
- ‘0’ only if EOF is reached without reading any fds and data.
- ‘1’ if all data that the caller expects are read (even if the caller reads fds exclusively, without any iovs)
- ‘-1’ otherwise, considered as errorQuoted 7 lines
qio_channel_readv_full_all()would return:
- ‘0’ if all the data that caller expects are read
- ‘-1’ otherwise, considered as errorHey Stefan,
Does this sound good to you?
The while (
nlocal_iov> 0) loop only runs if the caller has requested to read at least some data, so the fds-only case doesn't work yet.
Quoted 8 lines
This suggests that no current QEMU code relies on the fds-only case. Therefore you could change the doc comment to clarify this instead of adding support for this case to the code.
But if you would to fully support the fds-only case that would be even better.
Stefan
We are working on sending the next revision out. We could handle the fds-only case by altering the while loop condition to be:
((nlocal_iov > 0) || local_fds)
For reference, we would need to handle the following cases:
len < 0; !partial, !*nfds => ret = -1; len = 0; !partial, !*nfds => ret = 0; len < 0; partial, !*nfds => ret = -1; errmsg; len = 0; partial, !*nfds => ret = -1; errmsg; len < 0; partial, *nfds => ret = -1; errmsg, clearfds len < 0; !partial, *nfds => ret = -1; errmsg, clearfds len = 0; partial, *nfds => ret = -1; errmsg, clearfds len = 0; !partial, *nfds => ret = -1; errmsg, clearfds
len = 0; !niov; (nfds && *nfds) => ret = 1 /* fds-only */
len > 0 => ret 1
Thank you!
--
Jag
Re: [PATCH v19 08/20] io: add qio_channel_readv_full_all_eof & qio_channel_readv_full_all helpers
Quoted 105 lines · Jag Raman
Quoted 83 lines · Stefan Hajnoczi
Quoted 67 lines · Jag Raman
Quoted 51 lines · Daniel P. Berrangé
Quoted 25 lines · Jag Raman
Quoted 19 lines · Stefan Hajnoczi
Review
Quoted 13 lines · Jagannathan Raman
(quoted diff) +5
-0+int qio_channel_readv_full_all(QIOChannel *ioc, + const struct iovec *iov, + size_t niov, + int **fds, size_t *nfds, + Error **errp){
(quoted diff) +1
-2- int ret = qio_channel_readv_all_eof(ioc, iov, niov, errp); + int ret = qio_channel_readv_full_all_eof(ioc, iov, niov, fds, nfds, errp); if (ret == 0) { - ret = -1; error_setg(errp, "Unexpected end-of-file before all bytes were read");Quoted 5 lines
qio_channel_readv_full_all_eof()can read file descriptors but no data and return 0.Here that case is converted into an error and the file descriptors aren't closed, freed, and fds/nfds isn't cleared.
Quoted 8 lines
That’s a valid point. I’m wondering if the fix for this case should be in
qio_channel_readv_full_all_eof(), instead of here.
qio_channel_readv_full_all_eof()should probably return error (-1) if the amount of data read does not matchiov_size(). If the caller is only expecting to read fds, and not any data, it would indicate that by setting iov to NULL and/or setting niov=0. If the caller is setting these parameters, it means it is expecting data.Does that sound good?The API spec for the existing
_eof()methods says:* The function will wait for all requested data
* to be read, yielding from the current coroutine
* if required.
*
* If end-of-file occurs before any data is read,
* no error is reported; otherwise, if it occurs
* before all requested data has been read, an error
* will be reported.Quoted 6 lines
IOW, return '0' is *only* valid if we've not read anything. I consider file descriptors to be something.
IOW,
qio_channel_readv_full_all_eofmust only return 0, if it didn't read any data and also didn't receive any file descriptors. So yeah, we must return -1 in the scenario Stefan describesThat makes sense to me. Reading “fds" is something, which is different from our previous understanding. I thought data only meant iov, and not fds.
So the return values for
qio_channel_readv_full_all_eof()would be:
- ‘0’ only if EOF is reached without reading any fds and data.
- ‘1’ if all data that the caller expects are read (even if the caller reads fds exclusively, without any iovs)
- ‘-1’ otherwise, considered as errorQuoted 7 lines
qio_channel_readv_full_all()would return:
- ‘0’ if all the data that caller expects are read
- ‘-1’ otherwise, considered as errorHey Stefan,
Does this sound good to you?
The while (
nlocal_iov> 0) loop only runs if the caller has requested to read at least some data, so the fds-only case doesn't work yet.Quoted 8 lines
This suggests that no current QEMU code relies on the fds-only case. Therefore you could change the doc comment to clarify this instead of adding support for this case to the code.
But if you would to fully support the fds-only case that would be even better.
Stefan
We are working on sending the next revision out. We could handle the fds-only case by altering the while loop condition to be:
((nlocal_iov> 0) ||local_fds)For reference, we would need to handle the following cases:
len < 0; !partial, !*nfds => ret = -1; len = 0; !partial, !*nfds => ret = 0; len < 0; partial, !*nfds => ret = -1; errmsg; len = 0; partial, !*nfds => ret = -1; errmsg; len < 0; partial, *nfds => ret = -1; errmsg, clearfds len < 0; !partial, *nfds => ret = -1; errmsg, clearfds len = 0; partial, *nfds => ret = -1; errmsg, clearfds len = 0; !partial, *nfds => ret = -1; errmsg, clearfds
Quoted 2 lines
len = 0; !niov; (nfds && *nfds) => ret = 1 /* fds-only */
len > 0 => ret 1
Yes, I think that looks right.
Stefan