Initial support for multi-process Qemu
PATCH v12 3/19 memory: alloc RAM from file at offset
In reply to Jagannathan RamanAllow RAM MemoryRegion to be created from an offset in a file, instead of allocating at offset of 0 by default. This is needed to synchronize RAM between QEMU & remote process.
--- include/exec/memory.h | 2 ++ include/exec/ram_addr.h | 2 +- include/qemu/mmap-alloc.h | 3 ++- backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 ++- softmmu/memory.c | 3 ++- softmmu/physmem.c | 11 +++++++---- util/mmap-alloc.c | 7 ++++--- util/oslib-posix.c | 2 +- 9 files changed, 22 insertions(+), 13 deletions(-)
include/exec/memory.h +2 -0
@@ -980,6 +980,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr,* @size: size of the region. * @share: %true if memory must be mmaped with the MAP_SHARED flag * @fd: the fd to mmap. +* @offset: offset within the file referenced by fd * @errp: pointer to Error*, to store an error if it happens. * * Note that this function does not do anything to cause the data in the
@@ -991,6 +992,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp);#endif
include/exec/ram_addr.h +1 -1
@@ -121,7 +121,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, Error **errp);RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp); +off_t offset,Error **errp);RAMBlock *qemu_ram_alloc_from_ptr (ram_addr_t size,void *host,MemoryRegion *mr,Error **errp);
include/qemu/mmap-alloc.h +2 -1
@@ -25,7 +25,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem); + bool is_pmem, + off_t start);void qemu_ram_munmap (int fd,void *ptr,size_t size);
backends/hostmem-memfd.c +1 -1
@@ -55,7 +55,7 @@ memfd_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) name =host_memory_backend_get_name (backend);memory_region_init_ram_from_fd (&backend->mr,OBJECT (backend), name, backend->size, - backend->share, fd, errp); + backend->share, fd,0 , errp);g_free (name); }
hw/misc/ivshmem.c +2 -1
@@ -494,7 +494,8 @@ static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)/* mmap the region and map into the BAR2 */ memory_region_init_ram_from_fd (&s->server_bar2,OBJECT (s), -"ivshmem.bar2" , size, true, fd, &local_err); +"ivshmem.bar2" , size, true, fd,0 , + &local_err);if (local_err) {error_propagate (errp, local_err);return ;
softmmu/memory.c +2 -1
@@ -1612,6 +1612,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp) { Error *err = NULL;
@@ -1621,7 +1622,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->ram_block =qemu_ram_alloc_from_fd (size, mr, share ?RAM_SHARED :0 , - fd, &err); + fd, offset, &err); mr->dirty_log_mask =tcg_enabled () ? (1 <<DIRTY_MEMORY_CODE ) :0 ;if (err) { mr->size =int128_zero ();
softmmu/physmem.c +7 -4
@@ -1461,6 +1461,7 @@ static void *file_ram_alloc(RAMBlock *block,ram_addr_t memory, int fd, bool truncate, + off_t offset, Error **errp) { void *area;
@@ -1511,7 +1512,8 @@ static void *file_ram_alloc(RAMBlock *block, } area =qemu_ram_mmap (fd, memory, block->mr->align, - block->flags &RAM_SHARED , block->flags &RAM_PMEM ); + block->flags &RAM_SHARED , block->flags &RAM_PMEM , + offset);if (area ==MAP_FAILED ) {error_setg_errno (errp, errno,"unable to map backing store for guest RAM" );
@@ -1943,7 +1945,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)#ifdef CONFIG_POSIX RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp) +off_t offset,Error **errp) {RAMBlock *new_block;Error *local_err =NULL ;
@@ -1996,7 +1998,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, new_block->used_length = size; new_block->max_length = size; new_block->flags = ram_flags; - new_block->host =file_ram_alloc (new_block, size, fd, !file_size, errp); + new_block->host =file_ram_alloc (new_block, size, fd, !file_size, offset, + errp);if (!new_block->host) {g_free (new_block);return NULL ;
@@ -2026,7 +2029,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,return NULL ; } - block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd, errp); + block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd,0 , errp);if (!block) {if (created) {unlink (mem_path);
util/mmap-alloc.c +4 -3
@@ -86,7 +86,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem) + bool is_pmem, + off_t start) { int flags;int map_sync_flags =0 ;
@@ -147,7 +148,7 @@ void *qemu_ram_mmap(int fd, offset =QEMU_ALIGN_UP ((uintptr_t )guardptr, align) - (uintptr_t )guardptr; ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags | map_sync_flags, fd,0 ); + flags | map_sync_flags, fd, start);if (ptr ==MAP_FAILED && map_sync_flags) {if (errno ==ENOTSUP ) {
@@ -172,7 +173,7 @@ void *qemu_ram_mmap(int fd,* we will remove these flags to handle compatibility. */ ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags, fd,0 ); + flags, fd, start); }if (ptr ==MAP_FAILED ) {
util/oslib-posix.c +1 -1
@@ -227,7 +227,7 @@ void *qemu_memalign(size_t alignment, size_t size)void *qemu_anon_ram_alloc (size_t size,uint64_t *alignment,bool shared) {size_t align =QEMU_VMALLOC_ALIGN ; -void *ptr =qemu_ram_mmap (-1 , size, align, shared, false); +void *ptr =qemu_ram_mmap (-1 , size, align, shared, false,0 );if (ptr ==MAP_FAILED ) {return NULL ;
Signature · 1 line
1.8.3.1
v12 cover Initial support for multi-process Qemu
Hello,
This is the v12 of the patchset. Thank you very much for the review of the v11 of the series.
We made changes to the following patches in this version:
- Moved patches 18 & 19 in v11 to the front of the series based on feedback from Phil
- [PATCH v12 02/19 ] multi-process: add configure and usage information
- [PATCH v12 04/19 ] multi-process: Add config option for multi-process QEMU
- [PATCH v12 08/19] multi-process: define MPQemuMsg format and transmission functions.
In summary, we replaced "scripts/mpqemu-launcher.py" with "tests/multiprocess/multiprocess-lsi53c895a.py". We tested this test on x86_64 and aarch64 architectures, which we have access to.
We changed the name of the config variable called CONFIG_MPQEMU to CONFIG_MULTIPROCESS. We also moved the config variable definition out of the "configure" script, and into the Kconfig system. Previously, the user specified if multiprocess was enabled using the "--enable-mpqemu" argument to the configure script. In this version, we changed that. The multiprocess support is enabled automatically if Kconfig system detects KVM support. This is needed to run acceptance tests in the future.
We are working on acceptance tests (tests/acceptance/) for this project. However, we have hit a roadblock and are working with the avocado-devel community to resolve the issue.
We noticed that checkpatch.pl script flagged a warning for Patch 4 for this series, but we don't believe that's a valid concern. We generated the patches using QEMU's git orderfile
(git format-patch -O scripts/git.orderfile ...).
To touch upon the history of this project, we posted the Proof Of Concept patches before the BoF session in 2018. Subsequently, we have posted 11 versions on the qemu-devel mailing list. You can find them by following the links below ([1] - [11]).Following people contributed to the design and implementation of this project:
Jagannathan Raman <[email protected]>
Elena Ufimtseva <[email protected]>
John G Johnson <[email protected]>
Stefan Hajnoczi <[email protected]>
Konrad Wilk <[email protected]>
Kanth Ghatraju <[email protected]>
We would like to thank QEMU community for your feedback in the design and implementation of this project.Qemu wiki page:
https://wiki.qemu.org/Features/MultiProcessQEMU
For the full concept writeup about QEMU multi-process, please refer to docs/devel/qemu-multiprocess.rst. Also see docs/qemu-multiprocess.txt for usage information. We welcome all your ideas, concerns, and questions for this patchset.
Thank you!
[POC]: https://www.mail-archive.com/[email protected]/msg566538.html
[1]: https://www.mail-archive.com/[email protected]/msg602285.html
[2]: https://www.mail-archive.com/[email protected]/msg624877.html
[3]: https://www.mail-archive.com/[email protected]/msg642000.html
[4]: https://www.mail-archive.com/[email protected]/msg655118.html
[5]: https://www.mail-archive.com/[email protected]/msg682429.html
[6]: https://www.mail-archive.com/[email protected]/msg697484.html
[7]: https://patchew.org/QEMU/[email protected]/
[8]: https://www.mail-archive.com/[email protected]/msg727007.html
[9]: https://www.mail-archive.com/[email protected]/msg734275.html
[10]: https://www.mail-archive.com/[email protected]/msg747638.html
[11]: https://www.mail-archive.com/[email protected]/msg750972.htmlThank you!
Elena Ufimtseva (7):
multi-process: add configure and usage information
multi-process: add qio channel function to transmit data and fds
multi-process: define MPQemuMsg format and transmission functions
multi-process: introduce proxy object
multi-process: add proxy communication functions
multi-process: Forward PCI config space acceses to the remote process
multi-process: perform device reset in the remote process
Jagannathan Raman (11):
memory: alloc RAM from file at offset
multi-process: Add config option for multi-process QEMU
multi-process: setup PCI host bridge for remote device
multi-process: setup a machine object for remote device process
multi-process: Initialize message handler in remote device
multi-process: Associate fd of a PCIDevice with its object
multi-process: setup memory manager for remote device
multi-process: PCI BAR read/write handling for proxy & remote
endpoints
multi-process: Synchronize remote memory
multi-process: create IOHUB object to handle irq
multi-process: Retrieve PCI info from remote process
John G Johnson (1):
multi-process: add the concept description to
docs/devel/qemu-multiprocess
docs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 ++++++++++++++++++++++++++ docs/multi-process.rst | 66 ++ include/exec/memory.h | 2 + include/exec/ram_addr.h | 2 +- include/hw/pci-host/remote.h | 31 + include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++ include/hw/remote/machine.h | 40 ++ include/hw/remote/memory-sync.h | 27 + include/hw/remote/memory.h | 19 + include/hw/remote/mpqemu-link.h | 98 +++ include/hw/remote/proxy.h | 53 ++ include/hw/remote/remote-obj.h | 42 ++ include/io/channel.h | 24 + include/qemu/mmap-alloc.h | 3 +- backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 +- hw/pci-host/remote.c | 75 ++ hw/remote/iohub.c | 123 ++++ hw/remote/machine.c | 79 +++ hw/remote/memory-sync.c | 210 ++++++ hw/remote/memory.c | 58 ++ hw/remote/message.c | 241 +++++++ hw/remote/mpqemu-link.c | 308 ++++++++ hw/remote/proxy.c | 378 ++++++++++ hw/remote/remote-obj.c | 154 ++++ io/channel.c | 45 ++ softmmu/memory.c | 3 +- softmmu/physmem.c | 11 +- util/mmap-alloc.c | 7 +- util/oslib-posix.c | 2 +- MAINTAINERS | 25 + accel/Kconfig | 1 + hw/Kconfig | 1 + hw/meson.build | 1 + hw/pci-host/Kconfig | 3 + hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 4 + hw/remote/meson.build | 13 + tests/multiprocess/multiprocess-lsi53c895a.py | 92 +++ 41 files changed, 3246 insertions(+), 13 deletions(-) create mode 100644 docs/devel/multi-process.rst create mode 100644 docs/multi-process.rst create mode 100644 include/hw/pci-host/remote.h create mode 100644 include/hw/remote/iohub.h create mode 100644 include/hw/remote/machine.h create mode 100644 include/hw/remote/memory-sync.h create mode 100644 include/hw/remote/memory.h create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 include/hw/remote/proxy.h create mode 100644 include/hw/remote/remote-obj.h create mode 100644 hw/pci-host/remote.c create mode 100644 hw/remote/iohub.c create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/memory-sync.c create mode 100644 hw/remote/memory.c create mode 100644 hw/remote/message.c create mode 100644 hw/remote/mpqemu-link.c create mode 100644 hw/remote/proxy.c create mode 100644 hw/remote/remote-obj.c create mode 100644 hw/remote/Kconfig create mode 100644 hw/remote/meson.build create mode 100755 tests/multiprocess/multiprocess-lsi53c895a.py
Signature · 1 line
1.8.3.1
PATCH v12 7/19 multi-process: add qio channel function to transmit data and fds
From: Elena Ufimtseva <[email protected]>
Adds QIO channel functions that transmits the input iovs as well as the supplied fds.
--- include/io/channel.h | 24 ++++++++++++++++++++++++ io/channel.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+)
include/io/channel.h +24 -0
@@ -773,5 +773,29 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_read, IOHandler *io_write, void *opaque); +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp);#endif /* QIO_CHANNEL_H */
io/channel.c +45 -0
51 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v12 1/19 multi-process: add the concept description to docs/devel/qemu-multiprocess
In reply to Jagannathan RamanFrom: John G Johnson <[email protected]>
--- docs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 +++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 7 + 3 files changed, 974 insertions(+) create mode 100644 docs/devel/multi-process.rst
docs/devel/index.rst +1 -0
@@ -35,3 +35,4 @@ Contents: clocks qom block-coroutine-wrapper + multi-process
docs/devel/multi-process.rst new file +966 -0
966 lines not on this page · raw message
MAINTAINERS +7 -0
@@ -3130,6 +3130,13 @@ S: Maintained F: hw/semihosting/ F: include/hw/semihosting/ +Multi-process QEMU +M: Elena Ufimtseva <[email protected]> +M: Jagannathan Raman <[email protected]> +M: John G Johnson <[email protected]> +S: Maintained +F: docs/devel/multi-process.rst + Build and test automation ------------------------- Build and test automation
Signature · 1 line
1.8.3.1
PATCH v12 6/19 multi-process: setup a machine object for remote device process
In reply to Jagannathan Ramanx-remote-machine object sets up various subsystems of the remote device process. Instantiate PCI host bridge object and initialize RAM, IO & PCI memory regions.
--- include/hw/pci-host/remote.h | 1 + include/hw/remote/machine.h | 28 ++++++++++++++++++ hw/remote/machine.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/meson.build | 1 + hw/remote/meson.build | 5 ++++ 6 files changed, 106 insertions(+) create mode 100644 include/hw/remote/machine.h create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/meson.build
include/hw/pci-host/remote.h +1 -0
@@ -25,6 +25,7 @@ typedef struct RemotePCIHost {MemoryRegion *mr_pci_mem;MemoryRegion *mr_sys_io; + MemoryRegion *mr_sys_mem; } RemotePCIHost;#endif
include/hw/remote/machine.h new file +28 -0
@@ -0,0 +1,28 @@ +/* +* Remote machine configuration +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MACHINE_H +#define REMOTE_MACHINE_H + +#include "qom/object.h" +#include "hw/boards.h" +#include "hw/pci-host/remote.h" + +typedef struct RemoteMachineState { +MachineState parent_obj; + +RemotePCIHost *host; +}RemoteMachineState ; + +#define TYPE_REMOTE_MACHINE "x-remote-machine" +#define REMOTE_MACHINE (obj) \ + OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) + +#endif
hw/remote/machine.c new file +69 -0
69 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3139,6 +3139,8 @@ F: docs/devel/multi-process.rst F: tests/multiprocess/multiprocess-lsi53c895a.py F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h +F: hw/remote/machine.c +F: include/hw/remote/machine.h Build and test automation -------------------------
hw/meson.build +1 -0
@@ -56,6 +56,7 @@ subdir('moxie') subdir('nios2') subdir('openrisc') subdir('ppc') +subdir('remote') subdir('riscv') subdir('rx') subdir('s390x')
hw/remote/meson.build new file +5 -0
@@ -0,0 +1,5 @@ +remote_ss = ss.source_set() + +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) + +softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v12 9/19 multi-process: Initialize message handler in remote device
In reply to Jagannathan RamanInitializes the message handler function in the remote process. It is called whenever there's an event pending on QIOChannel that registers this function.
--- include/hw/remote/machine.h | 9 +++++++ hw/remote/message.c | 61 +++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 4 files changed, 72 insertions(+) create mode 100644 hw/remote/message.c
include/hw/remote/machine.h +9 -0
@@ -14,6 +14,7 @@#include "qom/object.h" #include "hw/boards.h" #include "hw/pci-host/remote.h" +#include "io/channel.h" typedef struct RemoteMachineState { MachineState parent_obj;
@@ -21,8 +22,16 @@ typedef struct RemoteMachineState {RemotePCIHost *host; } RemoteMachineState; +/* Used to pass to co-routine device and ioc. */ +typedef struct RemoteCommDev { +PCIDevice *dev; +QIOChannel *ioc; +}RemoteCommDev ; +#define TYPE_REMOTE_MACHINE "x-remote-machine"#define REMOTE_MACHINE (obj) \ OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) +void coroutine_fn mpqemu_remote_msg_loop_co(void *data); +#endif
hw/remote/message.c new file +61 -0
61 lines not on this page · raw message
MAINTAINERS +1 -0
@@ -3143,6 +3143,7 @@ F: hw/remote/machine.c F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h +F: hw/remote/message.c Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -2,5 +2,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v12 2/19 multi-process: add configure and usage information
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Adds documentation explaining the command-line arguments needed to use multi-process. Also adds a python script that illustrates the usage.
--- docs/multi-process.rst | 66 +++++++++++++++++++ MAINTAINERS | 1 + tests/multiprocess/multiprocess-lsi53c895a.py | 92 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 docs/multi-process.rst create mode 100755 tests/multiprocess/multiprocess-lsi53c895a.py
docs/multi-process.rst new file +66 -0
@@ -0,0 +1,66 @@ +Multi-process QEMU +================== + +This document describes how to configure and use multi-process qemu. +For the design document refer to docs/devel/qemu-multiprocess. + +1) Configuration +---------------- + +multi-process is enabled by default for targets that enable KVM + + +2) Usage +-------- + +Multi-process QEMU requires an orchestrator to launch. Please refer to a +light-weight python based orchestrator for mpqemu in +scripts/mpqemu-launcher.py to lauch QEMU in multi-process mode. + +Following is a description of command-line used to launch mpqemu. + +* Orchestrator: + + - The Orchestrator creates a unix socketpair + + - It launches the remote process and passes one of the + sockets to it via command-line. + + - It then launches QEMU and specifies the other socket as an option + to the Proxy device object + +* Remote Process: + + - QEMU can enter remote process mode by using the "remote" machine + option. + + - The orchestrator creates a "remote-object" with details about + the device and the file descriptor for the device + + - The remaining options are no different from how one launches QEMU with + devices. + + - Example command-line for the remote process is as follows: + + /usr/bin/qemu-system-x86_64 \ + -machine x-remote \ + -device lsi53c895a,id=lsi0 \ + -drive id=drive_image2,file=/build/ol7-nvme-test-1.qcow2 \ + -device scsi-hd,id=drive2,drive=drive_image2,bus=lsi0.0,scsi-id=0 \ + -object x-remote-object,id=robj1,devid=lsi1,fd=4, + +* QEMU: + + - Since parts of the RAM are shared between QEMU & remote process, a + memory-backend-memfd is required to facilitate this, as follows: + + -object memory-backend-memfd,id=mem,size=2G + + - A "x-pci-proxy-dev" device is created for each of the PCI devices emulated + in the remote process. A "socket" sub-option specifies the other end of + unix channel created by orchestrator. The "id" sub-option must be specified + and should be the same as the "id" specified for the remote PCI device + + - Example commandline for QEMU is as follows: + + -device x-pci-proxy-dev,id=lsi0,socket=3
MAINTAINERS +1 -0
@@ -3136,6 +3136,7 @@ M: Jagannathan Raman <[email protected]> M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst +F: tests/multiprocess/multiprocess-lsi53c895a.py Build and test automation -------------------------
tests/multiprocess/multiprocess-lsi53c895a.py new file +92 -0
92 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v12 5/19 multi-process: setup PCI host bridge for remote device
In reply to Jagannathan RamanPCI host bridge is setup for the remote device process. It is implemented using remote-pcihost object. It is an extension of the PCI host bridge setup by QEMU.
Remote-pcihost configures a PCI bus which could be used by the remote PCI device to latch on to.
--- include/hw/pci-host/remote.h | 30 ++++++++++++++++++ hw/pci-host/remote.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/pci-host/Kconfig | 3 ++ hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 1 + 6 files changed, 112 insertions(+) create mode 100644 include/hw/pci-host/remote.h create mode 100644 hw/pci-host/remote.c
include/hw/pci-host/remote.h new file +30 -0
@@ -0,0 +1,30 @@ +/* +* PCI Host for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_PCIHOST_H +#define REMOTE_PCIHOST_H + +#include "exec/memory.h" +#include "hw/pci/pcie_host.h" + +#define TYPE_REMOTE_HOST_DEVICE "remote-pcihost" +#define REMOTE_HOST_DEVICE (obj) \ + OBJECT_CHECK(RemotePCIHost, (obj), TYPE_REMOTE_HOST_DEVICE) + +typedef struct RemotePCIHost { +/*< private >*/ +PCIExpressHost parent_obj; +/*< public >*/ + +MemoryRegion *mr_pci_mem; +MemoryRegion *mr_sys_io; +}RemotePCIHost ; + +#endif
hw/pci-host/remote.c new file +75 -0
75 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3137,6 +3137,8 @@ M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst F: tests/multiprocess/multiprocess-lsi53c895a.py +F: hw/pci-host/remote.c +F: include/hw/pci-host/remote.h Build and test automation -------------------------
hw/pci-host/Kconfig +3 -0
@@ -60,3 +60,6 @@ config PCI_BONITO select PCI select UNIMP bool + +config MULTIPROCESS_HOST + bool
hw/pci-host/meson.build +1 -0
@@ -9,6 +9,7 @@ pci_ss.add(when: 'CONFIG_PCI_EXPRESS_XILINX', if_true: files('xilinx-pcie.c')) pci_ss.add(when: 'CONFIG_PCI_I440FX', if_true: files('i440fx.c')) pci_ss.add(when: 'CONFIG_PCI_SABRE', if_true: files('sabre.c')) pci_ss.add(when: 'CONFIG_XEN_IGD_PASSTHROUGH', if_true: files('xen_igd_pt.c')) +pci_ss.add(when: 'CONFIG_MULTIPROCESS_HOST', if_true: files('remote.c')) # PPC devices pci_ss.add(when: 'CONFIG_PREP_PCI', if_true: files('prep.c'))
hw/remote/Kconfig +1 -0
@@ -1,3 +1,4 @@ config MULTIPROCESS bool depends on PCI && KVM + select MULTIPROCESS_HOST
Signature · 1 line
1.8.3.1
PATCH v12 12/19 multi-process: introduce proxy object
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Defines a PCI Device proxy object as a child of TYPE_PCI_DEVICE.
--- include/hw/remote/proxy.h | 36 +++++++++++++++++ hw/remote/proxy.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 137 insertions(+) create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/remote/proxy.c
include/hw/remote/proxy.h new file +36 -0
@@ -0,0 +1,36 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef PROXY_H +#define PROXY_H + +#include "hw/pci/pci.h" +#include "io/channel.h" + +#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev" + +#define PCI_PROXY_DEV (obj) \ + OBJECT_CHECK(PCIProxyDev, (obj), TYPE_PCI_PROXY_DEV) +typedef struct PCIProxyDev PCIProxyDev ; + +struct PCIProxyDev { +PCIDevice parent_dev; +char *fd; + +/* +* Mutex used to protect the QIOChannel fd from +* the concurrent access by the VCPUs since proxy +* blocks while awaiting for the replies from the +* process remote. +*/ +QemuMutex io_mutex; +QIOChannel *ioc; +Error *migration_blocker; +}; + +#endif /* PROXY_H */
hw/remote/proxy.c new file +98 -0
98 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3148,6 +3148,8 @@ F: include/hw/remote/remote-obj.h F: hw/remote/remote-obj.c F: include/hw/remote/memory.h F: hw/remote/memory.c +F: hw/remote/proxy.c +F: include/hw/remote/proxy.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -4,6 +4,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c'))
Signature · 1 line
1.8.3.1
PATCH v12 15/19 multi-process: PCI BAR read/write handling for proxy & remote endpoints
In reply to Jagannathan RamanProxy device object implements handler for PCI BAR writes and reads. The handler uses BAR_WRITE/BAR_READ message to communicate to the remote process with the BAR address and value to be written/read. The remote process implements handler for BAR_WRITE/BAR_READ message.
--- include/hw/remote/mpqemu-link.h | 10 +++++ include/hw/remote/proxy.h | 10 +++++ hw/remote/message.c | 87 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 6 +++ hw/remote/proxy.c | 60 ++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+)
include/hw/remote/mpqemu-link.h +10 -0
@@ -36,6 +36,8 @@ typedef enum {RET_MSG ,PCI_CONFIG_WRITE ,PCI_CONFIG_READ , +BAR_WRITE , +BAR_READ ,MPQEMU_CMD_MAX , } MPQemuCmd;
@@ -51,6 +53,13 @@ typedef struct {int l; } PciConfDataMsg; +typedef struct { +hwaddr addr; +uint64_t val; +unsigned size; +bool memory; +}BarAccessMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -70,6 +79,7 @@ typedef struct {uint64_t u64;PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; +BarAccessMsg bar_access; } data;int fds[REMOTE_MAX_FDS ];
include/hw/remote/proxy.h +10 -0
@@ -16,8 +16,17 @@#define PCI_PROXY_DEV (obj) \ OBJECT_CHECK(PCIProxyDev, (obj), TYPE_PCI_PROXY_DEV) +typedef struct PCIProxyDev PCIProxyDev ; +typedef struct ProxyMemoryRegion { +PCIProxyDev *dev; +MemoryRegion mr; +bool memory; +bool present; +uint8_t type; +}ProxyMemoryRegion ; +struct PCIProxyDev {PCIDevice parent_dev;char *fd;
@@ -31,6 +40,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; + ProxyMemoryRegion region[PCI_NUM_REGIONS]; };#endif /* PROXY_H */
hw/remote/message.c +87 -0
107 lines not on this page · raw message
hw/remote/mpqemu-link.c +6 -0
@@ -289,6 +289,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case BAR_WRITE : +case BAR_READ : +if ((msg->size !=sizeof (BarAccessMsg)) || (msg->num_fds !=0 )) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +60 -0
63 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v12 13/19 multi-process: add proxy communication functions
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
--- include/hw/remote/mpqemu-link.h | 4 ++++ hw/remote/mpqemu-link.c | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
include/hw/remote/mpqemu-link.h +4 -0
@@ -15,6 +15,8 @@#include "qemu/thread.h" #include "io/channel.h" #include "exec/hwaddr.h" +#include "io/channel-socket.h" +#include "hw/remote/proxy.h" #define REMOTE_MAX_FDS 8
@@ -65,6 +67,8 @@ typedef struct {int num_fds; } MPQemuMsg; +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp);void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp);void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp);
hw/remote/mpqemu-link.c +38 -0
@@ -17,6 +17,7 @@#include "qemu/iov.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" +#include "io/channel.h" /* * Send message over the ioc QIOChannel.
@@ -219,6 +220,43 @@ fail: } } +/* +* Called from VCPU thread in non-coroutine context. +*/ +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp) +{ +MPQemuMsg msg_reply = {0 }; +uint64_t ret =UINT64_MAX ; +Error *local_err =NULL ; + +qemu_mutex_lock (&pdev->io_mutex); +mpqemu_msg_send (msg, pdev->ioc, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto exit_send; + } + +mpqemu_msg_recv (&msg_reply, pdev->ioc, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto exit_send; + } + +if (!mpqemu_msg_valid (&msg_reply) || msg_reply.cmd !=RET_MSG ) { +error_setg (errp,"ERROR: Invalid reply received for command %d" , + msg->cmd); + goto exit_send; + }else { + ret = msg_reply.data.u64; + } + + exit_send: +qemu_mutex_unlock (&pdev->io_mutex); + +return ret; +} +bool mpqemu_msg_valid (MPQemuMsg *msg) {if (msg->cmd >=MPQEMU_CMD_MAX && msg->cmd <0 ) {
Signature · 1 line
1.8.3.1
PATCH v12 11/19 multi-process: setup memory manager for remote device
In reply to Jagannathan RamanSyncSysMemMsg message format is defined. It is used to send file descriptors of the RAM regions to remote device.
RAM on the remote device is configured with a set of file descriptors. Old RAM regions are deleted and new regions, each with an fd, is added to the RAM.
--- include/hw/remote/memory.h | 19 ++++++++++++++ include/hw/remote/mpqemu-link.h | 13 +++++++++ hw/remote/memory.c | 58 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 11 ++++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 2 ++ 6 files changed, 105 insertions(+) create mode 100644 include/hw/remote/memory.h create mode 100644 hw/remote/memory.c
include/hw/remote/memory.h new file +19 -0
@@ -0,0 +1,19 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MEMORY_H +#define REMOTE_MEMORY_H + +#include "exec/hwaddr.h" +#include "hw/remote/mpqemu-link.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp); + +#endif
include/hw/remote/mpqemu-link.h +13 -0
@@ -14,6 +14,7 @@#include "qom/object.h" #include "qemu/thread.h" #include "io/channel.h" +#include "exec/hwaddr.h" #define REMOTE_MAX_FDS 8
@@ -24,12 +25,22 @@* * MPQemuCmd enum type to specify the command to be executed on the remote * device. +* +* SYNC_SYSMEM Shares QEMU's RAM with remote device's RAM */ typedef enum {MPQEMU_CMD_INIT , +SYNC_SYSMEM , +RET_MSG ,MPQEMU_CMD_MAX , }MPQemuCmd ; +typedef struct { +hwaddr gpas[REMOTE_MAX_FDS ]; +uint64_t sizes[REMOTE_MAX_FDS ]; +off_t offsets[REMOTE_MAX_FDS ]; +}SyncSysmemMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -40,12 +51,14 @@ typedef enum {* MPQemuMsg Format of the message sent to the remote device from QEMU. * */ + typedef struct { int cmd; size_t size; union { uint64_t u64; + SyncSysmemMsg sync_sysmem; } data; int fds[REMOTE_MAX_FDS];
hw/remote/memory.c new file +58 -0
58 lines not on this page · raw message
hw/remote/mpqemu-link.c +11 -0
@@ -238,5 +238,16 @@ bool mpqemu_msg_valid(MPQemuMsg *msg) } } +/* Verify message specific fields. */ +switch (msg->cmd) { +case SYNC_SYSMEM : +if (msg->num_fds ==0 || msg->size !=sizeof (SyncSysmemMsg)) { +return false; + } +break ; +default : +break ; + } +return true; }
MAINTAINERS +2 -0
@@ -3146,6 +3146,8 @@ F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c F: include/hw/remote/remote-obj.h F: hw/remote/remote-obj.c +F: include/hw/remote/memory.h +F: hw/remote/memory.c Build and test automation -------------------------
hw/remote/meson.build +2 -0
@@ -5,4 +5,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) + softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v12 4/19 multi-process: Add config option for multi-process QEMU
In reply to Jagannathan RamanAdd a configuration option to separate multi-process code
--- accel/Kconfig | 1 + hw/Kconfig | 1 + hw/remote/Kconfig | 3 +++ 3 files changed, 5 insertions(+) create mode 100644 hw/remote/Kconfig
accel/Kconfig +1 -0
@@ -3,6 +3,7 @@ config TCG config KVM bool + select MULTIPROCESS config XEN bool
hw/Kconfig +1 -0
@@ -27,6 +27,7 @@ source pci-host/Kconfig source pcmcia/Kconfig source pci/Kconfig source rdma/Kconfig +source remote/Kconfig source rtc/Kconfig source scsi/Kconfig source sd/Kconfig
hw/remote/Kconfig new file +3 -0
@@ -0,0 +1,3 @@ +config MULTIPROCESS + bool + depends on PCI && KVM
Signature · 1 line
1.8.3.1
PATCH v12 16/19 multi-process: Synchronize remote memory
In reply to Jagannathan RamanAdd memory-listener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sends SYNC_SYSMEM message to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler for SYNC_SYSMEM message.
--- include/hw/remote/memory-sync.h | 27 ++++++ include/hw/remote/proxy.h | 2 + hw/remote/memory-sync.c | 210 ++++++++++++++++++++++++++++++++++++++++ hw/remote/message.c | 5 + hw/remote/proxy.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 253 insertions(+) create mode 100644 include/hw/remote/memory-sync.h create mode 100644 hw/remote/memory-sync.c
include/hw/remote/memory-sync.h new file +27 -0
@@ -0,0 +1,27 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef MEMORY_SYNC_H +#define MEMORY_SYNC_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct RemoteMemSync { +MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +}RemoteMemSync ; + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc); +void deconfigure_memory_sync (RemoteMemSync *sync); + +#endif
include/hw/remote/proxy.h +2 -0
@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/memory-sync.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"
@@ -40,6 +41,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +RemoteMemSync sync;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };
hw/remote/memory-sync.c new file +210 -0
210 lines not on this page · raw message
hw/remote/message.c +5 -0
@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);
@@ -64,6 +65,10 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ; +default : error_setg(&local_err, "Unknown command (%d) received for device %s (pid=%d)",
hw/remote/proxy.c +6 -0
@@ -18,6 +18,8 @@#include "migration/blocker.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/memory-sync.h" +#include "qom/object.h" static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {
@@ -58,6 +60,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +configure_memory_sync (&dev->sync, dev->ioc); }static void pci_proxy_dev_exit (PCIDevice *pdev)
@@ -69,6 +73,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +deconfigure_memory_sync (&dev->sync); } static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,
MAINTAINERS +2 -0
@@ -3150,6 +3150,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/memory-sync.c +F: include/hw/remote/memory-sync.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory-sync.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v12 18/19 multi-process: Retrieve PCI info from remote process
In reply to Jagannathan RamanRetrieve PCI configuration info about the remote device and configure the Proxy PCI object based on the returned information
--- hw/remote/proxy.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+)
hw/remote/proxy.c +85 -0
@@ -24,6 +24,8 @@#include "sysemu/kvm.h" #include "util/event_notifier-posix.c" +static void probe_pci_info (PCIDevice *dev,Error **errp); +static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) { pdev->ioc =qio_channel_new_fd (fd, errp);
@@ -87,6 +89,7 @@ static void setup_irqfd(PCIProxyDev *dev)static void pci_proxy_dev_realize (PCIDevice *device,Error **errp) {PCIProxyDev *dev =PCI_PROXY_DEV (device); +uint8_t *pci_conf = device->config;int fd;if (dev->fd) {
@@ -114,9 +117,14 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + pci_conf[PCI_LATENCY_TIMER ] =0xff ; + pci_conf[PCI_INTERRUPT_PIN ] =0x01 ; +configure_memory_sync (&dev->sync, dev->ioc);setup_irqfd (dev); + +probe_pci_info (PCI_DEVICE (dev), errp); }static void pci_proxy_dev_exit (PCIDevice *pdev)
@@ -271,3 +279,80 @@ const MemoryRegionOps proxy_mr_ops = { .max_access_size = 8, }, }; + +static void probe_pci_info (PCIDevice *dev,Error **errp) +{ +PCIDeviceClass *pc =PCI_DEVICE_GET_CLASS (dev); +uint32_t orig_val, new_val, base_class, val; +PCIProxyDev *pdev =PCI_PROXY_DEV (dev); +DeviceClass *dc =DEVICE_CLASS (pc); +uint8_t type; +int i, size; +char *name; + +config_op_send (pdev,PCI_VENDOR_ID , &val,2 ,PCI_CONFIG_READ ); + pc->vendor_id = (uint16_t )val; + +config_op_send (pdev,PCI_DEVICE_ID , &val,2 ,PCI_CONFIG_READ ); + pc->device_id = (uint16_t )val; + +config_op_send (pdev,PCI_CLASS_DEVICE , &val,2 ,PCI_CONFIG_READ ); + pc->class_id = (uint16_t )val; + +config_op_send (pdev,PCI_SUBSYSTEM_ID , &val,2 ,PCI_CONFIG_READ ); + pc->subsystem_id = (uint16_t )val; + + base_class = pc->class_id >>4 ; +switch (base_class) { +case PCI_BASE_CLASS_BRIDGE : +set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); +break ; +case PCI_BASE_CLASS_STORAGE : +set_bit (DEVICE_CATEGORY_STORAGE , dc->categories); +break ; +case PCI_BASE_CLASS_NETWORK : +set_bit (DEVICE_CATEGORY_NETWORK , dc->categories); +break ; +case PCI_BASE_CLASS_INPUT : +set_bit (DEVICE_CATEGORY_INPUT , dc->categories); +break ; +case PCI_BASE_CLASS_DISPLAY : +set_bit (DEVICE_CATEGORY_DISPLAY , dc->categories); +break ; +case PCI_BASE_CLASS_PROCESSOR : +set_bit (DEVICE_CATEGORY_CPU , dc->categories); +break ; +default : +set_bit (DEVICE_CATEGORY_MISC , dc->categories); +break ; + } + +for (i =0 ; i <PCI_NUM_REGIONS ; i++) { +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &orig_val,4 , +PCI_CONFIG_READ ); + new_val =0xffffffff ; +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &new_val,4 , +PCI_CONFIG_WRITE ); +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &new_val,4 , +PCI_CONFIG_READ ); + size = (~(new_val &0xFFFFFFF0 )) +1 ; +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &orig_val,4 , +PCI_CONFIG_WRITE ); + type = (new_val &0x1 ) ? +PCI_BASE_ADDRESS_SPACE_IO :PCI_BASE_ADDRESS_SPACE_MEMORY ; + +if (size) { + pdev->region[i].dev = pdev; + pdev->region[i].present = true; +if (type ==PCI_BASE_ADDRESS_SPACE_MEMORY ) { + pdev->region[i].memory = true; + } + name =g_strdup_printf ("bar-region-%d" , i); +memory_region_init_io (&pdev->region[i].mr,OBJECT (pdev), + &proxy_mr_ops, &pdev->region[i], + name, size); +pci_register_bar (dev, i, type, &pdev->region[i].mr); +g_free (name); + } + } +}
Signature · 1 line
1.8.3.1
PATCH v12 10/19 multi-process: Associate fd of a PCIDevice with its object
In reply to Jagannathan RamanAssociate the file descriptor for a PCIDevice in remote process with DeviceState object.
--- include/hw/remote/remote-obj.h | 42 +++++++++++ hw/remote/message.c | 1 + hw/remote/remote-obj.c | 154 +++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 5 files changed, 200 insertions(+) create mode 100644 include/hw/remote/remote-obj.h create mode 100644 hw/remote/remote-obj.c
include/hw/remote/remote-obj.h new file +42 -0
@@ -0,0 +1,42 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later. +* +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_OBJECT_H +#define REMOTE_OBJECT_H + +#include "io/channel.h" +#include "qemu/notify.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT (obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS (obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS (klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}RemoteObjectClass ; + +typedef struct { +/* private */ +Object parent; + +Notifier machine_done; + +int fd; +char *devid; +QIOChannel *ioc; +}RemoteObject ; + +#endif
hw/remote/message.c +1 -0
@@ -56,6 +56,7 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data) } }qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); +g_free (com);return ; }
hw/remote/remote-obj.c new file +154 -0
154 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3144,6 +3144,8 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: include/hw/remote/remote-obj.h +F: hw/remote/remote-obj.c Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v12 17/19 multi-process: create IOHUB object to handle irq
In reply to Jagannathan RamanIOHUB object is added to manage PCI IRQs. It uses KVM_IRQFD ioctl to create irqfd to injecting PCI interrupts to the guest. IOHUB object forwards the irqfd to the remote process. Remote process uses this fd to directly send interrupts to the guest, bypassing QEMU.
--- include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++++++++++++++ include/hw/remote/machine.h | 3 + include/hw/remote/mpqemu-link.h | 1 + include/hw/remote/proxy.h | 5 ++ hw/remote/iohub.c | 123 ++++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 10 ++++ hw/remote/message.c | 4 ++ hw/remote/mpqemu-link.c | 5 ++ hw/remote/proxy.c | 58 +++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 12 files changed, 257 insertions(+) create mode 100644 include/hw/remote/iohub.h create mode 100644 hw/remote/iohub.c
include/hw/pci/pci_ids.h +3 -0
@@ -192,6 +192,9 @@#define PCI_DEVICE_ID_SUN_SIMBA 0x5000#define PCI_DEVICE_ID_SUN_SABRE 0xa000 +#define PCI_VENDOR_ID_ORACLE 0x108e +#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000 +#define PCI_VENDOR_ID_CMD 0x1095#define PCI_DEVICE_ID_CMD_646 0x0646
include/hw/remote/iohub.h new file +42 -0
42 lines not on this page · raw message
include/hw/remote/machine.h +3 -0
@@ -15,11 +15,14 @@#include "hw/boards.h" #include "hw/pci-host/remote.h" #include "io/channel.h" +#include "hw/remote/iohub.h" typedef struct RemoteMachineState {MachineState parent_obj;RemotePCIHost *host; + +RemoteIOHubState iohub; }RemoteMachineState ;/* Used to pass to co-routine device and ioc. */
include/hw/remote/mpqemu-link.h +1 -0
@@ -38,6 +38,7 @@ typedef enum {PCI_CONFIG_READ ,BAR_WRITE ,BAR_READ , +SET_IRQFD ,MPQEMU_CMD_MAX , } MPQemuCmd;
include/hw/remote/proxy.h +5 -0
@@ -12,6 +12,7 @@#include "hw/pci/pci.h" #include "io/channel.h" #include "hw/remote/memory-sync.h" +#include "qemu/event_notifier.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"
@@ -42,6 +43,10 @@ struct PCIProxyDev {QIOChannel *ioc;Error *migration_blocker;RemoteMemSync sync; +int virq; +EventNotifier intr; +EventNotifier resample; +ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };
hw/remote/iohub.c new file +123 -0
123 lines not on this page · raw message
hw/remote/machine.c +10 -0
@@ -20,12 +20,15 @@#include "exec/address-spaces.h" #include "exec/memory.h" #include "qapi/error.h" +#include "hw/pci/pci_host.h" +#include "hw/remote/iohub.h" static void remote_machine_init (MachineState *machine) {MemoryRegion *system_memory, *system_io, *pci_memory;RemoteMachineState *s =REMOTE_MACHINE (machine);RemotePCIHost *rem_host; +PCIHostState *pci_host; system_memory =get_system_memory (); system_io =get_system_io ();
@@ -45,6 +48,13 @@ static void remote_machine_init(MachineState *machine)memory_region_add_subregion_overlap (system_memory,0x0 , pci_memory,-1 );qdev_realize (DEVICE (rem_host),sysbus_get_default (), &error_fatal); + + pci_host =PCI_HOST_BRIDGE (rem_host); + +remote_iohub_init (&s->iohub); + +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, + &s->iohub,REMOTE_IOHUB_NB_PIRQS ); }static void remote_machine_class_init (ObjectClass *oc,void *data)
hw/remote/message.c +4 -0
@@ -18,6 +18,7 @@#include "hw/pci/pci.h" #include "exec/memattrs.h" #include "hw/remote/memory.h" +#include "hw/remote/iohub.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);
@@ -68,6 +69,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case SYNC_SYSMEM :remote_sysmem_reconfig (&msg, &local_err);break ; +case SET_IRQFD : +process_set_irqfd_msg (pci_dev, &msg); +break ;default : error_setg(&local_err,
hw/remote/mpqemu-link.c +5 -0
@@ -295,6 +295,11 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case SET_IRQFD : +if (msg->size || (msg->num_fds !=2 )) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +58 -0
82 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3152,6 +3152,8 @@ F: hw/remote/proxy.c F: include/hw/remote/proxy.h F: hw/remote/memory-sync.c F: include/hw/remote/memory-sync.h +F: hw/remote/iohub.c +F: include/hw/remote/iohub.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -5,6 +5,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory-sync.c'))
Signature · 1 line
1.8.3.1
PATCH v12 19/19 multi-process: perform device reset in the remote process
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Perform device reset in the remote process when QEMU performs device reset. This is required to reset the internal state (like registers, etc...) of emulated devices
--- include/hw/remote/mpqemu-link.h | 1 + hw/remote/message.c | 23 ++++++++++++++++++++++- hw/remote/proxy.c | 20 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-)
include/hw/remote/mpqemu-link.h +1 -0
@@ -39,6 +39,7 @@ typedef enum {BAR_WRITE ,BAR_READ ,SET_IRQFD , +DEVICE_RESET ,MPQEMU_CMD_MAX , } MPQemuCmd;
hw/remote/message.c +22 -1
@@ -19,6 +19,7 @@#include "exec/memattrs.h" #include "hw/remote/memory.h" #include "hw/remote/iohub.h" +#include "sysemu/reset.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);
@@ -26,6 +27,8 @@ static void process_config_read(QIOChannel *ioc, PCIDevice *dev, MPQemuMsg *msg);static void process_bar_write (QIOChannel *ioc,MPQemuMsg *msg,Error **errp);static void process_bar_read (QIOChannel *ioc,MPQemuMsg *msg,Error **errp); +static void process_device_reset_msg (QIOChannel *ioc,PCIDevice *dev, +Error **errp); void coroutine_fn mpqemu_remote_msg_loop_co(void *data) {
@@ -72,7 +75,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case SET_IRQFD :process_set_irqfd_msg (pci_dev, &msg);break ; - +case DEVICE_RESET : +process_device_reset_msg (com->ioc, pci_dev, &local_err); +break ;default : error_setg(&local_err, "Unknown command (%d) received for device %s (pid=%d)",
@@ -218,3 +223,19 @@ fail: "in remote process pid=%d", getpid()); } } + +static void process_device_reset_msg (QIOChannel *ioc,PCIDevice *dev, +Error **errp) +{ +DeviceClass *dc =DEVICE_GET_CLASS (dev); +DeviceState *s =DEVICE (dev); +MPQemuMsg ret = {0 }; + +if (dc->reset) { + dc->reset (s); + } + + ret.cmd =RET_MSG ; + +mpqemu_msg_send (&ret, ioc, errp); +}
hw/remote/proxy.c +20 -0
@@ -25,6 +25,7 @@#include "util/event_notifier-posix.c" static void probe_pci_info (PCIDevice *dev,Error **errp); +static void proxy_device_reset (DeviceState *dev);static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {
@@ -199,6 +200,8 @@ static void pci_proxy_dev_class_init(ObjectClass *klass, void *data) k->config_read = pci_proxy_read_config; k->config_write = pci_proxy_write_config; + dc->reset = proxy_device_reset; +device_class_set_props (dc, proxy_properties); }
@@ -356,3 +359,20 @@ static void probe_pci_info(PCIDevice *dev, Error **errp) } } } + +static void proxy_device_reset (DeviceState *dev) +{ +PCIProxyDev *pdev =PCI_PROXY_DEV (dev); +MPQemuMsg msg = {0 }; +Error *local_err =NULL ; + + msg.cmd =DEVICE_RESET ; + msg.size =0 ; + + (void )mpqemu_msg_send_and_await_reply (&msg, pdev, &local_err); +if (local_err) { +error_report ("Failed to send DEVICE_RESET to the remote process" ); +error_free (local_err); + } + +}
Signature · 1 line
1.8.3.1
PATCH v12 14/19 multi-process: Forward PCI config space acceses to the remote process
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
The Proxy Object sends the PCI config space accesses as messages to the remote process over the communication channel
--- include/hw/remote/mpqemu-link.h | 9 ++++++ hw/remote/message.c | 62 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 6 ++++ hw/remote/proxy.c | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+)
include/hw/remote/mpqemu-link.h +9 -0
@@ -34,6 +34,8 @@ typedef enum {MPQEMU_CMD_INIT ,SYNC_SYSMEM ,RET_MSG , +PCI_CONFIG_WRITE , +PCI_CONFIG_READ ,MPQEMU_CMD_MAX , } MPQemuCmd;
@@ -43,6 +45,12 @@ typedef struct {off_t offsets[REMOTE_MAX_FDS ]; } SyncSysmemMsg; +typedef struct { +uint32_t addr; +uint32_t val; +int l; +}PciConfDataMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -60,6 +68,7 @@ typedef struct {union {uint64_t u64; +PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; } data;
hw/remote/message.c +62 -0
77 lines not on this page · raw message
hw/remote/mpqemu-link.c +6 -0
@@ -283,6 +283,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case PCI_CONFIG_WRITE : +case PCI_CONFIG_READ : +if (msg->size !=sizeof (PciConfDataMsg)) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +51 -0
69 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v12 8/19 multi-process: define MPQemuMsg format and transmission functions
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks.
Define transmission functions used by proxy and by remote.
There are certain restrictions on where its safe to use these functions:
- From main loop in co-routine context. Will block the main loop if not in
co-routine context;
- From vCPU thread with no co-routine context and if the channel is not part
of the main loop handling;
- From IOThread within co-routine context, outside of co-routine context will
block IOThread;--- include/hw/remote/mpqemu-link.h | 60 ++++++++++ hw/remote/mpqemu-link.c | 242 ++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 305 insertions(+) create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.c
include/hw/remote/mpqemu-link.h new file +60 -0
@@ -0,0 +1,60 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef MPQEMU_LINK_H +#define MPQEMU_LINK_H + +#include "qom/object.h" +#include "qemu/thread.h" +#include "io/channel.h" + +#define REMOTE_MAX_FDS 8 + +#define MPQEMU_MSG_HDR_SIZE offsetof(MPQemuMsg, data.u64) + +/** +* MPQemuCmd: +* +* MPQemuCmd enum type to specify the command to be executed on the remote +* device. +*/ +typedef enum { +MPQEMU_CMD_INIT , +MPQEMU_CMD_MAX , +}MPQemuCmd ; + +/** +* MPQemuMsg: +* @cmd: The remote command +* @size: Size of the data to be shared +* @data: Structured data +* @fds: File descriptors to be shared with remote device +* +* MPQemuMsg Format of the message sent to the remote device from QEMU. +* +*/ +typedef struct { +int cmd; +size_t size; + +union { +uint64_t u64; + } data; + +int fds[REMOTE_MAX_FDS ]; +int num_fds; +}MPQemuMsg ; + +void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); + +bool mpqemu_msg_valid (MPQemuMsg *msg); + +#endif
hw/remote/mpqemu-link.c new file +242 -0
242 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3141,6 +3141,8 @@ F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h F: hw/remote/machine.c F: include/hw/remote/machine.h +F: hw/remote/mpqemu-link.c +F: include/hw/remote/mpqemu-link.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -1,5 +1,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
Quoted 2 lines · Jagannathan Raman
This is the v12 of the patchset. Thank you very much for the review of the v11 of the series.
I'm in favor of merging this for QEMU 6.0. The command-line interface has the x- prefix so QEMU is not committing to a stable interface. Changes needed to support additional device types or to switch to the vfio-user protocol can be made later.
Jag, Elena, JJ: I suggest getting your GPG key to Peter Maydell so you can send multi-process QEMU pull requests.
Quoted 9 lines · Stefan Hajnoczi
Quoted 2 lines · Jagannathan Raman
This is the v12 of the patchset. Thank you very much for the review of the v11 of the series.
I'm in favor of merging this for QEMU 6.0. The command-line interface has the x- prefix so QEMU is not committing to a stable interface. Changes needed to support additional device types or to switch to the vfio-user protocol can be made later.
Woot! Thank you Stefan!
Quoted 4 lines
Jag, Elena, JJ: I suggest getting your GPG key to Peter Maydell so you can send multi-process QEMU pull requests.
Reviewed-by Stefan Hajnoczi <[email protected]>
In progress.
Do we need to add some tagging for the PULL patches? Should we include the git repo and have the proper tag as well?
Elena
Quoted 5 lines · Stefan Hajnoczi
Quoted 2 lines · Jagannathan Raman
This is the v12 of the patchset. Thank you very much for the review of the v11 of the series.
Quoted 7 lines
I'm in favor of merging this for QEMU 6.0. The command-line interface has the x- prefix so QEMU is not committing to a stable interface. Changes needed to support additional device types or to switch to the vfio-user protocol can be made later.
Jag, Elena, JJ: I suggest getting your GPG key to Peter Maydell so you can send multi-process QEMU pull requests.
I would prefer to see this going through the tree of an established QEMU developer who's already sending pullrequests, at least initially.
thanks
-- PMM
Re: [PATCH v12 02/19] multi-process: add configure and usage information
In reply to Jagannathan RamanHi
tests/multiprocess/multiprocess-lsi53c895a.py
Quoted 113 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds documentation explaining the command-line arguments needed to use multi-process. Also adds a python script that illustrates the usage.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- docs/multi-process.rst | 66 +++++++++++++++++++ MAINTAINERS | 1 + tests/multiprocess/multiprocess-lsi53c895a.py | 92 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 docs/multi-process.rst create mode 100755 tests/multiprocess/multiprocess-lsi53c895a.pydocs/multi-process.rst new file +59
-0@@ -0,0 +1,66 @@ +Multi-process QEMU +================== + +This document describes how to configure and use multi-process qemu. +For the design document refer to docs/devel/qemu-multiprocess. + +1) Configuration +---------------- + +multi-process is enabled by default for targets that enable KVM + + +2) Usage +-------- + +Multi-process QEMU requires an orchestrator to launch. Please refer to a +light-weight python based orchestrator for mpqemu in +scripts/mpqemu-launcher.py to lauch QEMU in multi-process mode. + +Following is a description of command-line used to launch mpqemu. + +* Orchestrator: + + - The Orchestrator creates a unix socketpair + + - It launches the remote process and passes one of the + sockets to it via command-line. + + - It then launches QEMU and specifies the other socket as an option + to the Proxy device object + +* Remote Process: + + - QEMU can enter remote process mode by using the "remote" machine + option. + + - The orchestrator creates a "remote-object" with details about + the device and the file descriptor for the device + + - The remaining options are no different from how one launches QEMU with + devices. + + - Example command-line for the remote process is as follows: + + /usr/bin/qemu-system-x86_64 \ + -machine x-remote \ + -device lsi53c895a,id=lsi0 \ + -drive id=drive_image2,file=/build/ol7-nvme-test-1.qcow2 \ + -device scsi-hd,id=drive2,drive=drive_image2,bus=lsi0.0,scsi-id=0 \ + -object x-remote-object,id=robj1,devid=lsi1,fd=4, + +* QEMU: + + - Since parts of the RAM are shared between QEMU & remote process, a + memory-backend-memfd is required to facilitate this, as follows: + + -object memory-backend-memfd,id=mem,size=2G + + - A "x-pci-proxy-dev" device is created for each of the PCI devicesemulated
+ in the remote process. A "socket" sub-option specifies the other end
of
+ unix channel created by orchestrator. The "id" sub-option must be
specifieddocs/multi-process.rst new file +5
-0+ and should be the same as the "id" specified for the remote PCI device + + - Example commandline for QEMU is as follows: + + -device x-pci-proxy-dev,id=lsi0,socket=3MAINTAINERS +1
-0@@ -3136,6 +3136,7 @@ M: Jagannathan Raman <[email protected]> M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst +F: tests/multiprocess/multiprocess-lsi53c895a.py Build and test automation -------------------------tests/multiprocess/multiprocess-lsi53c895a.py new file +0
-0
This might not be appropriate under qemu tree tests/ imho. It's not a test, at best it complements the documentation.
@@ -0,0 +1,92 @@
Quoted 35 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +11
-0+#!/usr/bin/env python3 + +import urllib.request +import subprocess +import argparse +import socket +import sys +import os + +arch = os.uname()[4] +proc_path = os.path.join(os.getcwd(), '..', '..', 'build',arch+'-softmmu',
tests/multiprocess/multiprocess-lsi53c895a.py new file +3
-0+ 'qemu-system-'+arch) + +parser = argparse.ArgumentParser(description='Launcher for multi-processQEMU')
+parser.add_argument('--bin', required=False, help='location of QEMU binary',tests/multiprocess/multiprocess-lsi53c895a.py new file +14
-0+ metavar='bin'); +args = parser.parse_args() + +if args.bin is not None: + proc_path = args.bin + +if not os.path.isfile(proc_path): + sys.exit('QEMU binary not found') + +kernel_path = os.path.join(os.getcwd(), 'vmlinuz') +initrd_path = os.path.join(os.getcwd(), 'initrd') + +proxy_cmd = [ proc_path, \ + '-name', 'Fedora', '-smp', '4', '-m', '2048', '-cpu','host', \
tests/multiprocess/multiprocess-lsi53c895a.py
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +5
-0+ '-object', 'memory-backend-memfd,id=sysmem-file,size=2G', \ + '-numa', 'node,memdev=sysmem-file', \ + '-kernel', kernel_path, '-initrd', initrd_path, \ + '-vnc', ':0', \ + '-monitor', 'unix:/home/qemu-sock,server,nowait', \
That path is odd. Make it a TemporaryFile, or an argument. Even simpler, use socketpair()
+ ]
Quoted 18 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +17
-0+ +if arch == 'x86_64': + print('Downloading images for arch x86_64') + kernel_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/x86_64/os/images/' \ + 'pxeboot/vmlinuz' + initrd_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/x86_64/os/images/' \ + 'pxeboot/initrd.img' + proxy_cmd.append('-machine') + proxy_cmd.append('pc,accel=kvm') + proxy_cmd.append('-append') + proxy_cmd.append('rdinit=/bin/bash console=ttyS0 console=tty0') +elif arch == 'aarch64': + print('Downloading images for arch aarch64') + kernel_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/aarch64/os/images/' \
tests/multiprocess/multiprocess-lsi53c895a.py line 62
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+ 'pxeboot/vmlinuz' + initrd_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/aarch64/os/images/' \ + 'pxeboot/initrd.img' + proxy_cmd.append('-machine') + proxy_cmd.append('virt,gic-version=3') + proxy_cmd.append('-accel') + proxy_cmd.append('kvm') + proxy_cmd.append('-append') + proxy_cmd.append('rdinit=/bin/bash')
We have vm-based testing under tests/vm, can we imagine extending that instead?
To not delay further the series, I would suggest to drop it for now.
+else:
Quoted 41 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +29
-0+ sys.exit('Arch %s not tested' % arch) + +urllib.request.urlretrieve(kernel_url, kernel_path) +urllib.request.urlretrieve(initrd_url, initrd_path) + +proxy, remote = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + +proxy_cmd.append('-device') +proxy_cmd.append('x-pci-proxy-dev,id=lsi1,fd='+str(proxy.fileno())) + +remote_cmd = [ proc_path, \ + '-machine', 'x-remote', \ + '-device', 'lsi53c895a,id=lsi1', \ + '-object', \ + 'x-remote-object,id=robj1,devid=lsi1,fd='+str(remote.fileno()), \ + '-display', 'none', \ + '-monitor', 'unix:/home/rem-sock,server,nowait', \ + ] + +pid = os.fork(); + +if pid: + # In Proxy + print('Launching QEMU with Proxy object'); + process = subprocess.Popen(proxy_cmd, pass_fds=[proxy.fileno()]) +else: + # In remote + print('Launching Remote process'); + process = subprocess.Popen(remote_cmd, pass_fds=[remote.fileno(), 0,1, 2])
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 03/19] memory: alloc RAM from file at offset
In reply to Jagannathan RamanQuoted 59 lines · Jagannathan Raman
Allow RAM MemoryRegion to be created from an offset in a file, instead of allocating at offset of 0 by default. This is needed to synchronize RAM between QEMU & remote process.
Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/exec/memory.h | 2 ++ include/exec/ram_addr.h | 2 +- include/qemu/mmap-alloc.h | 3 ++- backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 ++- softmmu/memory.c | 3 ++- softmmu/physmem.c | 11 +++++++---- util/mmap-alloc.c | 7 ++++--- util/oslib-posix.c | 2 +- 9 files changed, 22 insertions(+), 13 deletions(-)include/exec/memory.h +2
-0@@ -980,6 +980,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr,* @size: size of the region. * @share: %true if memory must be mmaped with the MAP_SHARED flag * @fd: the fd to mmap. +* @offset: offset within the file referenced by fd * @errp: pointer to Error*, to store an error if it happens. * * Note that this function does not do anything to cause the data in the @@ -991,6 +992,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp);#endif include/exec/ram_addr.h +1
-1@@ -121,7 +121,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, Error **errp);RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp); +off_t offset,Error **errp); RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,MemoryRegion *mr, Error **errp);include/qemu/mmap-alloc.h +0
-0
include/qemu/mmap-alloc.h void *qemu_ram_mmap(int fd, · line 30
Quoted 7 lines
include/qemu/mmap-alloc.h +2
-1@@ -25,7 +25,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem); + bool is_pmem, + off_t start);
I'd suggest to keep the variable name "offset", to avoid potential confusion (it's also the name of the mmap() argument).
Quoted 160 lines
include/qemu/mmap-alloc.h +0
-0void qemu_ram_munmap (int fd,void *ptr,size_t size);backends/hostmem-memfd.c +1
-1@@ -55,7 +55,7 @@ memfd_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) name =host_memory_backend_get_name (backend);memory_region_init_ram_from_fd (&backend->mr,OBJECT (backend), name, backend->size, - backend->share, fd, errp); + backend->share, fd,0 , errp);g_free (name); }hw/misc/ivshmem.c +2
-1@@ -494,7 +494,8 @@ static void process_msg_shmem(IVShmemState *s, int fd, Error **errp) /* mmap the region and map into the BAR2 */ memory_region_init_ram_from_fd(&s->server_bar2, OBJECT(s), -"ivshmem.bar2" , size, true, fd, &local_err); +"ivshmem.bar2" , size, true, fd,0 , + &local_err);if (local_err) {error_propagate(errp, local_err); return;softmmu/memory.c +2
-1@@ -1612,6 +1612,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp) { Error *err = NULL;@@ -1621,7 +1622,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->ram_block =qemu_ram_alloc_from_fd (size, mr, share ?RAM_SHARED :0 , - fd, &err); + fd, offset, &err); mr->dirty_log_mask =tcg_enabled () ? (1 <<DIRTY_MEMORY_CODE ) :0 ;if (err) { mr->size =int128_zero ();softmmu/physmem.c +3
-1@@ -1461,6 +1461,7 @@ static void *file_ram_alloc(RAMBlock *block,ram_addr_t memory, int fd, bool truncate, + off_t offset, Error **errp) { void *area;@@ -1511,7 +1512,8 @@ static void *file_ram_alloc(RAMBlock *block, } area =qemu_ram_mmap (fd, memory, block->mr->align, - block->flags &RAM_SHARED , block->flags &RAM_PMEM ); + block->flags &RAM_SHARED , block->flags &RAM_PMEM , + offset);if (area ==MAP_FAILED ) {error_setg_errno(errp, errno, "unable to map backing store for guest RAM");softmmu/physmem.c +1
-1@@ -1943,7 +1945,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)#ifdef CONFIG_POSIX RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp) +off_t offset,Error **errp) {RAMBlock *new_block;Error *local_err = NULL;softmmu/physmem.c +2
-1@@ -1996,7 +1998,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, new_block->used_length = size; new_block->max_length = size; new_block->flags = ram_flags; - new_block->host =file_ram_alloc (new_block, size, fd, !file_size, errp); + new_block->host =file_ram_alloc (new_block, size, fd, !file_size, offset, + errp);if (!new_block->host) { g_free(new_block); return NULL;softmmu/physmem.c +1
-1@@ -2026,7 +2029,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size,MemoryRegion *mr, return NULL; } - block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd, errp); + block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd,0 , errp);if (!block) {if (created) {unlink(mem_path);util/mmap-alloc.c +3
-2@@ -86,7 +86,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem) + bool is_pmem, + off_t start) { int flags;int map_sync_flags =0 ;@@ -147,7 +148,7 @@ void *qemu_ram_mmap(int fd, offset =QEMU_ALIGN_UP ((uintptr_t )guardptr, align) - (uintptr_t )guardptr; ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags | map_sync_flags, fd,0 ); + flags | map_sync_flags, fd, start);if (ptr ==MAP_FAILED && map_sync_flags) {if (errno == ENOTSUP) {util/mmap-alloc.c +1
-1@@ -172,7 +173,7 @@ void *qemu_ram_mmap(int fd,* we will remove these flags to handle compatibility. */ ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags, fd,0 ); + flags, fd, start); }if (ptr ==MAP_FAILED ) {util/oslib-posix.c +1
-2@@ -227,7 +227,7 @@ void *qemu_memalign(size_t alignment, size_t size)void *qemu_anon_ram_alloc (size_t size,uint64_t *alignment,bool shared) {size_t align =QEMU_VMALLOC_ALIGN ; -void *ptr =qemu_ram_mmap (-1 , size, align, shared, false); +void *ptr =qemu_ram_mmap (-1 , size, align, shared, false,0 );if (ptr ==MAP_FAILED ) {return NULL ; --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 03/19] memory: alloc RAM from file at offset
Quoted 67 lines · Marc-André Lureau
Quoted 62 lines · Jagannathan Raman
Allow RAM MemoryRegion to be created from an offset in a file, instead of allocating at offset of 0 by default. This is needed to synchronize RAM between QEMU & remote process.
Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/exec/memory.h | 2 ++ include/exec/ram_addr.h | 2 +- include/qemu/mmap-alloc.h | 3 ++- backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 ++- softmmu/memory.c | 3 ++- softmmu/physmem.c | 11 +++++++---- util/mmap-alloc.c | 7 ++++--- util/oslib-posix.c | 2 +- 9 files changed, 22 insertions(+), 13 deletions(-)include/exec/memory.h +1
-0@@ -980,6 +980,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr, * @size: size of the region. * @share: %true if memory must be mmaped with the MAP_SHARED flag * @fd: the fd to mmap. + * @offset: offset within the file referenced by fd * @errp: pointer to Error*, to store an error if it happens. ** Note that this function does not do anything to cause the data in the
include/exec/memory.h +1
-0@@ -991,6 +992,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp);#endif include/exec/ram_addr.h +1
-1@@ -121,7 +121,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, Error **errp);RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp); +off_t offset,Error **errp); RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,MemoryRegion *mr, Error **errp);include/qemu/mmap-alloc.h +0
-0@@ -25,7 +25,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align,
Quoted 9 lines
include/qemu/mmap-alloc.h void *qemu_ram_mmap(int fd, · line 30
Quoted 5 lines
include/qemu/mmap-alloc.h +2
-1bool shared, - bool is_pmem); + bool is_pmem, + off_t start);I'd suggest to keep the variable name "offset", to avoid potential confusion (it's also the name of the
mmap()argument).
I realize the inner offset variable will need to be renamed then, I'd suggest guard_offset for instance.
Quoted 167 lines
Quoted 160 lines
include/qemu/mmap-alloc.h +0
-0void qemu_ram_munmap (int fd,void *ptr,size_t size);backends/hostmem-memfd.c +1
-1@@ -55,7 +55,7 @@ memfd_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) name =host_memory_backend_get_name (backend);memory_region_init_ram_from_fd (&backend->mr,OBJECT (backend), name, backend->size, - backend->share, fd, errp); + backend->share, fd,0 , errp);g_free (name); }hw/misc/ivshmem.c +2
-1@@ -494,7 +494,8 @@ static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)/* mmap the region and map into the BAR2 */ memory_region_init_ram_from_fd (&s->server_bar2,OBJECT (s), -"ivshmem.bar2" , size, true, fd, &local_err); +"ivshmem.bar2" , size, true, fd,0 , + &local_err);if (local_err) {error_propagate(errp, local_err); return;softmmu/memory.c +1
-0@@ -1612,6 +1612,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp) {Error *err = NULL;softmmu/memory.c +1
-1@@ -1621,7 +1622,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->ram_block =qemu_ram_alloc_from_fd (size, mr, share ?RAM_SHARED :0 , - fd, &err); + fd, offset, &err); mr->dirty_log_mask =tcg_enabled () ? (1 <<DIRTY_MEMORY_CODE ) :0 ;if (err) {mr->size = int128_zero();softmmu/physmem.c +3
-1@@ -1461,6 +1461,7 @@ static void *file_ram_alloc(RAMBlock *block,ram_addr_t memory, int fd, bool truncate, + off_t offset, Error **errp) { void *area;@@ -1511,7 +1512,8 @@ static void *file_ram_alloc(RAMBlock *block, } area =qemu_ram_mmap (fd, memory, block->mr->align, - block->flags &RAM_SHARED , block->flags &RAM_PMEM ); + block->flags &RAM_SHARED , block->flags &RAM_PMEM , + offset);if (area ==MAP_FAILED ) {error_setg_errno(errp, errno, "unable to map backing store for guest RAM");softmmu/physmem.c +1
-1@@ -1943,7 +1945,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)#ifdef CONFIG_POSIX RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp) +off_t offset,Error **errp) {RAMBlock *new_block;Error *local_err = NULL;softmmu/physmem.c +2
-1@@ -1996,7 +1998,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, new_block->used_length = size; new_block->max_length = size; new_block->flags = ram_flags; - new_block->host =file_ram_alloc (new_block, size, fd, !file_size, errp); + new_block->host =file_ram_alloc (new_block, size, fd, !file_size, offset, + errp);if (!new_block->host) { g_free(new_block); return NULL;softmmu/physmem.c +1
-1@@ -2026,7 +2029,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size,MemoryRegion *mr, return NULL; } - block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd, errp); + block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd,0 , errp);if (!block) {if (created) {unlink(mem_path);util/mmap-alloc.c +3
-2@@ -86,7 +86,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem) + bool is_pmem, + off_t start) { int flags;int map_sync_flags =0 ;@@ -147,7 +148,7 @@ void *qemu_ram_mmap(int fd, offset =QEMU_ALIGN_UP ((uintptr_t )guardptr, align) - (uintptr_t )guardptr; ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags | map_sync_flags, fd,0 ); + flags | map_sync_flags, fd, start);if (ptr ==MAP_FAILED && map_sync_flags) {if (errno == ENOTSUP) {util/mmap-alloc.c +1
-1@@ -172,7 +173,7 @@ void *qemu_ram_mmap(int fd,* we will remove these flags to handle compatibility. */ ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags, fd,0 ); + flags, fd, start); }if (ptr ==MAP_FAILED ) {util/oslib-posix.c +1
-2@@ -227,7 +227,7 @@ void *qemu_memalign(size_t alignment, size_t size)void *qemu_anon_ram_alloc (size_t size,uint64_t *alignment,bool shared) {size_t align =QEMU_VMALLOC_ALIGN ; -void *ptr =qemu_ram_mmap (-1 , size, align, shared, false); +void *ptr =qemu_ram_mmap (-1 , size, align, shared, false,0 );if (ptr ==MAP_FAILED ) {return NULL ; --Quoted 3 lines
1.8.3.1
--
Marc-André Lureau
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 05/19] multi-process: setup PCI host bridge for remote device
In reply to Jagannathan RamanQuoted 160 lines · Jagannathan Raman
PCI host bridge is setup for the remote device process. It is implemented using remote-pcihost object. It is an extension of the PCI host bridge setup by QEMU.
Remote-pcihost configures a PCI bus which could be used by the remote PCI device to latch on to.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/pci-host/remote.h | 30 ++++++++++++++++++ hw/pci-host/remote.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/pci-host/Kconfig | 3 ++ hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 1 + 6 files changed, 112 insertions(+) create mode 100644 include/hw/pci-host/remote.h create mode 100644 hw/pci-host/remote.cinclude/hw/pci-host/remote.h new file +6
-0@@ -0,0 +1,30 @@ +/* +* PCI Host for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/pci-host/remote.h new file +21
-0+ +#ifndef REMOTE_PCIHOST_H +#define REMOTE_PCIHOST_H + +#include "exec/memory.h" +#include "hw/pci/pcie_host.h" + +#define TYPE_REMOTE_HOST_DEVICE "remote-pcihost" +#define REMOTE_HOST_DEVICE (obj) \ + OBJECT_CHECK(RemotePCIHost, (obj), TYPE_REMOTE_HOST_DEVICE) + +typedef struct RemotePCIHost { +/*< private >*/ +PCIExpressHost parent_obj; +/*< public >*/ + +MemoryRegion *mr_pci_mem; +MemoryRegion *mr_sys_io; +}RemotePCIHost ; + +#endif hw/pci-host/remote.c new file +19
-0@@ -0,0 +1,75 @@ +/* +* Remote PCI host device +* +* Unlike PCI host devices that model physical hardware, the purpose +* of this PCI host is to host multi-process QEMU devices. +* +* Multi-process QEMU extends the PCI host of a QEMU machine into a +* remote process. Any PCI device attached to the remote process is +* visible in the QEMU guest. This allows existing QEMU device models +* to be reused in the remote process. +* +* This PCI host is purely a container for PCI devices. It's fake in the +* sense that the guest never sees this PCI host and has no way of +* accessing it. Its job is just to provide the environment that QEMU +* PCI device models need when running in a remote process. +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/pci-host/remote.c new file +53
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/pci/pci.h" +#include "hw/pci/pci_host.h" +#include "hw/pci/pcie_host.h" +#include "hw/qdev-properties.h" +#include "hw/pci-host/remote.h" +#include "exec/memory.h" + +static const char *remote_pcihost_root_bus_path (PCIHostState *host_bridge, +PCIBus *rootbus) +{ +return "0000:00" ; +} + +static void remote_pcihost_realize (DeviceState *dev,Error **errp) +{ +PCIHostState *pci =PCI_HOST_BRIDGE (dev); +RemotePCIHost *s =REMOTE_HOST_DEVICE (dev); + + pci->bus =pci_root_bus_new (DEVICE (s),"remote-pci" , + s->mr_pci_mem, s->mr_sys_io, +0 ,TYPE_PCIE_BUS ); +} + +static void remote_pcihost_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +PCIHostBridgeClass *hc =PCI_HOST_BRIDGE_CLASS (klass); + + hc->root_bus_path = remote_pcihost_root_bus_path; + dc->realize = remote_pcihost_realize; + + dc->user_creatable = false; +set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); + dc->fw_name ="pci" ; +} + +static const TypeInfo remote_pcihost_info = { + .name =TYPE_REMOTE_HOST_DEVICE , + .parent =TYPE_PCIE_HOST_BRIDGE , + .instance_size =sizeof (RemotePCIHost), + .class_init = remote_pcihost_class_init, +}; + +static void remote_pcihost_register (void ) +{ +type_register_static (&remote_pcihost_info); +} + +type_init (remote_pcihost_register)MAINTAINERS +2
-0@@ -3137,6 +3137,8 @@ M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst F: tests/multiprocess/multiprocess-lsi53c895a.py +F: hw/pci-host/remote.c +F: include/hw/pci-host/remote.h Build and test automation -------------------------hw/pci-host/Kconfig +0
-0
hw/pci-host/Kconfig config PCI_BONITO · line 65
Quoted 6 lines
hw/pci-host/Kconfig +3
-0@@ -60,3 +60,6 @@ config PCI_BONITO select PCI select UNIMP bool + +config MULTIPROCESS_HOST + bool
Why not REMOTE_PCIHOST ?
hw/pci-host/meson.build +0 -0
Quoted 26 lines
index e6d1b89..4147100 100644
hw/pci-host/meson.build +1
-0@@ -9,6 +9,7 @@ pci_ss.add(when: 'CONFIG_PCI_EXPRESS_XILINX', if_true: files('xilinx-pcie.c')) pci_ss.add(when: 'CONFIG_PCI_I440FX', if_true: files('i440fx.c')) pci_ss.add(when: 'CONFIG_PCI_SABRE', if_true: files('sabre.c')) pci_ss.add(when: 'CONFIG_XEN_IGD_PASSTHROUGH', if_true: files('xen_igd_pt.c')) +pci_ss.add(when: 'CONFIG_MULTIPROCESS_HOST', if_true: files('remote.c'))# PPC devices
pci_ss.add(when: 'CONFIG_PREP_PCI', if_true: files('prep.c'))hw/remote/Kconfig +1
-1@@ -1,3 +1,4 @@ config MULTIPROCESS bool depends on PCI && KVM + select MULTIPROCESS_HOST --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 05/19] multi-process: setup PCI host bridge for remote device
In reply to Jagannathan RamanQuoted 39 lines · Jagannathan Raman
PCI host bridge is setup for the remote device process. It is implemented using remote-pcihost object. It is an extension of the PCI host bridge setup by QEMU.
Remote-pcihost configures a PCI bus which could be used by the remote PCI device to latch on to.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/pci-host/remote.h | 30 ++++++++++++++++++ hw/pci-host/remote.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/pci-host/Kconfig | 3 ++ hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 1 + 6 files changed, 112 insertions(+) create mode 100644 include/hw/pci-host/remote.h create mode 100644 hw/pci-host/remote.cinclude/hw/pci-host/remote.h new file +6
-0@@ -0,0 +1,30 @@ +/* +* PCI Host for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
include/hw/pci-host/remote.h line 17
Quoted 8 lines
include/hw/pci-host/remote.h new file +8
-0+ +#ifndef REMOTE_PCIHOST_H +#define REMOTE_PCIHOST_H + +#include "exec/memory.h" +#include "hw/pci/pcie_host.h" + +#define TYPE_REMOTE_HOST_DEVICE "remote-pcihost"
I got confused by the following patch. Please try to be consistent.
TYPE_REMOTE_PCIHOST would help.
thanks
+#define REMOTE_HOST_DEVICE(obj) \
Quoted 2 lines
+
OBJECT_CHECK(RemotePCIHost, (obj), TYPE_REMOTE_HOST_DEVICE)
REMOTE_PCIHOST
+
Quoted 143 lines
include/hw/pci-host/remote.h new file +10
-0+typedef struct RemotePCIHost { +/*< private >*/ +PCIExpressHost parent_obj; +/*< public >*/ + +MemoryRegion *mr_pci_mem; +MemoryRegion *mr_sys_io; +}RemotePCIHost ; + +#endif hw/pci-host/remote.c new file +19
-0@@ -0,0 +1,75 @@ +/* +* Remote PCI host device +* +* Unlike PCI host devices that model physical hardware, the purpose +* of this PCI host is to host multi-process QEMU devices. +* +* Multi-process QEMU extends the PCI host of a QEMU machine into a +* remote process. Any PCI device attached to the remote process is +* visible in the QEMU guest. This allows existing QEMU device models +* to be reused in the remote process. +* +* This PCI host is purely a container for PCI devices. It's fake in the +* sense that the guest never sees this PCI host and has no way of +* accessing it. Its job is just to provide the environment that QEMU +* PCI device models need when running in a remote process. +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/pci-host/remote.c new file +53
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/pci/pci.h" +#include "hw/pci/pci_host.h" +#include "hw/pci/pcie_host.h" +#include "hw/qdev-properties.h" +#include "hw/pci-host/remote.h" +#include "exec/memory.h" + +static const char *remote_pcihost_root_bus_path (PCIHostState *host_bridge, +PCIBus *rootbus) +{ +return "0000:00" ; +} + +static void remote_pcihost_realize (DeviceState *dev,Error **errp) +{ +PCIHostState *pci =PCI_HOST_BRIDGE (dev); +RemotePCIHost *s =REMOTE_HOST_DEVICE (dev); + + pci->bus =pci_root_bus_new (DEVICE (s),"remote-pci" , + s->mr_pci_mem, s->mr_sys_io, +0 ,TYPE_PCIE_BUS ); +} + +static void remote_pcihost_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +PCIHostBridgeClass *hc =PCI_HOST_BRIDGE_CLASS (klass); + + hc->root_bus_path = remote_pcihost_root_bus_path; + dc->realize = remote_pcihost_realize; + + dc->user_creatable = false; +set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); + dc->fw_name ="pci" ; +} + +static const TypeInfo remote_pcihost_info = { + .name =TYPE_REMOTE_HOST_DEVICE , + .parent =TYPE_PCIE_HOST_BRIDGE , + .instance_size =sizeof (RemotePCIHost), + .class_init = remote_pcihost_class_init, +}; + +static void remote_pcihost_register (void ) +{ +type_register_static (&remote_pcihost_info); +} + +type_init (remote_pcihost_register)MAINTAINERS +2
-0@@ -3137,6 +3137,8 @@ M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst F: tests/multiprocess/multiprocess-lsi53c895a.py +F: hw/pci-host/remote.c +F: include/hw/pci-host/remote.h Build and test automation -------------------------hw/pci-host/Kconfig +3
-0@@ -60,3 +60,6 @@ config PCI_BONITO select PCI select UNIMP bool + +config MULTIPROCESS_HOST + boolhw/pci-host/meson.build +1
-0@@ -9,6 +9,7 @@ pci_ss.add(when: 'CONFIG_PCI_EXPRESS_XILINX', if_true: files('xilinx-pcie.c')) pci_ss.add(when: 'CONFIG_PCI_I440FX', if_true: files('i440fx.c')) pci_ss.add(when: 'CONFIG_PCI_SABRE', if_true: files('sabre.c')) pci_ss.add(when: 'CONFIG_XEN_IGD_PASSTHROUGH', if_true: files('xen_igd_pt.c')) +pci_ss.add(when: 'CONFIG_MULTIPROCESS_HOST', if_true: files('remote.c'))# PPC devices
pci_ss.add(when: 'CONFIG_PREP_PCI', if_true: files('prep.c'))hw/remote/Kconfig +1
-1@@ -1,3 +1,4 @@ config MULTIPROCESS bool depends on PCI && KVM + select MULTIPROCESS_HOST --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 06/19] multi-process: setup a machine object for remote device process
In reply to Jagannathan RamanQuoted 26 lines · Jagannathan Raman
x-remote-machine object sets up various subsystems of the remote device process. Instantiate PCI host bridge object and initialize RAM, IO & PCI memory regions.
Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/pci-host/remote.h | 1 + include/hw/remote/machine.h | 28 ++++++++++++++++++ hw/remote/machine.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/meson.build | 1 + hw/remote/meson.build | 5 ++++ 6 files changed, 106 insertions(+) create mode 100644 include/hw/remote/machine.h create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/meson.buildinclude/hw/pci-host/remote.h +0
-0
include/hw/pci-host/remote.h typedef struct RemotePCIHost { · line 29
Quoted 5 lines
include/hw/pci-host/remote.h +1
-0@@ -25,6 +25,7 @@ typedef struct RemotePCIHost {MemoryRegion *mr_pci_mem;MemoryRegion *mr_sys_io; +MemoryRegion *mr_sys_mem;
Why is this not part of the previous patch?
} RemotePCIHost;
Quoted 91 lines
include/hw/pci-host/remote.h +0
-0#endif include/hw/remote/machine.h new file +6
-0@@ -0,0 +1,28 @@ +/* +* Remote machine configuration +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/machine.h new file +19
-0+ +#ifndef REMOTE_MACHINE_H +#define REMOTE_MACHINE_H + +#include "qom/object.h" +#include "hw/boards.h" +#include "hw/pci-host/remote.h" + +typedef struct RemoteMachineState { +MachineState parent_obj; + +RemotePCIHost *host; +}RemoteMachineState ; + +#define TYPE_REMOTE_MACHINE "x-remote-machine" +#define REMOTE_MACHINE (obj) \ + OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) + +#endif hw/remote/machine.c new file +4
-0@@ -0,0 +1,69 @@ +/* +* Machine for remote device +* +* This machine type is used by the remote device process in multi-process
+ * QEMU. QEMU device models depend on parent busses, interrupt controllers,
+ * memory regions, etc. The remote machine type offers this environment so
+ * that QEMU device models can be used as remote devices.
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/machine.c new file +30
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/machine.h" +#include "exec/address-spaces.h" +#include "exec/memory.h" +#include "qapi/error.h" + +static void remote_machine_init (MachineState *machine) +{ +MemoryRegion *system_memory, *system_io, *pci_memory; +RemoteMachineState *s =REMOTE_MACHINE (machine); +RemotePCIHost *rem_host; + + system_memory =get_system_memory (); + system_io =get_system_io (); + + pci_memory =g_new (MemoryRegion,1 ); +memory_region_init (pci_memory,NULL ,"pci" ,UINT64_MAX ); + + rem_host =REMOTE_HOST_DEVICE (qdev_new (TYPE_REMOTE_HOST_DEVICE )); + + rem_host->mr_pci_mem = pci_memory; + rem_host->mr_sys_mem = system_memory; + rem_host->mr_sys_io = system_io; + + s->host = rem_host; + + object_property_add_child(OBJECT(s), "remote-device",
Quoted 2 lines
OBJECT(rem_host));
"remote-pcihost" instead ?
+ memory_region_add_subregion_overlap(system_memory, 0x0, pci_memory,
hw/remote/machine.c line 54
Quoted 11 lines
-1);
hw/remote/machine.c new file +9
-0+ +qdev_realize (DEVICE (rem_host),sysbus_get_default (), &error_fatal); +} + +static void remote_machine_class_init (ObjectClass *oc,void *data) +{ +MachineClass *mc =MACHINE_CLASS (oc); + + mc->init = remote_machine_init;
Set mc->desc = "Experimental remote machine" ?
+}
Quoted 54 lines
hw/remote/machine.c new file +14
-0+ +static const TypeInfo remote_machine = { + .name =TYPE_REMOTE_MACHINE , + .parent =TYPE_MACHINE , + .instance_size =sizeof (RemoteMachineState), + .class_init = remote_machine_class_init, +}; + +static void remote_machine_register_types (void ) +{ +type_register_static (&remote_machine); +} + +type_init (remote_machine_register_types);MAINTAINERS +2
-0@@ -3139,6 +3139,8 @@ F: docs/devel/multi-process.rst F: tests/multiprocess/multiprocess-lsi53c895a.py F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h +F: hw/remote/machine.c +F: include/hw/remote/machine.h Build and test automation -------------------------hw/meson.build +1
-0@@ -56,6 +56,7 @@ subdir('moxie') subdir('nios2') subdir('openrisc') subdir('ppc') +subdir('remote') subdir('riscv') subdir('rx') subdir('s390x')hw/remote/meson.build new file +5
-1@@ -0,0 +1,5 @@ +remote_ss = ss.source_set() + +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) + +softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss) --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 02/19] multi-process: add configure and usage information
In reply to Jagannathan RamanQuoted 44 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds documentation explaining the command-line arguments needed to use multi-process. Also adds a python script that illustrates the usage.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- docs/multi-process.rst | 66 +++++++++++++++++++ MAINTAINERS | 1 + tests/multiprocess/multiprocess-lsi53c895a.py | 92 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 docs/multi-process.rst create mode 100755 tests/multiprocess/multiprocess-lsi53c895a.pytests/multiprocess/multiprocess-lsi53c895a.py new file +19
-0@@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +import urllib.request +import subprocess +import argparse +import socket +import sys +import os + +arch = os.uname()[4] +proc_path = os.path.join(os.getcwd(), '..', '..', 'build', arch+'-softmmu', + 'qemu-system-'+arch) + +parser = argparse.ArgumentParser(description='Launcher for multi-process QEMU') +parser.add_argument('--bin', required=False, help='location of QEMU binary', + metavar='bin'); +args = parser.parse_args() + +if args.bin is not None:
tests/multiprocess/multiprocess-lsi53c895a.py line 29
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+ proc_path = args.bin + +if not os.path.isfile(proc_path): + sys.exit('QEMU binary not found') + +kernel_path = os.path.join(os.getcwd(), 'vmlinuz') +initrd_path = os.path.join(os.getcwd(), 'initrd') + +proxy_cmd = [ proc_path, \ + '-name', 'Fedora', '-smp', '4', '-m', '2048', '-cpu', 'host', \
I wonder if setting 2 GB of RAM is too large for something that runs by default as a test.
Quoted 25 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +25
-0+ '-object', 'memory-backend-memfd,id=sysmem-file,size=2G', \ + '-numa', 'node,memdev=sysmem-file', \ + '-kernel', kernel_path, '-initrd', initrd_path, \ + '-vnc', ':0', \ + '-monitor', 'unix:/home/qemu-sock,server,nowait', \ + ] + +if arch == 'x86_64': + print('Downloading images for arch x86_64') + kernel_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/x86_64/os/images/' \ + 'pxeboot/vmlinuz' + initrd_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/x86_64/os/images/' \ + 'pxeboot/initrd.img' + proxy_cmd.append('-machine') + proxy_cmd.append('pc,accel=kvm') + proxy_cmd.append('-append') + proxy_cmd.append('rdinit=/bin/bash console=ttyS0 console=tty0') +elif arch == 'aarch64': + print('Downloading images for arch aarch64') + kernel_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/aarch64/os/images/' \ + 'pxeboot/vmlinuz' + initrd_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \
tests/multiprocess/multiprocess-lsi53c895a.py line 64
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+ 'releases/33/Everything/aarch64/os/images/' \ + 'pxeboot/initrd.img' + proxy_cmd.append('-machine') + proxy_cmd.append('virt,gic-version=3') + proxy_cmd.append('-accel') + proxy_cmd.append('kvm') + proxy_cmd.append('-append') + proxy_cmd.append('rdinit=/bin/bash') +else: + sys.exit('Arch %s not tested' % arch)
It doens't look like you really need a full OS here. Rather than downloading the fairly large Fedora images, I'd suggest just using the kernel that exists on the host OS already in /boot, and then building a tiny initrd that contains just a static linked busybox.
I have this helper script that could be imported into QEMU for this purpose:
https://gitlab.com/berrange/tiny-vm-tools/-/blob/master/make-tiny-image.py
And just skip the test if busybox doesn't exist, or if the vmlinux in /boot isn't accessible (Debian restricts it to root only IIRC)
Quoted 18 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +18
-0+ +urllib.request.urlretrieve(kernel_url, kernel_path) +urllib.request.urlretrieve(initrd_url, initrd_path) + +proxy, remote = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + +proxy_cmd.append('-device') +proxy_cmd.append('x-pci-proxy-dev,id=lsi1,fd='+str(proxy.fileno())) + +remote_cmd = [ proc_path, \ + '-machine', 'x-remote', \ + '-device', 'lsi53c895a,id=lsi1', \ + '-object', \ + 'x-remote-object,id=robj1,devid=lsi1,fd='+str(remote.fileno()), \ + '-display', 'none', \ + '-monitor', 'unix:/home/rem-sock,server,nowait', \ + ] +
tests/multiprocess/multiprocess-lsi53c895a.py line 92
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+pid = os.fork(); + +if pid: + # In Proxy + print('Launching QEMU with Proxy object'); + process = subprocess.Popen(proxy_cmd, pass_fds=[proxy.fileno()]) +else: + # In remote + print('Launching Remote process'); + process = subprocess.Popen(remote_cmd, pass_fds=[remote.fileno(), 0, 1, 2])
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v12 07/19] multi-process: add qio channel function to transmit data and fds
In reply to Jagannathan RamanQuoted 53 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds QIO channel functions that transmits the input iovs as well as the supplied fds.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/io/channel.h | 24 ++++++++++++++++++++++++ io/channel.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+)include/io/channel.h +24
-0@@ -773,5 +773,29 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_read, IOHandler *io_write, void *opaque); +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp);#endif /* QIO_CHANNEL_H */ io/channel.c +0
-0
io/channel.c int qio_channel_writev_all(QIOChannel *ioc, · line 199
Quoted 10 lines
io/channel.c +6
-0@@ -190,6 +190,51 @@ int qio_channel_writev_all(QIOChannel *ioc,return ret; } +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp) +{
Please make qio_channel_writev_all() call qio_channel_writev_full_all() to avoid logic duplication.
Quoted 46 lines
io/channel.c +39
-0+int ret =-1 ; + struct iovec *local_iov = g_new(struct iovec, niov); +struct iovec *local_iov_head = local_iov; +unsigned int nlocal_iov = niov; + + nlocal_iov =iov_copy (local_iov, nlocal_iov, + iov, niov, +0 ,iov_size (iov, niov)); + +while (nlocal_iov >0 ) { +ssize_t len; + len =qio_channel_writev_full (ioc, local_iov, nlocal_iov, fds, + nfds, errp); +if (len ==QIO_CHANNEL_ERR_BLOCK ) { +if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_OUT ); + }else { +qio_channel_wait (ioc,G_IO_OUT ); + } +continue ; + } +if (len <0 ) { + goto cleanup; + } + +iov_discard_front (&local_iov, &nlocal_iov, len); + +if (len >0 ) { + fds =NULL ; + nfds =0 ; + } + } + + ret =0 ; + cleanup: +g_free (local_iov_head); +return ret; +} + ssize_t qio_channel_readv(QIOChannel *ioc, const struct iovec *iov,
size_tniov,
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 08/19] multi-process: define MPQemuMsg format and transmission functions
In reply to Jagannathan RamanHi
Quoted 130 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks.
Define transmission functions used by proxy and by remote. There are certain restrictions on where its safe to use these functions:
- From main loop in co-routine context. Will block the main loop if not
inco-routine context; - From vCPU thread with no co-routine context and if the channel is not part of the main loop handling; - From IOThread within co-routine context, outside of co-routine context will block IOThread;Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>--- include/hw/remote/mpqemu-link.h | 60 ++++++++++ hw/remote/mpqemu-link.c | 242 ++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 305 insertions(+) create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.cinclude/hw/remote/mpqemu-link.h new file +6
-0@@ -0,0 +1,60 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/mpqemu-link.h new file +51
-0+ +#ifndef MPQEMU_LINK_H +#define MPQEMU_LINK_H + +#include "qom/object.h" +#include "qemu/thread.h" +#include "io/channel.h" + +#define REMOTE_MAX_FDS 8 + +#define MPQEMU_MSG_HDR_SIZE offsetof(MPQemuMsg, data.u64) + +/** +* MPQemuCmd: +* +* MPQemuCmd enum type to specify the command to be executed on the remote +* device. +*/ +typedef enum { +MPQEMU_CMD_INIT , +MPQEMU_CMD_MAX , +}MPQemuCmd ; + +/** +* MPQemuMsg: +* @cmd: The remote command +* @size: Size of the data to be shared +* @data: Structured data +* @fds: File descriptors to be shared with remote device +* +* MPQemuMsg Format of the message sent to the remote device from QEMU. +* +*/ +typedef struct { +int cmd; +size_t size; + +union { +uint64_t u64; + } data; + +int fds[REMOTE_MAX_FDS ]; +int num_fds; +}MPQemuMsg ; + +void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); + +bool mpqemu_msg_valid (MPQemuMsg *msg); + +#endif hw/remote/mpqemu-link.c new file +6
-0@@ -0,0 +1,242 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/mpqemu-link.c new file +15
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/module.h" +#include "hw/remote/mpqemu-link.h" +#include "qapi/error.h" +#include "qemu/iov.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" + +/* +* Send message over the ioc QIOChannel. +* This function is safe to call from: +* - From main loop in co-routine context. Will block the main loop if
Quoted 9 lines
not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not part
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;
Can drop the extra "From" on each line.
+ */
Quoted 7 lines
hw/remote/mpqemu-link.c new file +4
-0+void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) +{ +bool iolock =qemu_mutex_iothread_locked (); + bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()?
+ false : true;
I would introduce a qemu_in_iothread() helper (similar to qemu_in_coroutine() etc)
+ Error *local_err = NULL;
Quoted 12 lines
hw/remote/mpqemu-link.c new file +11
-0+struct iovec send[2 ] = {0 }; +int *fds =NULL ; +size_t nfds =0 ; + + send[0 ].iov_base = msg; + send[0 ].iov_len =MPQEMU_MSG_HDR_SIZE ; + + send[1 ].iov_base = (void *)&msg->data; + send[1 ].iov_len = msg->size; + +if (msg->num_fds) {
hw/remote/mpqemu-link.c line 57
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ nfds = msg->num_fds; + fds = msg->fds; + } +/* +* Dont use in IOThread out of co-routine context as +* it will block IOThread. +*/ +if (iothread) { +assert (qemu_in_coroutine ()); + }
or simply assert(!iothread || qemu_in_coroutine())
+ /*
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+* Skip unlocking/locking iothread when in IOThread running +* in co-routine context. Co-routine context is asserted above +* for IOThread case. +* Also skip this while in a co-routine in the main context. +*/ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_unlock_iothread (); + } + + (void)qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send), fds,
Quoted 3 lines
nfds,
+&local_err);
That extra (void) is probably unnecessary.
+
hw/remote/mpqemu-link.c line 80
Quoted 11 lines
hw/remote/mpqemu-link.c new file +10
-0+if (iolock && !iothread && !qemu_in_coroutine ()) { +/* See above comment why skip locking here. */ +qemu_mutex_lock_iothread (); + } + +if (errp) { +error_propagate (errp, local_err); + }else if (local_err) { +error_report_err (local_err); + }
Not sure this behaviour is recommended. Instead, a trace and an ERRP_GUARD would be more idiomatic.
hw/remote/mpqemu-link.c
Quoted 3 lines
hw/remote/mpqemu-link.c new file +2
-0+ +return ;
That's an unnecessary return. Why not return true/false based on error?
+}
Quoted 9 lines
hw/remote/mpqemu-link.c new file +5
-0+ +/* +* Read message from the ioc QIOChannel. +* This function is safe to call from: +* - From main loop in co-routine context. Will block the main loop if not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not part
Quoted 9 lines
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;
+ */
+staticssize_tmpqemu_read(QIOChannel *ioc, void *buf,size_tlen, int **fds,
+size_t*nfds, Error **errp)
+{
Quoted 13 lines
hw/remote/mpqemu-link.c new file +3
-0+struct iovec iov = { .iov_base = buf, .iov_len = len }; +bool iolock =qemu_mutex_iothread_locked (); + bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()hw/remote/mpqemu-link.c new file +8
-0+ ? false : true; +struct iovec *iovp = &iov; +Error *local_err =NULL ; +unsigned int niov =1 ; +size_t *l_nfds = nfds; +int **l_fds = fds; +ssize_t bytes =0 ; +size_t size;
hw/remote/mpqemu-link.c line 118
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ + size = iov.iov_len; + +/* +* Dont use in IOThread out of co-routine context as +* it will block IOThread. +*/ +if (iothread) { +assert (qemu_in_coroutine ()); + }
as above
Quoted 4 lines
hw/remote/mpqemu-link.c new file +3
-0+ +while (size >0 ) { + bytes = qio_channel_readv_full(ioc, iovp, niov, l_fds, l_nfds,
hw/remote/mpqemu-link.c line 131
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ &local_err); +if (bytes ==QIO_CHANNEL_ERR_BLOCK ) { +/* +* Skip unlocking/locking iothread when in IOThread running +* in co-routine context. Co-routine context is asserted above +* for IOThread case. +* Also skip this while in a co-routine in the main context. +*/ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_unlock_iothread ();
Why not lock the iothread at the beginning of the function and call a readv_full_all like we do for writes?
+ }
hw/remote/mpqemu-link.c line 142
Quoted 11 lines
hw/remote/mpqemu-link.c new file +10
-0+if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_IN ); + }else { +qio_channel_wait (ioc,G_IO_IN ); + } +/* See above comment why skip locking here. */ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_lock_iothread (); + } +continue ;
+ }
hw/remote/mpqemu-link.c line 155
Quoted 13 lines
hw/remote/mpqemu-link.c new file +12
-0+ +if (bytes <=0 ) { +error_propagate (errp, local_err); +return -EIO ; + } + + l_fds =NULL ; + l_nfds =NULL ; + + size -= bytes; + + (void )iov_discard_front (&iovp, &niov, bytes);
needless cast
+ }
hw/remote/mpqemu-link.c line 168
Quoted 13 lines
hw/remote/mpqemu-link.c new file +12
-0+ +return len - size; +} + +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) +{ +Error *local_err =NULL ; +int *fds =NULL ; +size_t nfds =0 ; +ssize_t len; + + len = mpqemu_read(ioc, (void *)msg, MPQEMU_MSG_HDR_SIZE, &fds, &nfds,
This cast is not necessary
+ &local_err);
Quoted 14 lines
hw/remote/mpqemu-link.c new file +13
-0+if (!local_err) { +if (len == -EIO ) { +error_setg (&local_err,"Connection closed." ); + goto fail; + } +if (len <0 ) { +error_setg (&local_err,"Message length is less than 0" ); + goto fail; + } +if (len !=MPQEMU_MSG_HDR_SIZE ) { +error_setg (&local_err,"Message header corrupted" ); + goto fail; + }
hw/remote/mpqemu-link.c line 192
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ }else { + goto fail; + } + +if (msg->size >sizeof (msg->data)) { +error_setg (&local_err,"Invalid size for message" ); + goto fail; + } + + if (mpqemu_read(ioc, (void *)&msg->data, msg->size, NULL, NULL,
that one too
+ &local_err) < 0) {
Quoted 79 lines
hw/remote/mpqemu-link.c new file +7
-0+ goto fail; + } + + msg->num_fds = nfds; +if (nfds >G_N_ELEMENTS (msg->fds)) { + error_setg(&local_err, + "Overflow error: received %zu fds, more than max of %dfds",
hw/remote/mpqemu-link.c new file +42
-0+ nfds, REMOTE_MAX_FDS); + goto fail; + }else if (nfds) { +memcpy (msg->fds, fds, nfds *sizeof (int )); + } + +fail: +while (local_err && nfds) { +close (fds[nfds -1 ]); + nfds--; + } + +g_free (fds); + +if (errp) { +error_propagate (errp, local_err); + }else if (local_err) { +error_report_err (local_err); + } +} + +bool mpqemu_msg_valid (MPQemuMsg *msg) +{ +if (msg->cmd >=MPQEMU_CMD_MAX && msg->cmd <0 ) { +return false; + } + +/* Verify FDs. */ +if (msg->num_fds >=REMOTE_MAX_FDS ) { +return false; + } + +if (msg->num_fds >0 ) { +for (int i =0 ; i < msg->num_fds; i++) { +if (fcntl (msg->fds[i],F_GETFL ) ==-1 ) { +return false; + } + } + } + +return true; +}MAINTAINERS +2
-0@@ -3141,6 +3141,8 @@ F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h F: hw/remote/machine.c F: include/hw/remote/machine.h +F: hw/remote/mpqemu-link.c +F: include/hw/remote/mpqemu-link.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -1,5 +1,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 09/19] multi-process: Initialize message handler in remote device
In reply to Jagannathan RamanHi
Quoted 76 lines · Jagannathan Raman
Initializes the message handler function in the remote process. It is called whenever there's an event pending on QIOChannel that registers this function.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/machine.h | 9 +++++++ hw/remote/message.c | 61 +++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 4 files changed, 72 insertions(+) create mode 100644 hw/remote/message.cinclude/hw/remote/machine.h +9
-0@@ -14,6 +14,7 @@#include "qom/object.h" #include "hw/boards.h" #include "hw/pci-host/remote.h" +#include "io/channel.h" typedef struct RemoteMachineState { MachineState parent_obj;@@ -21,8 +22,16 @@ typedef struct RemoteMachineState {RemotePCIHost *host; } RemoteMachineState; +/* Used to pass to co-routine device and ioc. */ +typedef struct RemoteCommDev { +PCIDevice *dev; +QIOChannel *ioc; +}RemoteCommDev ; +#define TYPE_REMOTE_MACHINE "x-remote-machine"#define REMOTE_MACHINE (obj) \ OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) +void coroutine_fn mpqemu_remote_msg_loop_co(void *data); +#endif hw/remote/message.c new file +4
-0@@ -0,0 +1,61 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
+ *
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/message.c new file +14
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/machine.h" +#include "io/channel.h" +#include "hw/remote/mpqemu-link.h" +#include "qapi/error.h" +#include "sysemu/runstate.h" + +void coroutine_fn mpqemu_remote_msg_loop_co(void *data) +{ +RemoteCommDev *com = (RemoteCommDev *)data; +PCIDevice *pci_dev =NULL ;
hw/remote/message.c line 32
Quoted 10 lines
hw/remote/message.c new file +10
-0+ + pci_dev = com->dev; +for (;;) { +MPQemuMsg msg = {0 }; +Error *local_err =NULL ; + +if (!com->ioc) { +error_report ("ERROR: No channel available" ); +break ; + }
Shouldn't this be assert() at the top?
hw/remote/message.c line 36
Quoted 5 lines
hw/remote/message.c new file +4
-0+mpqemu_msg_recv (&msg, com->ioc, &local_err); +if (local_err) { +error_report_err (local_err); +break ;
Error handling is not consistent in this function. Could you cleanup error code paths so error handling & reporting is done in one place?
+ }
Quoted 13 lines
hw/remote/message.c new file +11
-0+ +if (!mpqemu_msg_valid (&msg)) { +error_report ("Received invalid message from proxy" +"in remote process pid=%d" ,getpid ()); +break ; + } + +switch (msg.cmd) { +default : + error_setg(&local_err, + "Unknown command (%d) received for device %s(pid=%d)",
hw/remote/message.c line 54
Quoted 6 lines
hw/remote/message.c new file +6
-0+ msg.cmd, DEVICE(pci_dev)->id, getpid()); + } + +if (local_err) { +error_report_err (local_err); +qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN );
Presumably that error handling should be done outside of the for(;;) loop.
SHUTDOWN_CAUSE_HOST_ERROR might be more appropriate in this case, or perhaps introduce a new ShutdownCause?
+ break;
hw/remote/message.c line 60
Quoted 6 lines
hw/remote/message.c new file +5
-0+ } + } +qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); + +return ;
needless return statement
+}
Quoted 28 lines
MAINTAINERS +1
-0@@ -3143,6 +3143,7 @@ F: hw/remote/machine.c F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h +F: hw/remote/message.c Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -2,5 +2,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 10/19] multi-process: Associate fd of a PCIDevice with its object
In reply to Jagannathan RamanHi
Quoted 59 lines · Jagannathan Raman
Associate the file descriptor for a PCIDevice in remote process with DeviceState object.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/remote-obj.h | 42 +++++++++++ hw/remote/message.c | 1 + hw/remote/remote-obj.c | 154 +++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 5 files changed, 200 insertions(+) create mode 100644 include/hw/remote/remote-obj.h create mode 100644 hw/remote/remote-obj.cinclude/hw/remote/remote-obj.h new file +4
-0@@ -0,0 +1,42 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
+ *
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/remote-obj.h new file +24
-0+ +#ifndef REMOTE_OBJECT_H +#define REMOTE_OBJECT_H + +#include "io/channel.h" +#include "qemu/notify.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT(obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS(obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS(klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { + ObjectClass parent_class; + + unsigned int nr_devs; + unsigned int max_devs; +} RemoteObjectClass; + +typedef struct { + /* private */
include/hw/remote/remote-obj.h
Quoted 10 lines
include/hw/remote/remote-obj.h new file +10
-0+Object parent; + +Notifier machine_done; + +int fd; +char *devid; + QIOChannel *ioc; +} RemoteObject; + +#endif
There is no need for a header if the header isn't actually shared with various .c units. In this series, you can just declare those structs in remote-obj.c
hw/remote/message.c +0 -0
hw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void *data) · line 60
Quoted 9 lines
index 5d87bf4..1f2edc7 100644
hw/remote/message.c +1
-0@@ -56,6 +56,7 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data) } }qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); +g_free (com);
Should be squashed with the previous patch
Quoted 36 lines
hw/remote/message.c +0
-0return ; }hw/remote/remote-obj.c new file +4
-0@@ -0,0 +1,154 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
+ *
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/remote-obj.c new file +17
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/remote-obj.h" +#include "qemu/error-report.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" + +static void remote_object_set_fd(Object *obj, const char *str, Error**errp)
hw/remote/remote-obj.c
Quoted 4 lines
hw/remote/remote-obj.c new file +4
-0+{ +RemoteObject *o =REMOTE_OBJECT (obj); + + o->fd =atoi (str);
qemu_strtoi() has better error handling semantics. You may also want to
check it's a valid socket fd with fd_is_socket().
Alternatively, you could use qemu_open() which allows to open from QMP fdset ("/dev/fdset/..."). This should be more flexible.
+}
Quoted 9 lines
hw/remote/remote-obj.c new file +2
-0+ +static void remote_object_set_devid(Object *obj, const char *str, Error**errp)
hw/remote/remote-obj.c new file +5
-0+{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); +
hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+ o->devid =g_strdup (str); +} + +static void property_release_remote_object (Object *obj,const char *name, +void *opaque) +{ +Object *remote_object =OBJECT (opaque); + +object_unref (remote_object); +}
Hmm, an object property, discussed below.
+
Quoted 23 lines
hw/remote/remote-obj.c new file +23
-0+static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + object_property_add(OBJECT(dev), "remote-object", "object", NULL,
Quoted 4 lines
NULL,
+property_release_remote_object, (void
*)OBJECT(o));
In general, we are trying to avoid runtime/dynamic properties and slowly replacing them with class properties.
Furthermore, this is not the way QOM handles object properties. You should use object_class_property_add_link().
+ /* co-routine should free this. */
hw/remote/remote-obj.c
Quoted 6 lines
hw/remote/remote-obj.c new file +5
-0+ comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + };
+
Quoted 18 lines
hw/remote/remote-obj.c new file +11
-0+ co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev); +qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { + error_report("Reached maximum number of devices: %u",
k->max_devs);hw/remote/remote-obj.c new file +5
-0+return ; + } + + o->ioc =NULL ; + o->fd =-1 ;
hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+ o->devid =NULL ; + + k->nr_devs++; + +object_property_add_str (obj,"fd" ,NULL , remote_object_set_fd); +object_property_set_description (obj,"fd" , +"file descriptor for the object" ); +object_property_add_str (obj,"devid" ,NULL , remote_object_set_devid); +object_property_set_description (obj,"devid" , +"id of device to associate" );
Please use class properties
+
Quoted 16 lines
hw/remote/remote-obj.c new file +15
-0+ o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc));
hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+ + k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + + k->max_devs =1 ;
Could you explain that limitation, in a code comment and/or commit message?
+ k->nr_devs = 0;
Quoted 51 lines
hw/remote/remote-obj.c new file +22
-0+} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);MAINTAINERS +2
-0@@ -3144,6 +3144,8 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: include/hw/remote/remote-obj.h +F: hw/remote/remote-obj.c Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 11/19] multi-process: setup memory manager for remote device
In reply to Jagannathan RamanQuoted 62 lines · Jagannathan Raman
SyncSysMemMsg message format is defined. It is used to send file descriptors of the RAM regions to remote device.
RAM on the remote device is configured with a set of file descriptors. Old RAM regions are deleted and new regions, each with an fd, is added to the RAM.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/memory.h | 19 ++++++++++++++ include/hw/remote/mpqemu-link.h | 13 +++++++++ hw/remote/memory.c | 58 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 11 ++++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 2 ++ 6 files changed, 105 insertions(+) create mode 100644 include/hw/remote/memory.h create mode 100644 hw/remote/memory.cinclude/hw/remote/memory.h new file +6
-0@@ -0,0 +1,19 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/memory.h new file +10
-0+ +#ifndef REMOTE_MEMORY_H +#define REMOTE_MEMORY_H + +#include "exec/hwaddr.h" +#include "hw/remote/mpqemu-link.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp); + +#endif include/hw/remote/mpqemu-link.h +1
-0@@ -14,6 +14,7 @@#include "qom/object.h" #include "qemu/thread.h" #include "io/channel.h" +#include "exec/hwaddr.h" #define REMOTE_MAX_FDS 8
include/hw/remote/mpqemu-link.h line 30
Quoted 6 lines
include/hw/remote/mpqemu-link.h +2
-0@@ -24,12 +25,22 @@* * MPQemuCmd enum type to specify the command to be executed on the remote * device. +* +* SYNC_SYSMEM Shares QEMU's RAM with remote device's RAM
My understanding is that it's deliberately a private protocol between qemu and remote host processes, so that it can break anytime. And in the future it will be vfio-user based. Correct? It would be worth to leave a note about it somewhere.
*/
Quoted 59 lines
include/hw/remote/mpqemu-link.h +10
-0typedef enum {MPQEMU_CMD_INIT , +SYNC_SYSMEM , +RET_MSG ,MPQEMU_CMD_MAX , }MPQemuCmd ; +typedef struct { +hwaddr gpas[REMOTE_MAX_FDS ]; +uint64_t sizes[REMOTE_MAX_FDS ]; +off_t offsets[REMOTE_MAX_FDS ]; +}SyncSysmemMsg ; +/** * MPQemuMsg: * @cmd: The remote command @@ -40,12 +51,14 @@ typedef enum {* MPQemuMsg Format of the message sent to the remote device from QEMU. * */ + typedef struct { int cmd; size_t size; union { uint64_t u64; + SyncSysmemMsg sync_sysmem; } data; int fds[REMOTE_MAX_FDS];hw/remote/memory.c new file +6
-0@@ -0,0 +1,58 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/memory.c new file +11
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/memory.h" +#include "exec/address-spaces.h" +#include "exec/ram_addr.h" +#include "qapi/error.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp) +{
hw/remote/memory.c line 30
Quoted 10 lines
hw/remote/memory.c new file +10
-0+SyncSysmemMsg *sysmem_info = &msg->data.sync_sysmem; +MemoryRegion *sysmem, *subregion, *next; +static unsigned int suffix; +Error *local_err =NULL ; +char *name; +int region; + + sysmem =get_system_memory (); + +memory_region_transaction_begin ();
It looks like this transaction business isn't really helping here. Both del_subregion and add_subregion already handle it.
+
Quoted 8 lines
+ QTAILQ_FOREACH_SAFE(subregion,
&sysmem->subregions,subregions_link,
next) {hw/remote/memory.c new file +5
-0+if (subregion->ram) { +memory_region_del_subregion (sysmem, subregion); +object_unparent (OBJECT (subregion)); + } + }
hw/remote/memory.c line 47
Quoted 10 lines
hw/remote/memory.c new file +10
-0+ +for (region =0 ; region < msg->num_fds; region++) { + subregion =g_new (MemoryRegion,1 ); + name =g_strdup_printf ("remote-mem-%u" , suffix++); +memory_region_init_ram_from_fd (subregion,NULL , + name, sysmem_info->sizes[region], + true, msg->fds[region], + sysmem_info->offsets[region], + &local_err); +g_free (name);
We are quite happily using g_auto these days
hw/remote/memory.c line 49
Quoted 3 lines
hw/remote/memory.c new file +2
-0+if (local_err) { +error_propagate (errp, local_err);
here that would help prevent leaking subregion.
And ERRP_GUARD could also help remove local_err and error_propagate
+ break;
Quoted 53 lines
hw/remote/memory.c new file +8
-0+ } + +memory_region_add_subregion (sysmem, sysmem_info->gpas[region], + subregion); + } + +memory_region_transaction_commit (); +}hw/remote/mpqemu-link.c +11
-0@@ -238,5 +238,16 @@ bool mpqemu_msg_valid(MPQemuMsg *msg) } } +/* Verify message specific fields. */ +switch (msg->cmd) { +case SYNC_SYSMEM : +if (msg->num_fds ==0 || msg->size !=sizeof (SyncSysmemMsg)) { +return false; + } +break ; +default : +break ; + } +return true; }MAINTAINERS +2
-0@@ -3146,6 +3146,8 @@ F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c F: include/hw/remote/remote-obj.h F: hw/remote/remote-obj.c +F: include/hw/remote/memory.h +F: hw/remote/memory.c Build and test automation -------------------------hw/remote/meson.build +2
-0@@ -5,4 +5,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +
Quoted 5 lines
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
I can't comment much on the overall approach. That looks very straightforward to me, I would be surprised if there are not underlying subtleties. Again, is this only experimental until a vfio-user implementation emerge? If not, having unit tests would be important (if not required).
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 11/19] multi-process: setup memory manager for remote device
In reply to Jagannathan RamanQuoted 63 lines · Jagannathan Raman
SyncSysMemMsg message format is defined. It is used to send file descriptors of the RAM regions to remote device.
RAM on the remote device is configured with a set of file descriptors. Old RAM regions are deleted and new regions, each with an fd, is added to the RAM.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/memory.h | 19 ++++++++++++++ include/hw/remote/mpqemu-link.h | 13 +++++++++ hw/remote/memory.c | 58 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 11 ++++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 2 ++ 6 files changed, 105 insertions(+) create mode 100644 include/hw/remote/memory.h create mode 100644 hw/remote/memory.cinclude/hw/remote/memory.h new file +6
-0@@ -0,0 +1,19 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/memory.h new file +10
-0+ +#ifndef REMOTE_MEMORY_H +#define REMOTE_MEMORY_H + +#include "exec/hwaddr.h" +#include "hw/remote/mpqemu-link.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp); + +#endif include/hw/remote/mpqemu-link.h +1
-0@@ -14,6 +14,7 @@#include "qom/object.h" #include "qemu/thread.h" #include "io/channel.h" +#include "exec/hwaddr.h" #define REMOTE_MAX_FDS 8@@ -24,12 +25,22 @@*
include/hw/remote/mpqemu-link.h line 35
Quoted 10 lines
include/hw/remote/mpqemu-link.h +4
-0* MPQemuCmd enum type to specify the command to be executed on the remote * device. +* +* SYNC_SYSMEM Shares QEMU's RAM with remote device's RAM */ typedef enum { MPQEMU_CMD_INIT, + SYNC_SYSMEM, + RET_MSG,
RET_MSG is not used here, but later. ok.
MPQEMU_CMD_MAX,
Quoted 142 lines
include/hw/remote/mpqemu-link.h +8
-0} MPQemuCmd; +typedef struct { +hwaddr gpas[REMOTE_MAX_FDS ]; +uint64_t sizes[REMOTE_MAX_FDS ]; +off_t offsets[REMOTE_MAX_FDS ]; +}SyncSysmemMsg ; +/** * MPQemuMsg: * @cmd: The remote command @@ -40,12 +51,14 @@ typedef enum {* MPQemuMsg Format of the message sent to the remote device from QEMU. * */ + typedef struct { int cmd; size_t size; union { uint64_t u64; + SyncSysmemMsg sync_sysmem; } data; int fds[REMOTE_MAX_FDS];hw/remote/memory.c new file +6
-0@@ -0,0 +1,58 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/memory.c new file +23
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/memory.h" +#include "exec/address-spaces.h" +#include "exec/ram_addr.h" +#include "qapi/error.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp) +{ +SyncSysmemMsg *sysmem_info = &msg->data.sync_sysmem; +MemoryRegion *sysmem, *subregion, *next; +static unsigned int suffix; +Error *local_err =NULL ; +char *name; +int region; + + sysmem =get_system_memory (); + +memory_region_transaction_begin (); + + QTAILQ_FOREACH_SAFE(subregion, &sysmem->subregions, subregions_link,next) {
hw/remote/memory.c new file +26
-0+if (subregion->ram) { +memory_region_del_subregion (sysmem, subregion); +object_unparent (OBJECT (subregion)); + } + } + +for (region =0 ; region < msg->num_fds; region++) { + subregion =g_new (MemoryRegion,1 ); + name =g_strdup_printf ("remote-mem-%u" , suffix++); +memory_region_init_ram_from_fd (subregion,NULL , + name, sysmem_info->sizes[region], + true, msg->fds[region], + sysmem_info->offsets[region], + &local_err); +g_free (name); +if (local_err) { +error_propagate (errp, local_err); +break ; + } + +memory_region_add_subregion (sysmem, sysmem_info->gpas[region], + subregion); + } + +memory_region_transaction_commit (); +}hw/remote/mpqemu-link.c +11
-0@@ -238,5 +238,16 @@ bool mpqemu_msg_valid(MPQemuMsg *msg) } } +/* Verify message specific fields. */ +switch (msg->cmd) { +case SYNC_SYSMEM : +if (msg->num_fds ==0 || msg->size !=sizeof (SyncSysmemMsg)) { +return false; + } +break ; +default : +break ; + } +return true; }MAINTAINERS +2
-0@@ -3146,6 +3146,8 @@ F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c F: include/hw/remote/remote-obj.h F: hw/remote/remote-obj.c +F: include/hw/remote/memory.h +F: hw/remote/memory.c Build and test automation -------------------------hw/remote/meson.build +2
-0@@ -5,4 +5,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 10/19] multi-process: Associate fd of a PCIDevice with its object
In reply to Marc-André LureauQuoted 138 lines · Marc-André Lureau
Hi
Quoted 59 lines · Jagannathan Raman
Associate the file descriptor for a PCIDevice in remote process with DeviceState object.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/remote-obj.h | 42 +++++++++++ hw/remote/message.c | 1 + hw/remote/remote-obj.c | 154 +++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 5 files changed, 200 insertions(+) create mode 100644 include/hw/remote/remote-obj.h create mode 100644 hw/remote/remote-obj.cinclude/hw/remote/remote-obj.h new file +4
-0@@ -0,0 +1,42 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
+ *
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/remote-obj.h new file +24
-0+ +#ifndef REMOTE_OBJECT_H +#define REMOTE_OBJECT_H + +#include "io/channel.h" +#include "qemu/notify.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT(obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS(obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS(klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { + ObjectClass parent_class; + + unsigned int nr_devs; + unsigned int max_devs; +} RemoteObjectClass; + +typedef struct { + /* private */include/hw/remote/remote-obj.h
Quoted 10 lines
include/hw/remote/remote-obj.h new file +10
-0+Object parent; + +Notifier machine_done; + +int fd; +char *devid; + QIOChannel *ioc; +} RemoteObject; + +#endif There is no need for a header if the header isn't actually shared with various .c units. In this series, you can just declare those structs in
remote-obj.chw/remote/message.c +0
-0hw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void *data) · line 60
Quoted 9 lines
index 5d87bf4..1f2edc7 100644
hw/remote/message.c +1
-0@@ -56,6 +56,7 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data) } }qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); +g_free (com);Should be squashed with the previous patch
Quoted 36 lines
hw/remote/message.c +0
-0return ; }hw/remote/remote-obj.c new file +4
-0@@ -0,0 +1,154 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
+ *
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/remote-obj.c new file +17
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/remote-obj.h" +#include "qemu/error-report.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" + +static void remote_object_set_fd(Object *obj, const char *str, Error**errp)
hw/remote/remote-obj.c
Quoted 4 lines
hw/remote/remote-obj.c new file +4
-0+{ +RemoteObject *o =REMOTE_OBJECT (obj); + + o->fd =atoi (str);
qemu_strtoi()has better error handling semantics. You may also want to
check it's a valid socket fd withfd_is_socket().
Quoted 3 lines
Alternatively, you could use
qemu_open()which allows to open from QMP fdset ("/dev/fdset/..."). This should be more flexible.
A better alternative is qemu_parse_fd().
In some later patch, you use monitor_fd_param(monitor_cur(), ...) for x-pci-proxy-dev "fd" property.
That might be the right API, for consistency, use it here too?
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 12/19] multi-process: introduce proxy object
In reply to Jagannathan RamanHi
Quoted 33 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Defines a PCI Device proxy object as a child of
TYPE_PCI_DEVICE.Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/proxy.h | 36 +++++++++++++++++ hw/remote/proxy.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 137 insertions(+) create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/remote/proxy.cinclude/hw/remote/proxy.h new file +4
-0@@ -0,0 +1,36 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
include/hw/remote/proxy.h line 10
Quoted 3 lines
include/hw/remote/proxy.h new file +3
-0+ +#ifndef PROXY_H +#define PROXY_H
+
Quoted 46 lines
include/hw/remote/proxy.h new file +25
-0+#include "hw/pci/pci.h" +#include "io/channel.h" + +#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev" + +#define PCI_PROXY_DEV (obj) \ + OBJECT_CHECK(PCIProxyDev, (obj), TYPE_PCI_PROXY_DEV) +typedef struct PCIProxyDev PCIProxyDev ; + +struct PCIProxyDev { +PCIDevice parent_dev; +char *fd; + +/* +* Mutex used to protect the QIOChannel fd from +* the concurrent access by the VCPUs since proxy +* blocks while awaiting for the replies from the +* process remote. +*/ +QemuMutex io_mutex; +QIOChannel *ioc; +Error *migration_blocker; +}; + +#endif /* PROXY_H */ hw/remote/proxy.c new file +4
-0@@ -0,0 +1,98 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/proxy.c new file +6
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/proxy.h" +#include "hw/pci/pci.h"
hw/remote/proxy.c line 23
Quoted 10 lines
hw/remote/proxy.c new file +10
-0+#include "qapi/error.h" +#include "io/channel-util.h" +#include "hw/qdev-properties.h" +#include "monitor/monitor.h" +#include "migration/blocker.h" + +static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) +{ + pdev->ioc =qio_channel_new_fd (fd, errp); +}
That one line function (called once) should be inlined.
+
hw/remote/proxy.c line 29
Quoted 6 lines
hw/remote/proxy.c new file +5
-0+static Property proxy_properties[] = { +DEFINE_PROP_STRING ("fd" , PCIProxyDev, fd), +DEFINE_PROP_END_OF_LIST (), +}; +
Generally we put properties just above class_init, where it is used.
+static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)
Quoted 2 lines
+{
(errp shouldn't be NULL, but ERRP_GUARD would be safer)
+ PCIProxyDev *dev = PCI_PROXY_DEV(device);
Quoted 4 lines
hw/remote/proxy.c new file +3
-0+int fd; + +if (dev->fd) {
hw/remote/proxy.c line 45
Quoted 10 lines
hw/remote/proxy.c new file +10
-0+ fd =monitor_fd_param (monitor_cur (), dev->fd, errp); +if (fd ==-1 ) { +error_prepend (errp,"proxy: unable to parse fd: " ); +return ; + } +proxy_set_socket (dev, fd, errp); + }else { +error_setg (errp,"fd parameter not specified for %s" , +DEVICE (device)->id); +return ;
We prefer early return, to keep the code easy to read.
if (!dev->fd) {
return error...
}
the_normal_thing();
...
+ }
hw/remote/proxy.c line 51
Quoted 6 lines
hw/remote/proxy.c new file +5
-0+ +error_setg (&dev->migration_blocker,"%s does not support migration" , +TYPE_PCI_PROXY_DEV ); +if (migrate_add_blocker (dev->migration_blocker, errp)) { +error_free (dev->migration_blocker);
leave that free to dev_exit() ?
hw/remote/proxy.c line 54
Quoted 4 lines
hw/remote/proxy.c new file +3
-0+error_free (*errp); + dev->migration_blocker =NULL ; +error_setg (errp,"Failed to set migration blocker" );
Why not use the error from migrate_add_blocker?
+ }
hw/remote/proxy.c line 65
Quoted 11 lines
hw/remote/proxy.c new file +10
-0+ +qemu_mutex_init (&dev->io_mutex); +qio_channel_set_blocking (dev->ioc, true,NULL ); +} + +static void pci_proxy_dev_exit (PCIDevice *pdev) +{ +PCIProxyDev *dev =PCI_PROXY_DEV (pdev); + +qio_channel_close (dev->ioc,NULL );
on early return in realize, dev->ioc is NULL. This will crash.
+
Quoted 2 lines
+
migrate_del_blocker(dev->migration_blocker);
So is migration_blocker, but this should be safe with NULL
hw/remote/proxy.c
Quoted 3 lines
hw/remote/proxy.c new file +2
-0+ +error_free (dev->migration_blocker);
+}
Quoted 59 lines
hw/remote/proxy.c new file +28
-0+ +static void pci_proxy_dev_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +PCIDeviceClass *k =PCI_DEVICE_CLASS (klass); + + k->realize = pci_proxy_dev_realize; + k->exit = pci_proxy_dev_exit; +device_class_set_props (dc, proxy_properties); +} + +static const TypeInfo pci_proxy_dev_type_info = { + .name =TYPE_PCI_PROXY_DEV , + .parent =TYPE_PCI_DEVICE , + .instance_size =sizeof (PCIProxyDev), + .class_init = pci_proxy_dev_class_init, + .interfaces = (InterfaceInfo []) { + {INTERFACE_CONVENTIONAL_PCI_DEVICE }, + { }, + }, +}; + +static void pci_proxy_dev_register_types (void ) +{ +type_register_static (&pci_proxy_dev_type_info); +} + +type_init (pci_proxy_dev_register_types)MAINTAINERS +2
-0@@ -3148,6 +3148,8 @@ F: include/hw/remote/remote-obj.h F: hw/remote/remote-obj.c F: include/hw/remote/memory.h F: hw/remote/memory.c +F: hw/remote/proxy.c +F: include/hw/remote/proxy.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -4,6 +4,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c'))
specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c'))--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 13/19] multi-process: add proxy communication functions
In reply to Jagannathan RamanHi
Quoted 49 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/mpqemu-link.h | 4 ++++ hw/remote/mpqemu-link.c | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)include/hw/remote/mpqemu-link.h +4
-0@@ -15,6 +15,8 @@#include "qemu/thread.h" #include "io/channel.h" #include "exec/hwaddr.h" +#include "io/channel-socket.h" +#include "hw/remote/proxy.h" #define REMOTE_MAX_FDS 8@@ -65,6 +67,8 @@ typedef struct {int num_fds; } MPQemuMsg; +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp);void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp);void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp);hw/remote/mpqemu-link.c +1
-0@@ -17,6 +17,7 @@#include "qemu/iov.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" +#include "io/channel.h" /* * Send message over the ioc QIOChannel.
hw/remote/mpqemu-link.c fail: · line 225
Quoted 6 lines
hw/remote/mpqemu-link.c +2
-0@@ -219,6 +220,43 @@ fail: } } +/* +* Called from VCPU thread in non-coroutine context.
You could check that precondition in code early on, presumably.
The comment could use some further description, like "Send @msg and wait for a RET_MSG reply. Returns the associated u64 message code, or UINT64_MAX on error."
+ */
hw/remote/mpqemu-link.c fail:
Quoted 9 lines
hw/remote/mpqemu-link.c +7
-0+uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp) +{ +MPQemuMsg msg_reply = {0 }; +uint64_t ret =UINT64_MAX ; +Error *local_err =NULL ; +
Should work with ERRP_GUARD instead
+ qemu_mutex_lock(&pdev->io_mutex);
Quoted 1 line
Should work with QEMU_LOCK_GUARD
+ mpqemu_msg_send(msg, pdev->ioc, &local_err);
hw/remote/mpqemu-link.c fail: · line 239
Quoted 6 lines
hw/remote/mpqemu-link.c +5
-0+if (local_err) { +error_propagate (errp, local_err); + goto exit_send; + } +
By making mpqemu_msg_send() return true on success, this should then become simply
if (!mpqemu_msg_send(msg, pdev->ioc, errp))
return ret;
+ mpqemu_msg_recv(&msg_reply, pdev->ioc, &local_err);
hw/remote/mpqemu-link.c fail: · line 250
Quoted 11 lines
hw/remote/mpqemu-link.c +10
-0+if (local_err) { +error_propagate (errp, local_err); + goto exit_send; + } + +if (!mpqemu_msg_valid (&msg_reply) || msg_reply.cmd !=RET_MSG ) { +error_setg (errp,"ERROR: Invalid reply received for command %d" , + msg->cmd); + goto exit_send; + }else {
that else is unneeded.
The function with automatic cleanups should be simpler.
+ ret = msg_reply.data.u64;
Quoted 15 lines
hw/remote/mpqemu-link.c +8
-1+ } + + exit_send: +qemu_mutex_unlock (&pdev->io_mutex); + +return ret; +} +bool mpqemu_msg_valid (MPQemuMsg *msg) {if (msg->cmd >=MPQEMU_CMD_MAX && msg->cmd <0 ) { --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 14/19] multi-process: Forward PCI config space acceses to the remote process
In reply to Jagannathan RamanHi
Quoted 23 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
The Proxy Object sends the PCI config space accesses as messages to the remote process over the communication channel
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/mpqemu-link.h | 9 ++++++ hw/remote/message.c | 62 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 6 ++++ hw/remote/proxy.c | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+)include/hw/remote/mpqemu-link.h +0
-0
include/hw/remote/mpqemu-link.h typedef enum { · line 40
Quoted 7 lines
include/hw/remote/mpqemu-link.h +2
-0@@ -34,6 +34,8 @@ typedef enum {MPQEMU_CMD_INIT ,SYNC_SYSMEM ,RET_MSG , +PCI_CONFIG_WRITE , +PCI_CONFIG_READ , MPQEMU_CMD_MAX,
It would be a good idea to prefix all enums with MQEMU_CMD.
} MPQemuCmd;
include/hw/remote/mpqemu-link.h typedef struct { · line 52
Quoted 10 lines
include/hw/remote/mpqemu-link.h +4
-0@@ -43,6 +45,12 @@ typedef struct {off_t offsets[REMOTE_MAX_FDS ]; } SyncSysmemMsg; +typedef struct { + uint32_t addr; + uint32_t val; + int l;
"len" please, it's already short enough :)
+} PciConfDataMsg;
Quoted 30 lines
include/hw/remote/mpqemu-link.h +2
-0+/** * MPQemuMsg: * @cmd: The remote command @@ -60,6 +68,7 @@ typedef struct {union {uint64_t u64; +PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; } data;hw/remote/message.c +6
-0@@ -15,6 +15,12 @@#include "hw/remote/mpqemu-link.h" #include "qapi/error.h" #include "sysemu/runstate.h" +#include "hw/pci/pci.h" + +static void process_config_write (QIOChannel *ioc,PCIDevice *dev, +MPQemuMsg *msg); +static void process_config_read (QIOChannel *ioc,PCIDevice *dev, +MPQemuMsg *msg); void coroutine_fn mpqemu_remote_msg_loop_co(void *data) {
hw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void *data) · line 55
Quoted 7 lines
hw/remote/message.c +3
-0@@ -43,6 +49,12 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data) }switch (msg.cmd) { +case PCI_CONFIG_WRITE : +process_config_write (com->ioc, pci_dev, &msg); +break ;
Some other process_ functions take &local_err. I think this is a better idea, so error reporting is done in a single place from mpqemu_remote_msg_loop_co() for now
+ case PCI_CONFIG_READ:
Quoted 20 lines
hw/remote/message.c +2
-0+process_config_read (com->ioc, pci_dev, &msg); +break ;default : error_setg(&local_err, "Unknown command (%d) received for device %s(pid=%d)",
hw/remote/message.c +10
-0@@ -60,3 +72,53 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)return ; } + +static void process_config_write (QIOChannel *ioc,PCIDevice *dev, +MPQemuMsg *msg) +{ +PciConfDataMsg *conf = (PciConfDataMsg *)&msg->data.pci_conf_data; +MPQemuMsg ret = {0 }; +Error *local_err =NULL ; + +if ((conf->addr +sizeof (conf->val)) >pci_config_size (dev)) { + error_report("Bad address received when writing PCI config, pid
Quoted 3 lines
%d",
+getpid());
Should use FMT_pid.
But then, I am not sure some error messages should have PID, while most of them dont. That should be either the job of a log manager, or a custom logger function/option.
+ ret.data.u64 = UINT64_MAX;
hw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void *data) · line 97
Quoted 12 lines
hw/remote/message.c +11
-0+ }else { +pci_default_write_config (dev, conf->addr, conf->val, conf->l); + } + + ret.cmd =RET_MSG ; + ret.size =sizeof (ret.data.u64); + +mpqemu_msg_send (&ret, ioc, &local_err); +if (local_err) { +error_report ("Could not send message to proxy from pid %d" , +getpid ());
Here it leaks local_err, and ignores it. Use error_prepend instead?
+ }
Quoted 16 lines
hw/remote/message.c +11
-0+} + +static void process_config_read (QIOChannel *ioc,PCIDevice *dev, +MPQemuMsg *msg) +{ +PciConfDataMsg *conf = (PciConfDataMsg *)&msg->data.pci_conf_data; +MPQemuMsg ret = {0 }; +Error *local_err =NULL ; + +if ((conf->addr +sizeof (conf->val)) >pci_config_size (dev)) { + error_report("Bad address received when reading PCI config, pid%d",
hw/remote/message.c +3
-0+ getpid()); + ret.data.u64 =UINT64_MAX ; + }else {
hw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void *data) · line 122
Quoted 10 lines
hw/remote/message.c +10
-0+ ret.data.u64 =pci_default_read_config (dev, conf->addr, conf->l); + } + + ret.cmd =RET_MSG ; + ret.size =sizeof (ret.data.u64); + +mpqemu_msg_send (&ret, ioc, &local_err); +if (local_err) { +error_report ("Could not send message to proxy from pid %d" , +getpid ());
Same as earlier
+ }
Quoted 45 lines
hw/remote/message.c +1
-0+}hw/remote/mpqemu-link.c +6
-0@@ -283,6 +283,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case PCI_CONFIG_WRITE : +case PCI_CONFIG_READ : +if (msg->size !=sizeof (PciConfDataMsg)) { +return false; + } +break ;default :break ; }hw/remote/proxy.c +12
-0@@ -16,6 +16,8 @@#include "hw/qdev-properties.h" #include "monitor/monitor.h" #include "migration/blocker.h" +#include "hw/remote/mpqemu-link.h" +#include "qemu/error-report.h" static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {@@ -69,6 +71,52 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)error_free (dev->migration_blocker); } +static int config_op_send (PCIProxyDev *pdev,uint32_t addr,uint32_t *val, +int l,unsigned int op) +{ +MPQemuMsg msg = {0 }; +uint64_t ret = -EINVAL ; +Error *local_err =NULL ; + + msg.cmd = op; + msg.data.pci_conf_data.addr = addr; + msg.data.pci_conf_data.val = (op ==PCI_CONFIG_WRITE ) ? *val :0 ;
hw/remote/proxy.c static void pci_proxy_dev_exit(PCIDevice *pdev) · line 93
Quoted 10 lines
hw/remote/proxy.c +9
-0+ msg.data.pci_conf_data.l = l; + msg.size =sizeof (PciConfDataMsg); + + ret =mpqemu_msg_send_and_await_reply (&msg, pdev, &local_err); +if (local_err) { +error_report_err (local_err); + } +if (op ==PCI_CONFIG_READ ) { + *val = (uint32_t )ret;
That's a suspicious cast, without error checking.
+ }
hw/remote/proxy.c static void pci_proxy_dev_exit(PCIDevice *pdev)
Quoted 12 lines
hw/remote/proxy.c +9
-0+ +return ret; +} + +static uint32_t pci_proxy_read_config (PCIDevice *d,uint32_t addr,int len) +{ +uint32_t val; + + (void )config_op_send (PCI_PROXY_DEV (d), addr, &val, len,PCI_CONFIG_READ );
I don't know why (void)cast here, please enlighten me
+
Quoted 15 lines
hw/remote/proxy.c +4
-0+return val; +} + +static void pci_proxy_write_config(PCIDevice *d, uint32_t addr, uint32_tval,
hw/remote/proxy.c +10
-0+ int l) +{ +/* +* Some of the functions access the copy of remote device's PCI config +* space which is cached in the proxy device. Therefore, maintain +* it updated. +*/ +pci_default_write_config (d, addr, val, l); + + (void)config_op_send(PCI_PROXY_DEV(d), addr, &val, l,
Quoted 2 lines
PCI_CONFIG_WRITE);
again
+}
Quoted 19 lines
hw/remote/proxy.c +4
-1+static void pci_proxy_dev_class_init (ObjectClass *klass,void *data) {DeviceClass *dc =DEVICE_CLASS (klass);@@ -76,6 +124,9 @@ static void pci_proxy_dev_class_init(ObjectClass *klass, void *data) k->realize = pci_proxy_dev_realize; k->exit = pci_proxy_dev_exit; + k->config_read = pci_proxy_read_config; + k->config_write = pci_proxy_write_config; +device_class_set_props (dc, proxy_properties); } --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 16/19] multi-process: Synchronize remote memory
In reply to Jagannathan RamanHi
Quoted 48 lines · Jagannathan Raman
Add memory-listener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sendsSYNC_SYSMEMmessage to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler forSYNC_SYSMEMmessage.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/memory-sync.h | 27 ++++++ include/hw/remote/proxy.h | 2 + hw/remote/memory-sync.c | 210 ++++++++++++++++++++++++++++++++++++++++ hw/remote/message.c | 5 + hw/remote/proxy.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 253 insertions(+) create mode 100644 include/hw/remote/memory-sync.h create mode 100644 hw/remote/memory-sync.cinclude/hw/remote/memory-sync.h new file +4
-0@@ -0,0 +1,27 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/memory-sync.h new file +8
-0+ +#ifndef MEMORY_SYNC_H +#define MEMORY_SYNC_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct RemoteMemSync {
include/hw/remote/memory-sync.h line 25
Quoted 10 lines
include/hw/remote/memory-sync.h new file +10
-0+MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +} RemoteMemSync; + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc); +void deconfigure_memory_sync (RemoteMemSync *sync);
RemoteMemSync vs MemorySync, and function with _memory_sync suffixes... Naming things is hard, but trying to be consistent generally helps.
My understanding is that this is a proxy-dev helper to handle memory listening and sending SYNC_SYSMEM.
I would thus suggest naming it ProxyMemoryListener. It could eventually be folded in proxy.c
Please try to be consistent with header naming, structure naming, type, functions and enum prefixes etc.
proxy_memory_listener isn't that long imho.
+
Quoted 42 lines
include/hw/remote/memory-sync.h new file +1
-0+#endif include/hw/remote/proxy.h +2
-0@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/memory-sync.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"@@ -40,6 +41,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +RemoteMemSync sync;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };hw/remote/memory-sync.c new file +4
-0@@ -0,0 +1,210 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/memory-sync.c new file +6
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/compiler.h" +#include "qemu/int128.h"
hw/remote/memory-sync.c line 23
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+#include "qemu/range.h" +#include "exec/memory.h" +#include "exec/cpu-common.h" +#include "cpu.h" +#include "exec/ram_addr.h" +#include "exec/address-spaces.h" +#include "hw/remote/mpqemu-link.h" +#include "hw/remote/memory-sync.h" + +static void proxy_ml_begin (MemoryListener *listener)
I suggest to rename begin -> reset
+{
Quoted 4 lines
hw/remote/memory-sync.c new file +3
-0+RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); +int mrs; +
hw/remote/memory-sync.c line 37
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+for (mrs =0 ; mrs < sync->n_mr_sections; mrs++) { +memory_region_unref (sync->mr_sections[mrs].mr); + } + +g_free (sync->mr_sections); + sync->mr_sections =NULL ; + sync->n_mr_sections =0 ; +} + +static int get_fd_from_hostaddr (uint64_t host,ram_addr_t *offset)
This function is very similar to vhost_user_get_mr_data(). That suggests we could factor the code.
Perhaps a new memory_region_from_host_full(), or extend memory_region_from_host() with an extra optional "int *fd" argument.
Quoted 21 lines
hw/remote/memory-sync.c new file +21
-0+{ +MemoryRegion *mr; +ram_addr_t off; + +/** +* Assumes that the host address is a valid address as it's +* coming from the MemoryListener system. In the case host +* address is not valid, the following call would return +* the default subregion of "system_memory" region, and +* not NULL. So it's not possible to check for NULL here. +*/ + mr =memory_region_from_host ((void *)(uintptr_t )host, &off); + +if (offset) { + *offset = off; + } + +return memory_region_get_fd (mr); +} + +static bool proxy_mrs_can_merge(uint64_t host, uint64_t prev_host, size_t
Quoted 3 lines
size)
+{
This seems similar to vhost_user_can_merge().
+ bool merge;
hw/remote/memory-sync.c line 67
Quoted 8 lines
hw/remote/memory-sync.c new file +7
-0+int fd1, fd2; + + fd1 =get_fd_from_hostaddr (host,NULL ); + + fd2 =get_fd_from_hostaddr (prev_host,NULL ); + + merge = (fd1 == fd2);
This could be written in a simpler manner, ex:
if (get_fd_from_hostaddr(host, NULL) != get_fd_from_hostaddr(prev_host, NULL))
return false
+
Quoted 2 lines
+ merge &= ((
prev_host+ size) == host);
That check could be done early on before doing the more expensive memory_region_from_host() calls
+
Quoted 59 lines
hw/remote/memory-sync.c new file +58
-0+return merge; +} + +static bool try_merge (RemoteMemSync *sync,MemoryRegionSection *section) +{ +uint64_t mrs_size, mrs_gpa, mrs_page; +MemoryRegionSection *prev_sec; +bool merged = false; +uintptr_t mrs_host; +RAMBlock *mrs_rb; + +if (!sync->n_mr_sections) { +return false; + } + + mrs_rb = section->mr->ram_block; + mrs_page = (uint64_t )qemu_ram_pagesize (mrs_rb); + mrs_size =int128_get64 (section->size); + mrs_gpa = section->offset_within_address_space; + mrs_host = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + +if (get_fd_from_hostaddr (mrs_host,NULL ) <0 ) { +return true; + } + + mrs_host = mrs_host & ~(mrs_page -1 ); + mrs_gpa = mrs_gpa & ~(mrs_page -1 ); + mrs_size =ROUND_UP (mrs_size, mrs_page); + + prev_sec = sync->mr_sections + (sync->n_mr_sections -1 ); +uint64_t prev_gpa_start = prev_sec->offset_within_address_space; +uint64_t prev_size =int128_get64 (prev_sec->size); +uint64_t prev_gpa_end =range_get_last (prev_gpa_start, prev_size); +uint64_t prev_host_start = + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr) + + prev_sec->offset_within_region; +uint64_t prev_host_end =range_get_last (prev_host_start, prev_size); + +if (mrs_gpa <= (prev_gpa_end +1 )) { +g_assert (mrs_gpa > prev_gpa_start); + +if ((section->mr == prev_sec->mr) && +proxy_mrs_can_merge (mrs_host, prev_host_start, + (mrs_gpa - prev_gpa_start))) { +uint64_t max_end =MAX (prev_host_end, mrs_host + mrs_size); + merged = true; + prev_sec->offset_within_address_space = +MIN (prev_gpa_start, mrs_gpa); + prev_sec->offset_within_region = +MIN (prev_host_start, mrs_host) - + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr); + prev_sec->size =int128_make64 (max_end -MIN (prev_host_start, + mrs_host)); + } + } + +return merged;
hw/remote/memory-sync.c line 138
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+} + +static void proxy_ml_region_addnop (MemoryListener *listener, +MemoryRegionSection *section) +{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); + +if (!(memory_region_is_ram (section->mr) && + !memory_region_is_rom (section->mr))) { +return ;
A bit clearer in vhost.c:
if (memory_region_is_ram(mr) && !memory_region_is_rom(mr)) {
Quoted 40 lines
hw/remote/memory-sync.c new file +39
-0+ } + +if (try_merge (sync, section)) { +return ; + } + + ++sync->n_mr_sections; + sync->mr_sections =g_renew (MemoryRegionSection, sync->mr_sections, + sync->n_mr_sections); + sync->mr_sections[sync->n_mr_sections -1 ] = *section; + sync->mr_sections[sync->n_mr_sections -1 ].fv =NULL ; +memory_region_ref (section->mr); +} + +static void proxy_ml_commit (MemoryListener *listener) +{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); +MPQemuMsg msg; +MemoryRegionSection *section; +ram_addr_t offset; +uintptr_t host_addr; +int region; +Error *local_err =NULL ; + +memset (&msg,0 ,sizeof (MPQemuMsg)); + + msg.cmd =SYNC_SYSMEM ; + msg.num_fds = sync->n_mr_sections; + msg.size =sizeof (SyncSysmemMsg); +if (msg.num_fds >REMOTE_MAX_FDS ) { +error_report ("Number of fds is more than %d" ,REMOTE_MAX_FDS ); +return ; + } + +for (region =0 ; region < sync->n_mr_sections; region++) { + section = &sync->mr_sections[region]; + msg.data.sync_sysmem.gpas[region] = + section->offset_within_address_space; + msg.data.sync_sysmem.sizes[region] =int128_get64 (section->size);
hw/remote/memory-sync.c line 187
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+ host_addr = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + msg.fds[region] =get_fd_from_hostaddr (host_addr, &offset); + msg.data.sync_sysmem.offsets[region] = offset; + } +mpqemu_msg_send (&msg, sync->ioc, &local_err); +if (local_err) { +error_report ("Error in sending command %d" , msg.cmd); + } +}
That whole complex code above duplicates much of the logic in vhost.c. Can we try to factorize it instead?
+
Quoted 108 lines
hw/remote/memory-sync.c new file +22
-0+void deconfigure_memory_sync (RemoteMemSync *sync) +{ +memory_listener_unregister (&sync->listener); + +proxy_ml_begin (&sync->listener); +} + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc) +{ + sync->n_mr_sections =0 ; + sync->mr_sections =NULL ; + + sync->ioc = ioc; + + sync->listener.begin = proxy_ml_begin; + sync->listener.commit = proxy_ml_commit; + sync->listener.region_add = proxy_ml_region_addnop; + sync->listener.region_nop = proxy_ml_region_addnop; + sync->listener.priority =10 ; + +memory_listener_register (&sync->listener, &address_space_memory); +}hw/remote/message.c +5
-0@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);@@ -64,6 +65,10 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ; +default : error_setg(&local_err, "Unknown command (%d) received for device %s(pid=%d)",
hw/remote/proxy.c +4
-0@@ -18,6 +18,8 @@#include "migration/blocker.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/memory-sync.h" +#include "qom/object.h" static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {@@ -58,6 +60,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp) qemu_mutex_init(&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +configure_memory_sync (&dev->sync, dev->ioc); }static void
pci_proxy_dev_exit(PCIDevice *pdev)hw/remote/proxy.c +2
-0@@ -69,6 +73,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +deconfigure_memory_sync (&dev->sync); } static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,MAINTAINERS +2
-0@@ -3150,6 +3150,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/memory-sync.c +F: include/hw/remote/memory-sync.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory-sync.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 02/19] multi-process: add configure and usage information
In reply to Daniel P. BerrangéQuoted 104 lines · Daniel P. Berrangé
Quoted 44 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds documentation explaining the command-line arguments needed to use multi-process. Also adds a python script that illustrates the usage.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>---
docs/multi-process.rst | 66 +++++++++++++++++++ MAINTAINERS | 1 + tests/multiprocess/multiprocess-lsi53c895a.py | 92 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+)create mode 100644
docs/multi-process.rst
create mode 100755tests/multiprocess/multiprocess-lsi53c895a.pytests/multiprocess/multiprocess-lsi53c895a.py new file +19
-0@@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +import urllib.request +import subprocess +import argparse +import socket +import sys +import os + +arch = os.uname()[4] +proc_path = os.path.join(os.getcwd(), '..', '..', 'build', arch+'-softmmu', + 'qemu-system-'+arch) + +parser = argparse.ArgumentParser(description='Launcher for multi-process QEMU') +parser.add_argument('--bin', required=False, help='location of QEMU binary', + metavar='bin'); +args = parser.parse_args() + +if args.bin is not None:tests/multiprocess/multiprocess-lsi53c895a.py line 29
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+ proc_path = args.bin + +if not os.path.isfile(proc_path): + sys.exit('QEMU binary not found') + +kernel_path = os.path.join(os.getcwd(), 'vmlinuz') +initrd_path = os.path.join(os.getcwd(), 'initrd') + +proxy_cmd = [ proc_path, \ + '-name', 'Fedora', '-smp', '4', '-m', '2048', '-cpu', 'host', \I wonder if setting 2 GB of RAM is too large for something that runs by default as a test.
Quoted 25 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +25
-0+ '-object', 'memory-backend-memfd,id=sysmem-file,size=2G', \ + '-numa', 'node,memdev=sysmem-file', \ + '-kernel', kernel_path, '-initrd', initrd_path, \ + '-vnc', ':0', \ + '-monitor', 'unix:/home/qemu-sock,server,nowait', \ + ] + +if arch == 'x86_64': + print('Downloading images for arch x86_64') + kernel_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/x86_64/os/images/' \ + 'pxeboot/vmlinuz' + initrd_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/x86_64/os/images/' \ + 'pxeboot/initrd.img' + proxy_cmd.append('-machine') + proxy_cmd.append('pc,accel=kvm') + proxy_cmd.append('-append') + proxy_cmd.append('rdinit=/bin/bash console=ttyS0 console=tty0') +elif arch == 'aarch64': + print('Downloading images for arch aarch64') + kernel_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \ + 'releases/33/Everything/aarch64/os/images/' \ + 'pxeboot/vmlinuz' + initrd_url = 'https://dl.fedoraproject.org/pub/fedora/linux/' \tests/multiprocess/multiprocess-lsi53c895a.py line 64
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+ 'releases/33/Everything/aarch64/os/images/' \ + 'pxeboot/initrd.img' + proxy_cmd.append('-machine') + proxy_cmd.append('virt,gic-version=3') + proxy_cmd.append('-accel') + proxy_cmd.append('kvm') + proxy_cmd.append('-append') + proxy_cmd.append('rdinit=/bin/bash') +else: + sys.exit('Arch %s not tested' % arch)It doens't look like you really need a full OS here. Rather than downloading the fairly large Fedora images, I'd suggest just using the kernel that exists on the host OS already in /boot, and then building a tiny initrd that contains just a static linked busybox.
Quoted 4 lines
I have this helper script that could be imported into QEMU for this purpose:
https://gitlab.com/berrange/tiny-vm-tools/-/blob/master/make-tiny-image.py
Hi Daniel,
That’s a nifty script. I was trying to do something similar. Thank you for sharing!
—
Jag
Quoted 41 lines
And just skip the test if busybox doesn't exist, or if the vmlinux in /boot isn't accessible (Debian restricts it to root only IIRC)
Quoted 18 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +18
-0+ +urllib.request.urlretrieve(kernel_url, kernel_path) +urllib.request.urlretrieve(initrd_url, initrd_path) + +proxy, remote = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + +proxy_cmd.append('-device') +proxy_cmd.append('x-pci-proxy-dev,id=lsi1,fd='+str(proxy.fileno())) + +remote_cmd = [ proc_path, \ + '-machine', 'x-remote', \ + '-device', 'lsi53c895a,id=lsi1', \ + '-object', \ + 'x-remote-object,id=robj1,devid=lsi1,fd='+str(remote.fileno()), \ + '-display', 'none', \ + '-monitor', 'unix:/home/rem-sock,server,nowait', \ + ] +tests/multiprocess/multiprocess-lsi53c895a.py line 92
Quoted 10 lines
tests/multiprocess/multiprocess-lsi53c895a.py new file +10
-0+pid = os.fork(); + +if pid: + # In Proxy + print('Launching QEMU with Proxy object'); + process = subprocess.Popen(proxy_cmd, pass_fds=[proxy.fileno()]) +else: + # In remote + print('Launching Remote process'); + process = subprocess.Popen(remote_cmd, pass_fds=[remote.fileno(), 0, 1, 2])Regards,
DanielSignature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Re: [PATCH v12 16/19] multi-process: Synchronize remote memory
In reply to Marc-André LureauQuoted 313 lines · Marc-André Lureau
Hi
On Wed, Dec 2, 2020 at 12:23 AM Jagannathan Raman <[email protected]> wrote:
Add memory-listener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sendsSYNC_SYSMEMmessage to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler forSYNC_SYSMEMmessage.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/memory-sync.h | 27 ++++++ include/hw/remote/proxy.h | 2 + hw/remote/memory-sync.c | 210 ++++++++++++++++++++++++++++++++++++++++ hw/remote/message.c | 5 + hw/remote/proxy.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 253 insertions(+) create mode 100644 include/hw/remote/memory-sync.h create mode 100644 hw/remote/memory-sync.cinclude/hw/remote/memory-sync.h new file +25
-0@@ -0,0 +1,27 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef MEMORY_SYNC_H +#define MEMORY_SYNC_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct RemoteMemSync { +MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +}RemoteMemSync ; + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc); +void deconfigure_memory_sync (RemoteMemSync *sync);RemoteMemSync vs MemorySync, and function with
_memory_syncsuffixes... Naming things is hard, but trying to be consistent generally helps.My understanding is that this is a proxy-dev helper to handle memory listening and sending
SYNC_SYSMEM.I would thus suggest naming it ProxyMemoryListener. It could eventually be folded in
proxy.cPlease try to be consistent with header naming, structure naming, type, functions and enum prefixes etc.
proxy_memory_listenerisn't that long imho.include/hw/remote/memory-sync.h new file +2
-0+ +#endif include/hw/remote/proxy.h +2
-0@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/memory-sync.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"@@ -40,6 +41,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +RemoteMemSync sync;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };hw/remote/memory-sync.c new file +23
-0@@ -0,0 +1,210 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/compiler.h" +#include "qemu/int128.h" +#include "qemu/range.h" +#include "exec/memory.h" +#include "exec/cpu-common.h" +#include "cpu.h" +#include "exec/ram_addr.h" +#include "exec/address-spaces.h" +#include "hw/remote/mpqemu-link.h" +#include "hw/remote/memory-sync.h" + +static void proxy_ml_begin (MemoryListener *listener)I suggest to rename begin -> reset
hw/remote/memory-sync.c new file +14
-0+{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); +int mrs; + +for (mrs =0 ; mrs < sync->n_mr_sections; mrs++) { +memory_region_unref (sync->mr_sections[mrs].mr); + } + +g_free (sync->mr_sections); + sync->mr_sections =NULL ; + sync->n_mr_sections =0 ; +} + +static int get_fd_from_hostaddr (uint64_t host,ram_addr_t *offset)This function is very similar to
vhost_user_get_mr_data(). That suggests we could factor the code.Perhaps a new
memory_region_from_host_full(), or extendmemory_region_from_host()with an extra optional "int *fd" argument.hw/remote/memory-sync.c new file +22
-0+{ +MemoryRegion *mr; +ram_addr_t off; + +/** +* Assumes that the host address is a valid address as it's +* coming from the MemoryListener system. In the case host +* address is not valid, the following call would return +* the default subregion of "system_memory" region, and +* not NULL. So it's not possible to check for NULL here. +*/ + mr =memory_region_from_host ((void *)(uintptr_t )host, &off); + +if (offset) { + *offset = off; + } + +return memory_region_get_fd (mr); +} + +static bool proxy_mrs_can_merge (uint64_t host,uint64_t prev_host,size_t size) +{This seems similar to
vhost_user_can_merge().hw/remote/memory-sync.c new file +8
-0+bool merge; +int fd1, fd2; + + fd1 =get_fd_from_hostaddr (host,NULL ); + + fd2 =get_fd_from_hostaddr (prev_host,NULL ); + + merge = (fd1 == fd2);This could be written in a simpler manner, ex:
if (
get_fd_from_hostaddr(host, NULL)!=get_fd_from_hostaddr(prev_host, NULL))
return falsehw/remote/memory-sync.c new file +2
-0+ + merge &= ((prev_host + size) == host);That check could be done early on before doing the more expensive
memory_region_from_host()callshw/remote/memory-sync.c new file +69
-0+ +return merge; +} + +static bool try_merge (RemoteMemSync *sync,MemoryRegionSection *section) +{ +uint64_t mrs_size, mrs_gpa, mrs_page; +MemoryRegionSection *prev_sec; +bool merged = false; +uintptr_t mrs_host; +RAMBlock *mrs_rb; + +if (!sync->n_mr_sections) { +return false; + } + + mrs_rb = section->mr->ram_block; + mrs_page = (uint64_t )qemu_ram_pagesize (mrs_rb); + mrs_size =int128_get64 (section->size); + mrs_gpa = section->offset_within_address_space; + mrs_host = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + +if (get_fd_from_hostaddr (mrs_host,NULL ) <0 ) { +return true; + } + + mrs_host = mrs_host & ~(mrs_page -1 ); + mrs_gpa = mrs_gpa & ~(mrs_page -1 ); + mrs_size =ROUND_UP (mrs_size, mrs_page); + + prev_sec = sync->mr_sections + (sync->n_mr_sections -1 ); +uint64_t prev_gpa_start = prev_sec->offset_within_address_space; +uint64_t prev_size =int128_get64 (prev_sec->size); +uint64_t prev_gpa_end =range_get_last (prev_gpa_start, prev_size); +uint64_t prev_host_start = + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr) + + prev_sec->offset_within_region; +uint64_t prev_host_end =range_get_last (prev_host_start, prev_size); + +if (mrs_gpa <= (prev_gpa_end +1 )) { +g_assert (mrs_gpa > prev_gpa_start); + +if ((section->mr == prev_sec->mr) && +proxy_mrs_can_merge (mrs_host, prev_host_start, + (mrs_gpa - prev_gpa_start))) { +uint64_t max_end =MAX (prev_host_end, mrs_host + mrs_size); + merged = true; + prev_sec->offset_within_address_space = +MIN (prev_gpa_start, mrs_gpa); + prev_sec->offset_within_region = +MIN (prev_host_start, mrs_host) - + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr); + prev_sec->size =int128_make64 (max_end -MIN (prev_host_start, + mrs_host)); + } + } + +return merged; +} + +static void proxy_ml_region_addnop (MemoryListener *listener, +MemoryRegionSection *section) +{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); + +if (!(memory_region_is_ram (section->mr) && + !memory_region_is_rom (section->mr))) { +return ;A bit clearer in
vhost.c:
if (memory_region_is_ram(mr)&& !memory_region_is_rom(mr)) {hw/remote/memory-sync.c new file +49
-0+ } + +if (try_merge (sync, section)) { +return ; + } + + ++sync->n_mr_sections; + sync->mr_sections =g_renew (MemoryRegionSection, sync->mr_sections, + sync->n_mr_sections); + sync->mr_sections[sync->n_mr_sections -1 ] = *section; + sync->mr_sections[sync->n_mr_sections -1 ].fv =NULL ; +memory_region_ref (section->mr); +} + +static void proxy_ml_commit (MemoryListener *listener) +{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); +MPQemuMsg msg; +MemoryRegionSection *section; +ram_addr_t offset; +uintptr_t host_addr; +int region; +Error *local_err =NULL ; + +memset (&msg,0 ,sizeof (MPQemuMsg)); + + msg.cmd =SYNC_SYSMEM ; + msg.num_fds = sync->n_mr_sections; + msg.size =sizeof (SyncSysmemMsg); +if (msg.num_fds >REMOTE_MAX_FDS ) { +error_report ("Number of fds is more than %d" ,REMOTE_MAX_FDS ); +return ; + } + +for (region =0 ; region < sync->n_mr_sections; region++) { + section = &sync->mr_sections[region]; + msg.data.sync_sysmem.gpas[region] = + section->offset_within_address_space; + msg.data.sync_sysmem.sizes[region] =int128_get64 (section->size); + host_addr = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + msg.fds[region] =get_fd_from_hostaddr (host_addr, &offset); + msg.data.sync_sysmem.offsets[region] = offset; + } +mpqemu_msg_send (&msg, sync->ioc, &local_err); +if (local_err) { +error_report ("Error in sending command %d" , msg.cmd); + } +}
Quoted 1 line
That whole complex code above duplicates much of the logic in
vhost.c. Can we try to factorize it instead?
Hi Marc-Andre,
Thank you for sharing your feedback!
Would it be alright if we addressed this item alone in a separate patch in the future? Since this refactoring affects vhost code, we’re wondering it would be better to address it in a future patch to help with any regression analysis in the future.
Thank you!
—
Jag
Quoted 109 lines
hw/remote/memory-sync.c new file +23
-0+ +void deconfigure_memory_sync (RemoteMemSync *sync) +{ +memory_listener_unregister (&sync->listener); + +proxy_ml_begin (&sync->listener); +} + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc) +{ + sync->n_mr_sections =0 ; + sync->mr_sections =NULL ; + + sync->ioc = ioc; + + sync->listener.begin = proxy_ml_begin; + sync->listener.commit = proxy_ml_commit; + sync->listener.region_add = proxy_ml_region_addnop; + sync->listener.region_nop = proxy_ml_region_addnop; + sync->listener.priority =10 ; + +memory_listener_register (&sync->listener, &address_space_memory); +}hw/remote/message.c +5
-0@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);@@ -64,6 +65,10 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ; +default : error_setg(&local_err, "Unknown command (%d) received for device %s (pid=%d)",hw/remote/proxy.c +6
-0@@ -18,6 +18,8 @@#include "migration/blocker.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/memory-sync.h" +#include "qom/object.h" static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {@@ -58,6 +60,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +configure_memory_sync (&dev->sync, dev->ioc); }static void pci_proxy_dev_exit (PCIDevice *pdev)@@ -69,6 +73,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +deconfigure_memory_sync (&dev->sync); } static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,MAINTAINERS +2
-0@@ -3150,6 +3150,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/memory-sync.c +F: include/hw/remote/memory-sync.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory-sync.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)Signature · 6 lines
1.8.3.1 -- Marc-André Lureau
Re: [PATCH v12 06/19] multi-process: setup a machine object for remote device process
In reply to Marc-André LureauQuoted 37 lines · Marc-André Lureau
On Wed, Dec 2, 2020 at 12:23 AM Jagannathan Raman <[email protected]> wrote:
x-remote-machine object sets up various subsystems of the remote device process. Instantiate PCI host bridge object and initialize RAM, IO & PCI memory regions.Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/pci-host/remote.h | 1 + include/hw/remote/machine.h | 28 ++++++++++++++++++ hw/remote/machine.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/meson.build | 1 + hw/remote/meson.build | 5 ++++ 6 files changed, 106 insertions(+) create mode 100644 include/hw/remote/machine.h create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/meson.buildinclude/hw/pci-host/remote.h +1
-0@@ -25,6 +25,7 @@ typedef struct RemotePCIHost {MemoryRegion *mr_pci_mem;MemoryRegion *mr_sys_io; +MemoryRegion *mr_sys_mem;
Quoted 1 line
Why is this not part of the previous patch?
Hi Marc-Andre,
We originally defined this variable in a previous patch. But we were not using it in that patch.
Based on feedback we previously received, we moved it to this patch as it is used for the first time here.
Thank you!
—
Jag
Quoted 162 lines
} RemotePCIHost;
#endif
include/hw/remote/machine.h new file +28
-0@@ -0,0 +1,28 @@ +/* +* Remote machine configuration +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MACHINE_H +#define REMOTE_MACHINE_H + +#include "qom/object.h" +#include "hw/boards.h" +#include "hw/pci-host/remote.h" + +typedef struct RemoteMachineState { +MachineState parent_obj; + +RemotePCIHost *host; +}RemoteMachineState ; + +#define TYPE_REMOTE_MACHINE "x-remote-machine" +#define REMOTE_MACHINE (obj) \ + OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) + +#endif hw/remote/machine.c new file +44
-0@@ -0,0 +1,69 @@ +/* +* Machine for remote device +* +* This machine type is used by the remote device process in multi-process +* QEMU. QEMU device models depend on parent busses, interrupt controllers, +* memory regions, etc. The remote machine type offers this environment so +* that QEMU device models can be used as remote devices. +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/machine.h" +#include "exec/address-spaces.h" +#include "exec/memory.h" +#include "qapi/error.h" + +static void remote_machine_init (MachineState *machine) +{ +MemoryRegion *system_memory, *system_io, *pci_memory; +RemoteMachineState *s =REMOTE_MACHINE (machine); +RemotePCIHost *rem_host; + + system_memory =get_system_memory (); + system_io =get_system_io (); + + pci_memory =g_new (MemoryRegion,1 ); +memory_region_init (pci_memory,NULL ,"pci" ,UINT64_MAX ); + + rem_host =REMOTE_HOST_DEVICE (qdev_new (TYPE_REMOTE_HOST_DEVICE )); + + rem_host->mr_pci_mem = pci_memory; + rem_host->mr_sys_mem = system_memory; + rem_host->mr_sys_io = system_io; + + s->host = rem_host; + +object_property_add_child (OBJECT (s),"remote-device" ,OBJECT (rem_host));"remote-pcihost" instead ?
hw/remote/machine.c new file +10
-0+memory_region_add_subregion_overlap (system_memory,0x0 , pci_memory,-1 ); + +qdev_realize (DEVICE (rem_host),sysbus_get_default (), &error_fatal); +} + +static void remote_machine_class_init (ObjectClass *oc,void *data) +{ +MachineClass *mc =MACHINE_CLASS (oc); + + mc->init = remote_machine_init;Set
mc->desc= "Experimental remote machine" ?hw/remote/machine.c new file +15
-0+} + +static const TypeInfo remote_machine = { + .name =TYPE_REMOTE_MACHINE , + .parent =TYPE_MACHINE , + .instance_size =sizeof (RemoteMachineState), + .class_init = remote_machine_class_init, +}; + +static void remote_machine_register_types (void ) +{ +type_register_static (&remote_machine); +} + +type_init (remote_machine_register_types);MAINTAINERS +2
-0@@ -3139,6 +3139,8 @@ F: docs/devel/multi-process.rst F: tests/multiprocess/multiprocess-lsi53c895a.py F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h +F: hw/remote/machine.c +F: include/hw/remote/machine.h Build and test automation -------------------------hw/meson.build +1
-0@@ -56,6 +56,7 @@ subdir('moxie') subdir('nios2') subdir('openrisc') subdir('ppc') +subdir('remote') subdir('riscv') subdir('rx') subdir('s390x')hw/remote/meson.build new file +5
-0@@ -0,0 +1,5 @@ +remote_ss = ss.source_set() + +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) + +softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)Signature · 6 lines
1.8.3.1 -- Marc-André Lureau
Re: [PATCH v12 16/19] multi-process: Synchronize remote memory
In reply to Jag RamanQuoted 337 lines · Jag Raman
Quoted 3 lines
On Dec 8, 2020, at 8:57 AM, Marc-André Lureau <
Quoted 4 lines · [email protected]> wrote:
Hi
On Wed, Dec 2, 2020 at 12:23 AM Jagannathan Raman <[email protected]>
Quoted 7 lines · wrote:
Add memory-listener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sendsSYNC_SYSMEMmessage to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler forSYNC_SYSMEMmessage.Quoted 8 lines
Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/memory-sync.h | 27 ++++++ include/hw/remote/proxy.h | 2 + hw/remote/memory-sync.c | 210++++++++++++++++++++++++++++++++++++++++
Review
Quoted 9 lines
hw/remote/message.c | 5 + hw/remote/proxy.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 253 insertions(+) create mode 100644 include/hw/remote/memory-sync.h create mode 100644 hw/remote/memory-sync.c(quoted diff) +0
-0
b/include/hw/remote/memory-sync.hinclude/hw/remote/memory-sync.h line 4
Quoted 9 lines
new file mode 100644
index 0000000..785f76ainclude/hw/remote/memory-sync.h new file +4
-0@@ -0,0 +1,27 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
Quoted 22 lines
include/hw/remote/memory-sync.h new file +21
-0+* See the COPYING file in the top-level directory. +* +*/ + +#ifndef MEMORY_SYNC_H +#define MEMORY_SYNC_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct RemoteMemSync { +MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +}RemoteMemSync ; + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc); +void deconfigure_memory_sync (RemoteMemSync *sync);Quoted 4 lines
RemoteMemSync vs MemorySync, and function with
_memory_syncsuffixes... Naming things is hard, but trying to be consistent generally helps.My understanding is that this is a proxy-dev helper to handle memory
listening and sending
SYNC_SYSMEM.Quoted 2 lines
I would thus suggest naming it ProxyMemoryListener. It could eventually
be folded in
proxy.cQuoted 2 lines
Please try to be consistent with header naming, structure naming, type,
functions and enum prefixes etc.
Quoted 31 lines
proxy_memory_listenerisn't that long imho.include/hw/remote/memory-sync.h new file +2
-0+ +#endif include/hw/remote/proxy.h +2
-0@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/memory-sync.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"@@ -40,6 +41,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +RemoteMemSync sync;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };hw/remote/memory-sync.c new file +0
-0hw/remote/memory-sync.c line 4
Quoted 4 lines
hw/remote/memory-sync.c new file +4
-0@@ -0,0 +1,210 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
Quoted 22 lines
hw/remote/memory-sync.c new file +19
-0+* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/compiler.h" +#include "qemu/int128.h" +#include "qemu/range.h" +#include "exec/memory.h" +#include "exec/cpu-common.h" +#include "cpu.h" +#include "exec/ram_addr.h" +#include "exec/address-spaces.h" +#include "hw/remote/mpqemu-link.h" +#include "hw/remote/memory-sync.h" + +static void proxy_ml_begin (MemoryListener *listener)I suggest to rename begin -> reset
hw/remote/memory-sync.c
Quoted 2 lines
hw/remote/memory-sync.c new file +2
-0+{ + RemoteMemSync *sync = container_of(listener, RemoteMemSync,listener);
Quoted 13 lines
hw/remote/memory-sync.c new file +12
-0+int mrs; + +for (mrs =0 ; mrs < sync->n_mr_sections; mrs++) { +memory_region_unref (sync->mr_sections[mrs].mr); + } + +g_free (sync->mr_sections); + sync->mr_sections =NULL ; + sync->n_mr_sections =0 ; +} + +static int get_fd_from_hostaddr (uint64_t host,ram_addr_t *offset)Quoted 1 line
This function is very similar to
vhost_user_get_mr_data(). That suggestswe could factor the code.
Quoted 2 lines
Perhaps a new
memory_region_from_host_full(), or extend
memory_region_from_host()with an extra optional "int *fd" argument.Quoted 12 lines
hw/remote/memory-sync.c new file +11
-0+{ +MemoryRegion *mr; +ram_addr_t off; + +/** +* Assumes that the host address is a valid address as it's +* coming from the MemoryListener system. In the case host +* address is not valid, the following call would return +* the default subregion of "system_memory" region, and +* not NULL. So it's not possible to check for NULL here. +*/ hw/remote/memory-sync.c
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+ mr =memory_region_from_host ((void *)(uintptr_t )host, &off); + +if (offset) { + *offset = off; + } + +return memory_region_get_fd (mr); +} + +static bool proxy_mrs_can_merge(uint64_t host, uint64_t prev_host,
size_tsize)Quoted 13 lines
+{
This seems similar to
vhost_user_can_merge().hw/remote/memory-sync.c new file +8
-0+bool merge; +int fd1, fd2; + + fd1 =get_fd_from_hostaddr (host,NULL ); + + fd2 =get_fd_from_hostaddr (prev_host,NULL ); + + merge = (fd1 == fd2);Quoted 3 lines
This could be written in a simpler manner, ex:
if (
get_fd_from_hostaddr(host, NULL)!= get_fd_from_hostaddr(prev_host,NULL))
Quoted 6 lines
return false
hw/remote/memory-sync.c new file +2
-0+ + merge &= ((prev_host + size) == host);That check could be done early on before doing the more expensive
memory_region_from_host()callsQuoted 45 lines
hw/remote/memory-sync.c new file +44
-0+ +return merge; +} + +static bool try_merge (RemoteMemSync *sync,MemoryRegionSection *section) +{ +uint64_t mrs_size, mrs_gpa, mrs_page; +MemoryRegionSection *prev_sec; +bool merged = false; +uintptr_t mrs_host; +RAMBlock *mrs_rb; + +if (!sync->n_mr_sections) { +return false; + } + + mrs_rb = section->mr->ram_block; + mrs_page = (uint64_t )qemu_ram_pagesize (mrs_rb); + mrs_size =int128_get64 (section->size); + mrs_gpa = section->offset_within_address_space; + mrs_host = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + +if (get_fd_from_hostaddr (mrs_host,NULL ) <0 ) { +return true; + } + + mrs_host = mrs_host & ~(mrs_page -1 ); + mrs_gpa = mrs_gpa & ~(mrs_page -1 ); + mrs_size =ROUND_UP (mrs_size, mrs_page); + + prev_sec = sync->mr_sections + (sync->n_mr_sections -1 ); +uint64_t prev_gpa_start = prev_sec->offset_within_address_space; +uint64_t prev_size =int128_get64 (prev_sec->size); +uint64_t prev_gpa_end =range_get_last (prev_gpa_start, prev_size); +uint64_t prev_host_start = + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr) + + prev_sec->offset_within_region; +uint64_t prev_host_end =range_get_last (prev_host_start, prev_size); + +if (mrs_gpa <= (prev_gpa_end +1 )) { +g_assert (mrs_gpa > prev_gpa_start); + + if ((section->mr == prev_sec->mr) &&hw/remote/memory-sync.c
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+ proxy_mrs_can_merge(mrs_host, prev_host_start, + (mrs_gpa - prev_gpa_start))) { +uint64_t max_end =MAX (prev_host_end, mrs_host + mrs_size); + merged = true; + prev_sec->offset_within_address_space = +MIN (prev_gpa_start, mrs_gpa); + prev_sec->offset_within_region = +MIN (prev_host_start, mrs_host) - + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr); + prev_sec->size = int128_make64(max_end -MIN(prev_host_start,
hw/remote/memory-sync.c
Quoted 11 lines
hw/remote/memory-sync.c new file +11
-0+ mrs_host)); + } + } + +return merged; +} + +static void proxy_ml_region_addnop (MemoryListener *listener, +MemoryRegionSection *section) +{ + RemoteMemSync *sync = container_of(listener, RemoteMemSync,listener);
Quoted 15 lines
hw/remote/memory-sync.c new file +4
-0+ +if (!(memory_region_is_ram (section->mr) && + !memory_region_is_rom (section->mr))) { +return ;A bit clearer in
vhost.c:
if (memory_region_is_ram(mr)&& !memory_region_is_rom(mr)) {hw/remote/memory-sync.c new file +7
-0+ } + +if (try_merge (sync, section)) { +return ; + } + + ++sync->n_mr_sections;hw/remote/memory-sync.c
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+ sync->mr_sections =g_renew (MemoryRegionSection, sync->mr_sections, + sync->n_mr_sections); + sync->mr_sections[sync->n_mr_sections -1 ] = *section; + sync->mr_sections[sync->n_mr_sections -1 ].fv =NULL ; +memory_region_ref (section->mr); +} + +static void proxy_ml_commit (MemoryListener *listener) +{ + RemoteMemSync *sync = container_of(listener, RemoteMemSync,listener);
Quoted 12 lines
hw/remote/memory-sync.c new file +12
-0+MPQemuMsg msg; +MemoryRegionSection *section; +ram_addr_t offset; +uintptr_t host_addr; +int region; +Error *local_err =NULL ; + +memset (&msg,0 ,sizeof (MPQemuMsg)); + + msg.cmd =SYNC_SYSMEM ; + msg.num_fds = sync->n_mr_sections; + msg.size =sizeof (SyncSysmemMsg);hw/remote/memory-sync.c
Quoted 10 lines
hw/remote/memory-sync.c new file +10
-0+if (msg.num_fds >REMOTE_MAX_FDS ) { +error_report ("Number of fds is more than %d" ,REMOTE_MAX_FDS ); +return ; + } + +for (region =0 ; region < sync->n_mr_sections; region++) { + section = &sync->mr_sections[region]; + msg.data.sync_sysmem.gpas[region] = + section->offset_within_address_space; + msg.data.sync_sysmem.sizes[region] =
int128_get64(section->size);Quoted 11 lines
hw/remote/memory-sync.c new file +10
-0+ host_addr = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + msg.fds[region] =get_fd_from_hostaddr (host_addr, &offset); + msg.data.sync_sysmem.offsets[region] = offset; + } +mpqemu_msg_send (&msg, sync->ioc, &local_err); +if (local_err) { +error_report ("Error in sending command %d" , msg.cmd); + } +}Quoted 1 line
That whole complex code above duplicates much of the logic in
vhost.c.Can we try to factorize it instead?
Hi Marc-Andre,
Quoted 8 lines
Thank you for sharing your feedback!
Would it be alright if we addressed this item alone in a separate patch in the future? Since
this refactoring affects vhost code, we’re wondering it would be better to address it in a
future patch to help with any regression analysis in the future.
That's fine with me, but please leave a TODO note in the code then.
thanks
Quoted 121 lines
Thank you!
—
Jaghw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void
Quoted 37 lines
hw/remote/memory-sync.c new file +23
-0+ +void deconfigure_memory_sync (RemoteMemSync *sync) +{ +memory_listener_unregister (&sync->listener); + +proxy_ml_begin (&sync->listener); +} + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc) +{ + sync->n_mr_sections =0 ; + sync->mr_sections =NULL ; + + sync->ioc = ioc; + + sync->listener.begin = proxy_ml_begin; + sync->listener.commit = proxy_ml_commit; + sync->listener.region_add = proxy_ml_region_addnop; + sync->listener.region_nop = proxy_ml_region_addnop; + sync->listener.priority =10 ; + +memory_listener_register (&sync->listener, &address_space_memory); +}hw/remote/message.c +1
-0@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);@@ -64,6 +65,10 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void*data)
hw/remote/message.c void coroutine_fn mpqemu_remote_msg_loop_co(void *data)
Quoted 10 lines
hw/remote/message.c +4
-0case BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ; +default : error_setg(&local_err, "Unknown command (%d) received for device %s(pid=%d)",
hw/remote/proxy.c static void pci_proxy_dev_realize(PCIDevice *device,
Quoted 14 lines
hw/remote/proxy.c +2
-0@@ -18,6 +18,8 @@#include "migration/blocker.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/memory-sync.h" +#include "qom/object.h" static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {@@ -58,6 +60,8 @@ static void pci_proxy_dev_realize(PCIDevice *device,Error **errp)
Quoted 9 lines
hw/remote/proxy.c +2
-0qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +configure_memory_sync (&dev->sync, dev->ioc); }static void pci_proxy_dev_exit (PCIDevice *pdev)hw/remote/proxy.c static void pci_proxy_dev_exit(PCIDevice *pdev) · line 80
Quoted 8 lines
hw/remote/proxy.c +2
-0@@ -69,6 +73,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +deconfigure_memory_sync (&dev->sync); } static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t*val,
hw/remote/meson.build remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:
Quoted 18 lines
MAINTAINERS +2
-0@@ -3150,6 +3150,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/memory-sync.c +F: include/hw/remote/memory-sync.h Build and test automation -------------------------hw/remote/meson.build +0
-0@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:files('remote-obj.c'))
hw/remote/meson.build remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c'))
Quoted 4 lines
hw/remote/meson.build +1
-0remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:files('memory-sync.c'))
Quoted 9 lines
hw/remote/meson.build +0
-1softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss) --1.8.3.1
--
Marc-André Lureau
Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 08/19] multi-process: define MPQemuMsg format and transmission functions
In reply to Marc-André LureauQuoted 200 lines · Marc-André Lureau
Hi
Quoted 130 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks.
Define transmission functions used by proxy and by remote. There are certain restrictions on where its safe to use these functions:
- From main loop in co-routine context. Will block the main loop if not
inco-routine context; - From vCPU thread with no co-routine context and if the channel is not part of the main loop handling; - From IOThread within co-routine context, outside of co-routine context will block IOThread;Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>--- include/hw/remote/mpqemu-link.h | 60 ++++++++++ hw/remote/mpqemu-link.c | 242 ++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 305 insertions(+) create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.cinclude/hw/remote/mpqemu-link.h new file +6
-0@@ -0,0 +1,60 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/mpqemu-link.h new file +51
-0+ +#ifndef MPQEMU_LINK_H +#define MPQEMU_LINK_H + +#include "qom/object.h" +#include "qemu/thread.h" +#include "io/channel.h" + +#define REMOTE_MAX_FDS 8 + +#define MPQEMU_MSG_HDR_SIZE offsetof(MPQemuMsg, data.u64) + +/** +* MPQemuCmd: +* +* MPQemuCmd enum type to specify the command to be executed on the remote +* device. +*/ +typedef enum { +MPQEMU_CMD_INIT , +MPQEMU_CMD_MAX , +}MPQemuCmd ; + +/** +* MPQemuMsg: +* @cmd: The remote command +* @size: Size of the data to be shared +* @data: Structured data +* @fds: File descriptors to be shared with remote device +* +* MPQemuMsg Format of the message sent to the remote device from QEMU. +* +*/ +typedef struct { +int cmd; +size_t size; + +union { +uint64_t u64; + } data; + +int fds[REMOTE_MAX_FDS ]; +int num_fds; +}MPQemuMsg ; + +void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); + +bool mpqemu_msg_valid (MPQemuMsg *msg); + +#endif hw/remote/mpqemu-link.c new file +6
-0@@ -0,0 +1,242 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/mpqemu-link.c new file +15
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/module.h" +#include "hw/remote/mpqemu-link.h" +#include "qapi/error.h" +#include "qemu/iov.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" + +/* +* Send message over the ioc QIOChannel. +* This function is safe to call from: +* - From main loop in co-routine context. Will block the main loop if Quoted 9 lines
not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not part
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;Can drop the extra "From" on each line.
+ */
Quoted 7 lines
hw/remote/mpqemu-link.c new file +4
-0+void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) +{ +bool iolock =qemu_mutex_iothread_locked (); + bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()?
+ false : true;I would introduce a
qemu_in_iothread()helper (similar toqemu_in_coroutine()etc)+ Error
*local_err= NULL;Quoted 12 lines
hw/remote/mpqemu-link.c new file +11
-0+struct iovec send[2 ] = {0 }; +int *fds =NULL ; +size_t nfds =0 ; + + send[0 ].iov_base = msg; + send[0 ].iov_len =MPQEMU_MSG_HDR_SIZE ; + + send[1 ].iov_base = (void *)&msg->data; + send[1 ].iov_len = msg->size; + +if (msg->num_fds) {hw/remote/mpqemu-link.c line 57
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ nfds = msg->num_fds; + fds = msg->fds; + } +/* +* Dont use in IOThread out of co-routine context as +* it will block IOThread. +*/ +if (iothread) { +assert (qemu_in_coroutine ()); + }or simply
assert(!iothread || qemu_in_coroutine())+ /*
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+* Skip unlocking/locking iothread when in IOThread running +* in co-routine context. Co-routine context is asserted above +* for IOThread case. +* Also skip this while in a co-routine in the main context. +*/ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_unlock_iothread (); + } + + (void)qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send), fds,Quoted 3 lines
nfds,
+&local_err);
Quoted 16 lines
That extra (void) is probably unnecessary.
+
Quoted 11 lines
hw/remote/mpqemu-link.c new file +10
-0+if (iolock && !iothread && !qemu_in_coroutine ()) { +/* See above comment why skip locking here. */ +qemu_mutex_lock_iothread (); + } + +if (errp) { +error_propagate (errp, local_err); + }else if (local_err) { +error_report_err (local_err); + }
Hi Marc-Andre,
Thank you for reviewing the patches.
Quoted 2 lines
Not sure this behaviour is recommended. Instead, a trace and an
ERRP_GUARDwould be more idiomatic.
Did you mean to suggest using trace_ functions for the general use, not only the failure path. Just want to make sure I understood correctly.
Should the trace file subdirectory (in this case ./hw/remote/) be included into trace_events_subdirs of meson.build with the condition that CONFIG_MULTIPROCESS is enabled?
Something like
<snip>
config_devices_mak_file = target + '-config-devices.mak'
devconfig = keyval.load(meson.current_build_dir() / target + '-config-devices.mak') have_multiprocess = 'CONFIG_MULTIPROCESS' in devconfig
if have_multiproces
...'
</snip>
Thank you!
Elena
Quoted 230 lines
hw/remote/mpqemu-link.c
Quoted 5 lines
hw/remote/mpqemu-link.c new file +2
-0+ +return ;That's an unnecessary return. Why not return true/false based on error?
+}
Quoted 9 lines
hw/remote/mpqemu-link.c new file +5
-0+ +/* +* Read message from the ioc QIOChannel. +* This function is safe to call from: +* - From main loop in co-routine context. Will block the main loop if not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not partQuoted 9 lines
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;
+ */
+staticssize_tmpqemu_read(QIOChannel *ioc, void *buf,size_tlen, int **fds,
+size_t*nfds, Error **errp)+{
Quoted 13 lines
hw/remote/mpqemu-link.c new file +3
-0+struct iovec iov = { .iov_base = buf, .iov_len = len }; +bool iolock =qemu_mutex_iothread_locked (); + bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()hw/remote/mpqemu-link.c new file +8
-0+ ? false : true; +struct iovec *iovp = &iov; +Error *local_err =NULL ; +unsigned int niov =1 ; +size_t *l_nfds = nfds; +int **l_fds = fds; +ssize_t bytes =0 ; +size_t size;hw/remote/mpqemu-link.c line 118
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ + size = iov.iov_len; + +/* +* Dont use in IOThread out of co-routine context as +* it will block IOThread. +*/ +if (iothread) { +assert (qemu_in_coroutine ()); + }as above
Quoted 4 lines
hw/remote/mpqemu-link.c new file +3
-0+ +while (size >0 ) { + bytes = qio_channel_readv_full(ioc, iovp, niov, l_fds, l_nfds,hw/remote/mpqemu-link.c line 131
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ &local_err); +if (bytes ==QIO_CHANNEL_ERR_BLOCK ) { +/* +* Skip unlocking/locking iothread when in IOThread running +* in co-routine context. Co-routine context is asserted above +* for IOThread case. +* Also skip this while in a co-routine in the main context. +*/ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_unlock_iothread ();Why not lock the iothread at the beginning of the function and call a
readv_full_alllike we do for writes?+ }
hw/remote/mpqemu-link.c line 142
Quoted 11 lines
hw/remote/mpqemu-link.c new file +10
-0+if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_IN ); + }else { +qio_channel_wait (ioc,G_IO_IN ); + } +/* See above comment why skip locking here. */ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_lock_iothread (); + } +continue ;+ }
hw/remote/mpqemu-link.c line 155
Quoted 13 lines
hw/remote/mpqemu-link.c new file +12
-0+ +if (bytes <=0 ) { +error_propagate (errp, local_err); +return -EIO ; + } + + l_fds =NULL ; + l_nfds =NULL ; + + size -= bytes; + + (void )iov_discard_front (&iovp, &niov, bytes);needless cast
+ }
hw/remote/mpqemu-link.c line 168
Quoted 13 lines
hw/remote/mpqemu-link.c new file +12
-0+ +return len - size; +} + +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) +{ +Error *local_err =NULL ; +int *fds =NULL ; +size_t nfds =0 ; +ssize_t len; + + len = mpqemu_read(ioc, (void *)msg, MPQEMU_MSG_HDR_SIZE, &fds, &nfds,This cast is not necessary
+
&local_err);Quoted 14 lines
hw/remote/mpqemu-link.c new file +13
-0+if (!local_err) { +if (len == -EIO ) { +error_setg (&local_err,"Connection closed." ); + goto fail; + } +if (len <0 ) { +error_setg (&local_err,"Message length is less than 0" ); + goto fail; + } +if (len !=MPQEMU_MSG_HDR_SIZE ) { +error_setg (&local_err,"Message header corrupted" ); + goto fail; + }hw/remote/mpqemu-link.c line 192
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ }else { + goto fail; + } + +if (msg->size >sizeof (msg->data)) { +error_setg (&local_err,"Invalid size for message" ); + goto fail; + } + + if (mpqemu_read(ioc, (void *)&msg->data, msg->size, NULL, NULL,that one too
+
&local_err) < 0) {Quoted 79 lines
hw/remote/mpqemu-link.c new file +7
-0+ goto fail; + } + + msg->num_fds = nfds; +if (nfds >G_N_ELEMENTS (msg->fds)) { + error_setg(&local_err, + "Overflow error: received %zu fds, more than max of %dfds",
hw/remote/mpqemu-link.c new file +42
-0+ nfds, REMOTE_MAX_FDS); + goto fail; + }else if (nfds) { +memcpy (msg->fds, fds, nfds *sizeof (int )); + } + +fail: +while (local_err && nfds) { +close (fds[nfds -1 ]); + nfds--; + } + +g_free (fds); + +if (errp) { +error_propagate (errp, local_err); + }else if (local_err) { +error_report_err (local_err); + } +} + +bool mpqemu_msg_valid (MPQemuMsg *msg) +{ +if (msg->cmd >=MPQEMU_CMD_MAX && msg->cmd <0 ) { +return false; + } + +/* Verify FDs. */ +if (msg->num_fds >=REMOTE_MAX_FDS ) { +return false; + } + +if (msg->num_fds >0 ) { +for (int i =0 ; i < msg->num_fds; i++) { +if (fcntl (msg->fds[i],F_GETFL ) ==-1 ) { +return false; + } + } + } + +return true; +}MAINTAINERS +2
-0@@ -3141,6 +3141,8 @@ F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h F: hw/remote/machine.c F: include/hw/remote/machine.h +F: hw/remote/mpqemu-link.c +F: include/hw/remote/mpqemu-link.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -1,5 +1,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1Signature · 1 line
Marc-André Lureau
Re: [PATCH v12 08/19] multi-process: define MPQemuMsg format and transmission functions
Hi
Quoted 229 lines · Elena Ufimtseva
Quoted 14 lines · Marc-André Lureau
Hi
Quoted 9 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks. Define transmission functions used by proxy and by remote. There are certain restrictions on where its safe to use these functions:
- From main loop in co-routine context. Will block the main loop ifnot
Quoted 3 lines
Quoted 3 lines
in
co-routine context;- From vCPU thread with no co-routine context and if the channel is
not
Quoted 3 lines
Quoted 3 lines
part
of the main loop handling;- From IOThread within co-routine context, outside of co-routine
context
Quoted 41 lines
Quoted 41 lines
will
block IOThread;Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>--- include/hw/remote/mpqemu-link.h | 60 ++++++++++ hw/remote/mpqemu-link.c | 242 ++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 305 insertions(+) create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.cinclude/hw/remote/mpqemu-link.h new file +6
-0@@ -0,0 +1,60 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/mpqemu-link.h new file +8
-0+ +#ifndef MPQEMU_LINK_H +#define MPQEMU_LINK_H + +#include "qom/object.h" +#include "qemu/thread.h" +#include "io/channel.h" +Quoted 8 lines
include/hw/remote/mpqemu-link.h
Quoted 8 lines
include/hw/remote/mpqemu-link.h new file +8
-0+#define REMOTE_MAX_FDS 8 + +#define MPQEMU_MSG_HDR_SIZE offsetof(MPQemuMsg, data.u64) + +/** +* MPQemuCmd: +* +* MPQemuCmd enum type to specify the command to be executed on the remote
Quoted 7 lines
Quoted 7 lines
+ * device.
+ */include/hw/remote/mpqemu-link.h new file +5
-0+typedef enum { +MPQEMU_CMD_INIT , +MPQEMU_CMD_MAX , +}MPQemuCmd ; +Quoted 8 lines
Quoted 8 lines
+/**
+ * MPQemuMsg:
+ * @cmd: The remote command
+ * @size: Size of the data to be shared
+ * @data: Structured data
+ * @fds: File descriptors to be shared with remote device
+ *
+ * MPQemuMsg Format of the message sent to the remote device fromQEMU.
Quoted 101 lines
Quoted 51 lines
+ *
+ */include/hw/remote/mpqemu-link.h new file +18
-0+typedef struct { +int cmd; +size_t size; + +union { +uint64_t u64; + } data; + +int fds[REMOTE_MAX_FDS ]; +int num_fds; +}MPQemuMsg ; + +void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); + +bool mpqemu_msg_valid (MPQemuMsg *msg); + +#endif hw/remote/mpqemu-link.c new file +6
-0@@ -0,0 +1,242 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/mpqemu-link.c new file +15
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/module.h" +#include "hw/remote/mpqemu-link.h" +#include "qapi/error.h" +#include "qemu/iov.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" + +/* +* Send message over the ioc QIOChannel. +* This function is safe to call from: +* - From main loop in co-routine context. Will block the main loop if Quoted 9 lines
not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not part
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;Can drop the extra "From" on each line.
+ */
Quoted 7 lines
hw/remote/mpqemu-link.c new file +4
-0+void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) +{ +bool iolock =qemu_mutex_iothread_locked (); + bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()?
+ false : true;I would introduce a
qemu_in_iothread()helper (similar toqemu_in_coroutine()etc)+ Error
*local_err= NULL;Quoted 12 lines
hw/remote/mpqemu-link.c new file +11
-0+struct iovec send[2 ] = {0 }; +int *fds =NULL ; +size_t nfds =0 ; + + send[0 ].iov_base = msg; + send[0 ].iov_len =MPQEMU_MSG_HDR_SIZE ; + + send[1 ].iov_base = (void *)&msg->data; + send[1 ].iov_len = msg->size; + +if (msg->num_fds) {hw/remote/mpqemu-link.c
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ nfds = msg->num_fds; + fds = msg->fds; + } +/* +* Dont use in IOThread out of co-routine context as +* it will block IOThread. +*/ +if (iothread) { +assert (qemu_in_coroutine ()); + }or simply
assert(!iothread || qemu_in_coroutine())Quoted 11 lines
+ /*
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+* Skip unlocking/locking iothread when in IOThread running +* in co-routine context. Co-routine context is asserted above +* for IOThread case. +* Also skip this while in a co-routine in the main context. +*/ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_unlock_iothread (); + } + + (void)qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send),fds,
Quoted 4 lines
Quoted 3 lines
nfds,
+&local_err);Quoted 16 lines
That extra (void) is probably unnecessary.
+
Quoted 11 lines
hw/remote/mpqemu-link.c new file +10
-0+if (iolock && !iothread && !qemu_in_coroutine ()) { +/* See above comment why skip locking here. */ +qemu_mutex_lock_iothread (); + } + +if (errp) { +error_propagate (errp, local_err); + }else if (local_err) { +error_report_err (local_err); + }Hi Marc-Andre,
Thank you for reviewing the patches.
Quoted 8 lines
Quoted 1 line
Not sure this behaviour is recommended. Instead, a trace and an
ERRP_GUARDQuoted 1 line
would be more idiomatic.
Did you mean to suggest using
trace_functions for the general use, not only the
failure path. Just want to make sure I understood correctly.
That's what I would suggest for error handling: (not tested)
hw/remote/mpqemu-link.c +1 -0
@@ -31,10 +31,10 @@*/ void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) { +ERRP_GUARD ();bool iolock =qemu_mutex_iothread_locked (); bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context() ?
false : true;
- Error *local_err = NULL;
struct iovec send[2] = {0};
int *fds = NULL;
size_t nfds = 0;
@@ -66,21 +66,15 @@ void mpqemu_msg_send(MPQemuMsg *msg, QIOChannel *ioc,
Error **errp)
qemu_mutex_unlock_iothread();
}- (void)qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send), fds,
nfds,
- &local_err);
+ if (qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send), fds,
nfds, errp) == -1) {
+ trace_mpqemu_io_error(msg, ioc, error_get_pretty(*errp));
+ }
if (iolock && !iothread && !qemu_in_coroutine()) {
/* See above comment why skip locking here. */
qemu_mutex_lock_iothread();
}- if (errp) {
- error_propagate(errp, local_err);
- } else if (local_err) {
- error_report_err(local_err);
- }
-
- return;
}
Quoted 14 lines
Should the trace file subdirectory (in this case ./hw/remote/) be included into
trace_events_subdirsof meson.build with the condition thatCONFIG_MULTIPROCESSis enabled?Something like
<snip>
config_devices_mak_file= target + '-config-devices.mak' devconfig =keyval.load(meson.current_build_dir() / target + '-config-devices.mak')have_multiprocess= 'CONFIG_MULTIPROCESS' in devconfig
Quoted 5 lines
if
have_multiproces
...'</snip>
That shouldn't be necessary, do like the other hw/ traces, adding themself to trace_events_subdirs.
Signature · 1 line
Marc-André Lureau
Quoted 11 lines · Peter Maydell
Quoted 10 lines · Stefan Hajnoczi
Quoted 2 lines · Jagannathan Raman
This is the v12 of the patchset. Thank you very much for the review of the v11 of the series.
I'm in favor of merging this for QEMU 6.0. The command-line interface has the x- prefix so QEMU is not committing to a stable interface. Changes needed to support additional device types or to switch to the vfio-user protocol can be made later.
Quoted 6 lines
Quoted 2 lines
Jag, Elena, JJ: I suggest getting your GPG key to Peter Maydell so you can send multi-process QEMU pull requests.
I would prefer to see this going through the tree of an established QEMU developer who's already sending pullrequests, at least initially.
Once the discussion has completed I can send the patches in a pull request.
I don't want to be the bottleneck for all multi-process QEMU patches in the future though. That's why I think the authors should be able to send pull requests on their own after the initial code is merged. Much of this work is isolated an only affects multi-process QEMU and the feature is marked experimental. There is little risk of introducing instability for non-multi-process QEMU users/developers. Hence why this is a new subsystem and has MAINTAINERS files entries.
Does that sound good?
Stefan
Quoted 8 lines · Stefan Hajnoczi
Quoted 3 lines · Peter Maydell
I would prefer to see this going through the tree of an established QEMU developer who's already sending pullrequests, at least initially.
Once the discussion has completed I can send the patches in a pull request.
Quoted 7 lines
I don't want to be the bottleneck for all multi-process QEMU patches in the future though. That's why I think the authors should be able to send pull requests on their own after the initial code is merged. Much of this work is isolated an only affects multi-process QEMU and the feature is marked experimental. There is little risk of introducing instability for non-multi-process QEMU users/developers. Hence why this is a new subsystem and has
MAINTAINERSfiles entries.
My reasoning is basically that new pull-request senders are more work for me, because I have to make sure they have a GPG key set up, and then examine pull requests pretty carefully to check they're well-formed, all the sign-offs are correct, the changes aren't touching areas of the codebase that they shouldn't, and so on. That's particularly painful if the first pull request that comes through is a massive one rather than "here's a small number of patches with some bug fixes".
thanks
-- PMM
Re: [PATCH v12 08/19] multi-process: define MPQemuMsg format and transmission functions
In reply to Marc-André LureauQuoted 264 lines · Marc-André Lureau
Hi
Quoted 229 lines · Elena Ufimtseva
Quoted 14 lines · Marc-André Lureau
Hi
Quoted 9 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks. Define transmission functions used by proxy and by remote. There are certain restrictions on where its safe to use these functions:
- From main loop in co-routine context. Will block the main loop ifnot
Quoted 3 lines
Quoted 3 lines
in
co-routine context;- From vCPU thread with no co-routine context and if the channel is
not
Quoted 3 lines
Quoted 3 lines
part
of the main loop handling;- From IOThread within co-routine context, outside of co-routine
context
Quoted 41 lines
Quoted 41 lines
will
block IOThread;Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>--- include/hw/remote/mpqemu-link.h | 60 ++++++++++ hw/remote/mpqemu-link.c | 242 ++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 4 files changed, 305 insertions(+) create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.cinclude/hw/remote/mpqemu-link.h new file +6
-0@@ -0,0 +1,60 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/mpqemu-link.h new file +8
-0+ +#ifndef MPQEMU_LINK_H +#define MPQEMU_LINK_H + +#include "qom/object.h" +#include "qemu/thread.h" +#include "io/channel.h" +Quoted 8 lines
include/hw/remote/mpqemu-link.h
Quoted 8 lines
include/hw/remote/mpqemu-link.h new file +8
-0+#define REMOTE_MAX_FDS 8 + +#define MPQEMU_MSG_HDR_SIZE offsetof(MPQemuMsg, data.u64) + +/** +* MPQemuCmd: +* +* MPQemuCmd enum type to specify the command to be executed on the remote
Quoted 7 lines
Quoted 7 lines
+ * device.
+ */include/hw/remote/mpqemu-link.h new file +5
-0+typedef enum { +MPQEMU_CMD_INIT , +MPQEMU_CMD_MAX , +}MPQemuCmd ; +Quoted 8 lines
Quoted 8 lines
+/**
+ * MPQemuMsg:
+ * @cmd: The remote command
+ * @size: Size of the data to be shared
+ * @data: Structured data
+ * @fds: File descriptors to be shared with remote device
+ *
+ * MPQemuMsg Format of the message sent to the remote device fromQEMU.
Quoted 101 lines
Quoted 51 lines
+ *
+ */include/hw/remote/mpqemu-link.h new file +18
-0+typedef struct { +int cmd; +size_t size; + +union { +uint64_t u64; + } data; + +int fds[REMOTE_MAX_FDS ]; +int num_fds; +}MPQemuMsg ; + +void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +void mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); + +bool mpqemu_msg_valid (MPQemuMsg *msg); + +#endif hw/remote/mpqemu-link.c new file +6
-0@@ -0,0 +1,242 @@ +/* +* Communication channel between QEMU and remote device process +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/mpqemu-link.c new file +15
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/module.h" +#include "hw/remote/mpqemu-link.h" +#include "qapi/error.h" +#include "qemu/iov.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" + +/* +* Send message over the ioc QIOChannel. +* This function is safe to call from: +* - From main loop in co-routine context. Will block the main loop if Quoted 9 lines
not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not part
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;Can drop the extra "From" on each line.
+ */
Quoted 7 lines
hw/remote/mpqemu-link.c new file +4
-0+void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) +{ +bool iolock =qemu_mutex_iothread_locked (); + bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()?
+ false : true;I would introduce a
qemu_in_iothread()helper (similar toqemu_in_coroutine()etc)+ Error
*local_err= NULL;Quoted 12 lines
hw/remote/mpqemu-link.c new file +11
-0+struct iovec send[2 ] = {0 }; +int *fds =NULL ; +size_t nfds =0 ; + + send[0 ].iov_base = msg; + send[0 ].iov_len =MPQEMU_MSG_HDR_SIZE ; + + send[1 ].iov_base = (void *)&msg->data; + send[1 ].iov_len = msg->size; + +if (msg->num_fds) {hw/remote/mpqemu-link.c
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+ nfds = msg->num_fds; + fds = msg->fds; + } +/* +* Dont use in IOThread out of co-routine context as +* it will block IOThread. +*/ +if (iothread) { +assert (qemu_in_coroutine ()); + }or simply
assert(!iothread || qemu_in_coroutine())Quoted 11 lines
+ /*
Quoted 10 lines
hw/remote/mpqemu-link.c new file +10
-0+* Skip unlocking/locking iothread when in IOThread running +* in co-routine context. Co-routine context is asserted above +* for IOThread case. +* Also skip this while in a co-routine in the main context. +*/ +if (iolock && !iothread && !qemu_in_coroutine ()) { +qemu_mutex_unlock_iothread (); + } + + (void)qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send),fds,
Quoted 4 lines
Quoted 3 lines
nfds,
+&local_err);Quoted 16 lines
That extra (void) is probably unnecessary.
+
Quoted 11 lines
hw/remote/mpqemu-link.c new file +10
-0+if (iolock && !iothread && !qemu_in_coroutine ()) { +/* See above comment why skip locking here. */ +qemu_mutex_lock_iothread (); + } + +if (errp) { +error_propagate (errp, local_err); + }else if (local_err) { +error_report_err (local_err); + }Hi Marc-Andre,
Thank you for reviewing the patches.
Quoted 8 lines
Quoted 1 line
Not sure this behaviour is recommended. Instead, a trace and an
ERRP_GUARDQuoted 1 line
would be more idiomatic.
Did you mean to suggest using
trace_functions for the general use, not only the
failure path. Just want to make sure I understood correctly.That's what I would suggest for error handling: (not tested)
hw/remote/mpqemu-link.c +1
-0@@ -31,10 +31,10 @@*/ void mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp) { +ERRP_GUARD ();bool iolock =qemu_mutex_iothread_locked (); bool iothread = qemu_get_current_aio_context() ==
qemu_get_aio_context()?false : true;hw/remote/mpqemu-link.c +0
-1-Error *local_err =NULL ;struct iovec send[2 ] = {0 };int *fds =NULL ;size_t nfds =0 ;@@ -66,21 +66,15 @@ void mpqemu_msg_send(MPQemuMsg *msg, QIOChannel *ioc, Error **errp)
hw/remote/mpqemu-link.c void mpqemu_msg_send(MPQemuMsg *msg, QIOChannel *ioc, · line 74
Quoted 10 lines
hw/remote/mpqemu-link.c +3
-2qemu_mutex_unlock_iothread (); } - (void )qio_channel_writev_full_all (ioc, send,G_N_ELEMENTS (send), fds, nfds, - &local_err); + if (qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send), fds, nfds, errp) == -1) { +trace_mpqemu_io_error (msg, ioc,error_get_pretty (*errp)); + }
Thanks, that answers my question. I didn't see the examples that convinced me using trace events as the means of error reporting. Now I do :)
Quoted 32 lines
hw/remote/mpqemu-link.c +0
-7if (iolock && !iothread && !qemu_in_coroutine ()) {/* See above comment why skip locking here. */ qemu_mutex_lock_iothread (); } -if (errp) { -error_propagate (errp, local_err); - }else if (local_err) { -error_report_err (local_err); - } - -return ; }Quoted 18 lines
Should the trace file subdirectory (in this case ./hw/remote/) be included into
trace_events_subdirsof meson.build with the condition thatCONFIG_MULTIPROCESSis enabled?Something like
<snip>
config_devices_mak_file= target + '-config-devices.mak' devconfig =keyval.load(meson.current_build_dir() / target + '-config-devices.mak')have_multiprocess= 'CONFIG_MULTIPROCESS' in devconfig
Quoted 8 lines
Quoted 5 lines
if
have_multiproces
...'</snip>
That shouldn't be necessary, do like the other hw/ traces, adding themself to
trace_events_subdirs.
Got it, thank you!
Quoted 4 lines
Signature · 2 lines
-- Marc-André Lureau
Quoted 17 lines · Peter Maydell
Quoted 8 lines · Stefan Hajnoczi
Quoted 3 lines · Peter Maydell
I would prefer to see this going through the tree of an established QEMU developer who's already sending pullrequests, at least initially.
Once the discussion has completed I can send the patches in a pull request.
Quoted 7 lines
I don't want to be the bottleneck for all multi-process QEMU patches in the future though. That's why I think the authors should be able to send pull requests on their own after the initial code is merged. Much of this work is isolated an only affects multi-process QEMU and the feature is marked experimental. There is little risk of introducing instability for non-multi-process QEMU users/developers. Hence why this is a new subsystem and has
MAINTAINERSfiles entries.
Quoted 8 lines
My reasoning is basically that new pull-request senders are more work for me, because I have to make sure they have a GPG key set up, and then examine pull requests pretty carefully to check they're well-formed, all the sign-offs are correct, the changes aren't touching areas of the codebase that they shouldn't, and so on. That's particularly painful if the first pull request that comes through is a massive one rather than "here's a small number of patches with some bug fixes".
Thanks for explaining. I will merge this series when review has finished and send you a pull request.
Stefan
Re: [PATCH v12 16/19] multi-process: Synchronize remote memory
In reply to Marc-André LureauQuoted 326 lines · Marc-André Lureau
Quoted 311 lines · Jag Raman
On Dec 8, 2020, at 8:57 AM, Marc-André Lureau <[email protected]> wrote:
Hi
On Wed, Dec 2, 2020 at 12:23 AM Jagannathan Raman <[email protected]> wrote:
Add memory-listener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sendsSYNC_SYSMEMmessage to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler forSYNC_SYSMEMmessage.Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Elena Ufimtseva <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/memory-sync.h | 27 ++++++ include/hw/remote/proxy.h | 2 + hw/remote/memory-sync.c | 210 ++++++++++++++++++++++++++++++++++++++++ hw/remote/message.c | 5 + hw/remote/proxy.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 253 insertions(+) create mode 100644 include/hw/remote/memory-sync.h create mode 100644 hw/remote/memory-sync.cinclude/hw/remote/memory-sync.h new file +25
-0@@ -0,0 +1,27 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef MEMORY_SYNC_H +#define MEMORY_SYNC_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct RemoteMemSync { +MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +}RemoteMemSync ; + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc); +void deconfigure_memory_sync (RemoteMemSync *sync);RemoteMemSync vs MemorySync, and function with
_memory_syncsuffixes... Naming things is hard, but trying to be consistent generally helps.My understanding is that this is a proxy-dev helper to handle memory listening and sending
SYNC_SYSMEM.I would thus suggest naming it ProxyMemoryListener. It could eventually be folded in
proxy.cPlease try to be consistent with header naming, structure naming, type, functions and enum prefixes etc.
proxy_memory_listenerisn't that long imho.include/hw/remote/memory-sync.h new file +2
-0+ +#endif include/hw/remote/proxy.h +2
-0@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/memory-sync.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"@@ -40,6 +41,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +RemoteMemSync sync;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };hw/remote/memory-sync.c new file +23
-0@@ -0,0 +1,210 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/compiler.h" +#include "qemu/int128.h" +#include "qemu/range.h" +#include "exec/memory.h" +#include "exec/cpu-common.h" +#include "cpu.h" +#include "exec/ram_addr.h" +#include "exec/address-spaces.h" +#include "hw/remote/mpqemu-link.h" +#include "hw/remote/memory-sync.h" + +static void proxy_ml_begin (MemoryListener *listener)I suggest to rename begin -> reset
hw/remote/memory-sync.c new file +14
-0+{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); +int mrs; + +for (mrs =0 ; mrs < sync->n_mr_sections; mrs++) { +memory_region_unref (sync->mr_sections[mrs].mr); + } + +g_free (sync->mr_sections); + sync->mr_sections =NULL ; + sync->n_mr_sections =0 ; +} + +static int get_fd_from_hostaddr (uint64_t host,ram_addr_t *offset)This function is very similar to
vhost_user_get_mr_data(). That suggests we could factor the code.Perhaps a new
memory_region_from_host_full(), or extendmemory_region_from_host()with an extra optional "int *fd" argument.hw/remote/memory-sync.c new file +22
-0+{ +MemoryRegion *mr; +ram_addr_t off; + +/** +* Assumes that the host address is a valid address as it's +* coming from the MemoryListener system. In the case host +* address is not valid, the following call would return +* the default subregion of "system_memory" region, and +* not NULL. So it's not possible to check for NULL here. +*/ + mr =memory_region_from_host ((void *)(uintptr_t )host, &off); + +if (offset) { + *offset = off; + } + +return memory_region_get_fd (mr); +} + +static bool proxy_mrs_can_merge (uint64_t host,uint64_t prev_host,size_t size) +{This seems similar to
vhost_user_can_merge().hw/remote/memory-sync.c new file +8
-0+bool merge; +int fd1, fd2; + + fd1 =get_fd_from_hostaddr (host,NULL ); + + fd2 =get_fd_from_hostaddr (prev_host,NULL ); + + merge = (fd1 == fd2);This could be written in a simpler manner, ex:
if (
get_fd_from_hostaddr(host, NULL)!=get_fd_from_hostaddr(prev_host, NULL))
return falsehw/remote/memory-sync.c new file +2
-0+ + merge &= ((prev_host + size) == host);That check could be done early on before doing the more expensive
memory_region_from_host()callshw/remote/memory-sync.c new file +69
-0+ +return merge; +} + +static bool try_merge (RemoteMemSync *sync,MemoryRegionSection *section) +{ +uint64_t mrs_size, mrs_gpa, mrs_page; +MemoryRegionSection *prev_sec; +bool merged = false; +uintptr_t mrs_host; +RAMBlock *mrs_rb; + +if (!sync->n_mr_sections) { +return false; + } + + mrs_rb = section->mr->ram_block; + mrs_page = (uint64_t )qemu_ram_pagesize (mrs_rb); + mrs_size =int128_get64 (section->size); + mrs_gpa = section->offset_within_address_space; + mrs_host = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + +if (get_fd_from_hostaddr (mrs_host,NULL ) <0 ) { +return true; + } + + mrs_host = mrs_host & ~(mrs_page -1 ); + mrs_gpa = mrs_gpa & ~(mrs_page -1 ); + mrs_size =ROUND_UP (mrs_size, mrs_page); + + prev_sec = sync->mr_sections + (sync->n_mr_sections -1 ); +uint64_t prev_gpa_start = prev_sec->offset_within_address_space; +uint64_t prev_size =int128_get64 (prev_sec->size); +uint64_t prev_gpa_end =range_get_last (prev_gpa_start, prev_size); +uint64_t prev_host_start = + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr) + + prev_sec->offset_within_region; +uint64_t prev_host_end =range_get_last (prev_host_start, prev_size); + +if (mrs_gpa <= (prev_gpa_end +1 )) { +g_assert (mrs_gpa > prev_gpa_start); + +if ((section->mr == prev_sec->mr) && +proxy_mrs_can_merge (mrs_host, prev_host_start, + (mrs_gpa - prev_gpa_start))) { +uint64_t max_end =MAX (prev_host_end, mrs_host + mrs_size); + merged = true; + prev_sec->offset_within_address_space = +MIN (prev_gpa_start, mrs_gpa); + prev_sec->offset_within_region = +MIN (prev_host_start, mrs_host) - + (uintptr_t )memory_region_get_ram_ptr (prev_sec->mr); + prev_sec->size =int128_make64 (max_end -MIN (prev_host_start, + mrs_host)); + } + } + +return merged; +} + +static void proxy_ml_region_addnop (MemoryListener *listener, +MemoryRegionSection *section) +{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); + +if (!(memory_region_is_ram (section->mr) && + !memory_region_is_rom (section->mr))) { +return ;A bit clearer in
vhost.c:
if (memory_region_is_ram(mr)&& !memory_region_is_rom(mr)) {hw/remote/memory-sync.c new file +49
-0+ } + +if (try_merge (sync, section)) { +return ; + } + + ++sync->n_mr_sections; + sync->mr_sections =g_renew (MemoryRegionSection, sync->mr_sections, + sync->n_mr_sections); + sync->mr_sections[sync->n_mr_sections -1 ] = *section; + sync->mr_sections[sync->n_mr_sections -1 ].fv =NULL ; +memory_region_ref (section->mr); +} + +static void proxy_ml_commit (MemoryListener *listener) +{ +RemoteMemSync *sync =container_of (listener, RemoteMemSync, listener); +MPQemuMsg msg; +MemoryRegionSection *section; +ram_addr_t offset; +uintptr_t host_addr; +int region; +Error *local_err =NULL ; + +memset (&msg,0 ,sizeof (MPQemuMsg)); + + msg.cmd =SYNC_SYSMEM ; + msg.num_fds = sync->n_mr_sections; + msg.size =sizeof (SyncSysmemMsg); +if (msg.num_fds >REMOTE_MAX_FDS ) { +error_report ("Number of fds is more than %d" ,REMOTE_MAX_FDS ); +return ; + } + +for (region =0 ; region < sync->n_mr_sections; region++) { + section = &sync->mr_sections[region]; + msg.data.sync_sysmem.gpas[region] = + section->offset_within_address_space; + msg.data.sync_sysmem.sizes[region] =int128_get64 (section->size); + host_addr = (uintptr_t )memory_region_get_ram_ptr (section->mr) + + section->offset_within_region; + msg.fds[region] =get_fd_from_hostaddr (host_addr, &offset); + msg.data.sync_sysmem.offsets[region] = offset; + } +mpqemu_msg_send (&msg, sync->ioc, &local_err); +if (local_err) { +error_report ("Error in sending command %d" , msg.cmd); + } +}Quoted 1 line
That whole complex code above duplicates much of the logic in
vhost.c. Can we try to factorize it instead?Hi Marc-Andre,
Thank you for sharing your feedback!
Quoted 7 lines
Would it be alright if we addressed this item alone in a separate patch in the future? Since this refactoring affects vhost code, we’re wondering it would be better to address it in a future patch to help with any regression analysis in the future.
That's fine with me, but please leave a TODO note in the code then.
thanks
Thank you very much for confirming!
—
Jag
Quoted 120 lines
Thank you!
—
JagQuoted 109 lines
hw/remote/memory-sync.c new file +23
-0+ +void deconfigure_memory_sync (RemoteMemSync *sync) +{ +memory_listener_unregister (&sync->listener); + +proxy_ml_begin (&sync->listener); +} + +void configure_memory_sync (RemoteMemSync *sync,QIOChannel *ioc) +{ + sync->n_mr_sections =0 ; + sync->mr_sections =NULL ; + + sync->ioc = ioc; + + sync->listener.begin = proxy_ml_begin; + sync->listener.commit = proxy_ml_commit; + sync->listener.region_add = proxy_ml_region_addnop; + sync->listener.region_nop = proxy_ml_region_addnop; + sync->listener.priority =10 ; + +memory_listener_register (&sync->listener, &address_space_memory); +}hw/remote/message.c +5
-0@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg);@@ -64,6 +65,10 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ; +default : error_setg(&local_err, "Unknown command (%d) received for device %s (pid=%d)",hw/remote/proxy.c +6
-0@@ -18,6 +18,8 @@#include "migration/blocker.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/memory-sync.h" +#include "qom/object.h" static void proxy_set_socket (PCIProxyDev *pdev,int fd,Error **errp) {@@ -58,6 +60,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +configure_memory_sync (&dev->sync, dev->ioc); }static void pci_proxy_dev_exit (PCIDevice *pdev)@@ -69,6 +73,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +deconfigure_memory_sync (&dev->sync); } static int config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,MAINTAINERS +2
-0@@ -3150,6 +3150,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/memory-sync.c +F: include/hw/remote/memory-sync.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory-sync.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)Signature · 6 lines
1.8.3.1 -- Marc-André LureauSignature · 1 line
Marc-André Lureau