Initial support for multi-process Qemu
Subsystems (2)
PATCH v13 3/19 memory: alloc RAM from file at offset
In reply to Jagannathan RamanAllow RAM MemoryRegion to be created from an offset in a file, instead of allocating at offset of 0 by default. This is needed to synchronize RAM between QEMU & remote process.
--- include/exec/memory.h | 2 ++ include/exec/ram_addr.h | 2 +- include/qemu/mmap-alloc.h | 4 +++- backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 ++- softmmu/memory.c | 3 ++- softmmu/physmem.c | 11 +++++++---- util/mmap-alloc.c | 7 ++++--- util/oslib-posix.c | 2 +- 9 files changed, 23 insertions(+), 13 deletions(-)
include/exec/memory.h +2 -0
@@ -990,6 +990,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr,* @size: size of the region. * @share: %true if memory must be mmaped with the MAP_SHARED flag * @fd: the fd to mmap. +* @offset: offset within the file referenced by fd * @errp: pointer to Error*, to store an error if it happens. * * Note that this function does not do anything to cause the data in the
@@ -1001,6 +1002,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp);#endif
include/exec/ram_addr.h +1 -1
@@ -121,7 +121,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, Error **errp);RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp); +off_t offset,Error **errp);RAMBlock *qemu_ram_alloc_from_ptr (ram_addr_t size,void *host,MemoryRegion *mr,Error **errp);
include/qemu/mmap-alloc.h +3 -1
@@ -16,6 +16,7 @@ size_t qemu_mempath_getpagesize(const char *mem_path);* otherwise, the alignment in use will be determined by QEMU. * @shared: map has RAM_SHARED flag. * @is_pmem: map has RAM_PMEM flag. +* @map_offset: map starts at offset of map_offset from the start of fd * * Return: * On success, return a pointer to the mapped area.
@@ -25,7 +26,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem); + bool is_pmem, + off_t map_offset);void qemu_ram_munmap (int fd,void *ptr,size_t size);
backends/hostmem-memfd.c +1 -1
@@ -55,7 +55,7 @@ memfd_backend_memory_alloc(HostMemoryBackend *backend, Error **errp) name =host_memory_backend_get_name (backend);memory_region_init_ram_from_fd (&backend->mr,OBJECT (backend), name, backend->size, - backend->share, fd, errp); + backend->share, fd,0 , errp);g_free (name); }
hw/misc/ivshmem.c +2 -1
@@ -494,7 +494,8 @@ static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)/* mmap the region and map into the BAR2 */ memory_region_init_ram_from_fd (&s->server_bar2,OBJECT (s), -"ivshmem.bar2" , size, true, fd, &local_err); +"ivshmem.bar2" , size, true, fd,0 , + &local_err);if (local_err) {error_propagate (errp, local_err);return ;
softmmu/memory.c +2 -1
@@ -1612,6 +1612,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr,uint64_t size, bool share, int fd, + ram_addr_t offset, Error **errp) { Error *err = NULL;
@@ -1621,7 +1622,7 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, mr->destructor = memory_region_destructor_ram; mr->ram_block =qemu_ram_alloc_from_fd (size, mr, share ?RAM_SHARED :0 , - fd, &err); + fd, offset, &err); mr->dirty_log_mask =tcg_enabled () ? (1 <<DIRTY_MEMORY_CODE ) :0 ;if (err) { mr->size =int128_zero ();
softmmu/physmem.c +7 -4
@@ -1461,6 +1461,7 @@ static void *file_ram_alloc(RAMBlock *block,ram_addr_t memory, int fd, bool truncate, + off_t offset, Error **errp) { void *area;
@@ -1511,7 +1512,8 @@ static void *file_ram_alloc(RAMBlock *block, } area =qemu_ram_mmap (fd, memory, block->mr->align, - block->flags &RAM_SHARED , block->flags &RAM_PMEM ); + block->flags &RAM_SHARED , block->flags &RAM_PMEM , + offset);if (area ==MAP_FAILED ) {error_setg_errno (errp, errno,"unable to map backing store for guest RAM" );
@@ -1943,7 +1945,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp, bool shared)#ifdef CONFIG_POSIX RAMBlock *qemu_ram_alloc_from_fd (ram_addr_t size,MemoryRegion *mr,uint32_t ram_flags,int fd, -Error **errp) +off_t offset,Error **errp) {RAMBlock *new_block;Error *local_err =NULL ;
@@ -1996,7 +1998,8 @@ RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr, new_block->used_length = size; new_block->max_length = size; new_block->flags = ram_flags; - new_block->host =file_ram_alloc (new_block, size, fd, !file_size, errp); + new_block->host =file_ram_alloc (new_block, size, fd, !file_size, offset, + errp);if (!new_block->host) {g_free (new_block);return NULL ;
@@ -2026,7 +2029,7 @@ RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,return NULL ; } - block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd, errp); + block =qemu_ram_alloc_from_fd (size, mr, ram_flags, fd,0 , errp);if (!block) {if (created) {unlink (mem_path);
util/mmap-alloc.c +4 -3
@@ -86,7 +86,8 @@ void *qemu_ram_mmap(int fd,size_t size, size_t align, bool shared, - bool is_pmem) + bool is_pmem, + off_t map_offset) { int flags;int map_sync_flags =0 ;
@@ -147,7 +148,7 @@ void *qemu_ram_mmap(int fd, offset =QEMU_ALIGN_UP ((uintptr_t )guardptr, align) - (uintptr_t )guardptr; ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags | map_sync_flags, fd,0 ); + flags | map_sync_flags, fd, map_offset);if (ptr ==MAP_FAILED && map_sync_flags) {if (errno ==ENOTSUP ) {
@@ -172,7 +173,7 @@ void *qemu_ram_mmap(int fd,* we will remove these flags to handle compatibility. */ ptr =mmap (guardptr + offset, size,PROT_READ |PROT_WRITE , - flags, fd,0 ); + flags, fd, map_offset); }if (ptr ==MAP_FAILED ) {
util/oslib-posix.c +1 -1
@@ -227,7 +227,7 @@ void *qemu_memalign(size_t alignment, size_t size)void *qemu_anon_ram_alloc (size_t size,uint64_t *alignment,bool shared) {size_t align =QEMU_VMALLOC_ALIGN ; -void *ptr =qemu_ram_mmap (-1 , size, align, shared, false); +void *ptr =qemu_ram_mmap (-1 , size, align, shared, false,0 );if (ptr ==MAP_FAILED ) {return NULL ;
Signature · 1 line
1.8.3.1
PATCH v13 7/19 multi-process: add qio channel function to transmit data and fds
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Adds QIO channel functions that transmit and receive iovs along with fds.
--- include/io/channel.h | 50 +++++++++++++++++++++++++++++++++++++++++ io/channel.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-)
include/io/channel.h +50 -0
@@ -774,4 +774,54 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_write, void *opaque); +/** +* qio_channel_readv_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to read data to +* @niov: the length of the @iov array +* @fds: an array of file handles to read +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_readvv_full but will attempt +* to read all data specified (file handles and memory regions). +* The function will wait for all requested data +* to be read, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were read, or -1 on error +*/ + +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp); + +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp); +#endif /* QIO_CHANNEL_H */
io/channel.c +62 -1
90 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v13 18/19 multi-process: Retrieve PCI info from remote process
In reply to Jagannathan RamanRetrieve PCI configuration info about the remote device and configure the Proxy PCI object based on the returned information
--- hw/remote/proxy.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+)
hw/remote/proxy.c +84 -0
@@ -24,6 +24,8 @@#include "sysemu/kvm.h" #include "util/event_notifier-posix.c" +static void probe_pci_info (PCIDevice *dev,Error **errp); +static void proxy_intx_update (PCIDevice *pci_dev) {PCIProxyDev *dev =PCI_PROXY_DEV (pci_dev);
@@ -76,6 +78,7 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp) {ERRP_GUARD ();PCIProxyDev *dev =PCI_PROXY_DEV (device); +uint8_t *pci_conf = device->config;int fd;if (!dev->fd) {
@@ -98,9 +101,14 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + pci_conf[PCI_LATENCY_TIMER ] =0xff ; + pci_conf[PCI_INTERRUPT_PIN ] =0x01 ; +proxy_memory_listener_configure (&dev->proxy_listener, dev->ioc);setup_irqfd (dev); + +probe_pci_info (PCI_DEVICE (dev), errp); }static void pci_proxy_dev_exit (PCIDevice *pdev)
@@ -266,3 +274,79 @@ const MemoryRegionOps proxy_mr_ops = { .max_access_size = 8, }, }; + +static void probe_pci_info (PCIDevice *dev,Error **errp) +{ +PCIDeviceClass *pc =PCI_DEVICE_GET_CLASS (dev); +uint32_t orig_val, new_val, base_class, val; +PCIProxyDev *pdev =PCI_PROXY_DEV (dev); +DeviceClass *dc =DEVICE_CLASS (pc); +uint8_t type; +int i, size; + +config_op_send (pdev,PCI_VENDOR_ID , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->vendor_id = (uint16_t )val; + +config_op_send (pdev,PCI_DEVICE_ID , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->device_id = (uint16_t )val; + +config_op_send (pdev,PCI_CLASS_DEVICE , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->class_id = (uint16_t )val; + +config_op_send (pdev,PCI_SUBSYSTEM_ID , &val,2 ,MPQEMU_CMD_PCI_CFGREAD ); + pc->subsystem_id = (uint16_t )val; + + base_class = pc->class_id >>4 ; +switch (base_class) { +case PCI_BASE_CLASS_BRIDGE : +set_bit (DEVICE_CATEGORY_BRIDGE , dc->categories); +break ; +case PCI_BASE_CLASS_STORAGE : +set_bit (DEVICE_CATEGORY_STORAGE , dc->categories); +break ; +case PCI_BASE_CLASS_NETWORK : +set_bit (DEVICE_CATEGORY_NETWORK , dc->categories); +break ; +case PCI_BASE_CLASS_INPUT : +set_bit (DEVICE_CATEGORY_INPUT , dc->categories); +break ; +case PCI_BASE_CLASS_DISPLAY : +set_bit (DEVICE_CATEGORY_DISPLAY , dc->categories); +break ; +case PCI_BASE_CLASS_PROCESSOR : +set_bit (DEVICE_CATEGORY_CPU , dc->categories); +break ; +default : +set_bit (DEVICE_CATEGORY_MISC , dc->categories); +break ; + } + +for (i =0 ; i <PCI_NUM_REGIONS ; i++) { +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &orig_val,4 , +MPQEMU_CMD_PCI_CFGREAD ); + new_val =0xffffffff ; +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &new_val,4 , +MPQEMU_CMD_PCI_CFGWRITE ); +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &new_val,4 , +MPQEMU_CMD_PCI_CFGREAD ); + size = (~(new_val &0xFFFFFFF0 )) +1 ; +config_op_send (pdev,PCI_BASE_ADDRESS_0 + (4 * i), &orig_val,4 , +MPQEMU_CMD_PCI_CFGWRITE ); + type = (new_val &0x1 ) ? +PCI_BASE_ADDRESS_SPACE_IO :PCI_BASE_ADDRESS_SPACE_MEMORY ; + +if (size) { + g_autofree char *name; + pdev->region[i].dev = pdev; + pdev->region[i].present = true; +if (type ==PCI_BASE_ADDRESS_SPACE_MEMORY ) { + pdev->region[i].memory = true; + } + name =g_strdup_printf ("bar-region-%d" , i); +memory_region_init_io (&pdev->region[i].mr,OBJECT (pdev), + &proxy_mr_ops, &pdev->region[i], + name, size); +pci_register_bar (dev, i, type, &pdev->region[i].mr); + } + } +}
Signature · 1 line
1.8.3.1
PATCH v13 4/19 multi-process: Add config option for multi-process QEMU
In reply to Jagannathan RamanAdd a configuration option to separate multi-process code
--- accel/Kconfig | 1 + hw/Kconfig | 1 + hw/remote/Kconfig | 3 +++ 3 files changed, 5 insertions(+) create mode 100644 hw/remote/Kconfig
accel/Kconfig +1 -0
@@ -3,6 +3,7 @@ config TCG config KVM bool + select MULTIPROCESS config XEN bool
hw/Kconfig +1 -0
@@ -27,6 +27,7 @@ source pci-host/Kconfig source pcmcia/Kconfig source pci/Kconfig source rdma/Kconfig +source remote/Kconfig source rtc/Kconfig source scsi/Kconfig source sd/Kconfig
hw/remote/Kconfig new file +3 -0
@@ -0,0 +1,3 @@ +config MULTIPROCESS + bool + depends on PCI && KVM
Signature · 1 line
1.8.3.1
PATCH v13 13/19 multi-process: add proxy communication functions
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
--- include/hw/remote/mpqemu-link.h | 4 ++++ hw/remote/mpqemu-link.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+)
include/hw/remote/mpqemu-link.h +4 -0
@@ -15,6 +15,8 @@#include "qemu/thread.h" #include "io/channel.h" #include "exec/hwaddr.h" +#include "io/channel-socket.h" +#include "hw/remote/proxy.h" #define REMOTE_MAX_FDS 8
@@ -68,6 +70,8 @@ typedef struct {bool mpqemu_msg_send (MPQemuMsg *msg,QIOChannel *ioc,Error **errp);bool mpqemu_msg_recv (MPQemuMsg *msg,QIOChannel *ioc,Error **errp); +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp);bool mpqemu_msg_valid (MPQemuMsg *msg);#endif
hw/remote/mpqemu-link.c +34 -0
@@ -182,6 +182,40 @@ fail:return ret; } +/* +* Send msg and wait for a reply with command code RET_MSG. +* Returns the message received of size u64 or UINT64_MAX +* on error. +* Called from VCPU thread in non-coroutine context. +* Used by the Proxy object to communicate to remote processes. +*/ +uint64_t mpqemu_msg_send_and_await_reply (MPQemuMsg *msg,PCIProxyDev *pdev, +Error **errp) +{ +ERRP_GUARD (); +MPQemuMsg msg_reply = {0 }; +uint64_t ret =UINT64_MAX ; + +assert (!qemu_in_coroutine ()); + +QEMU_LOCK_GUARD (&pdev->io_mutex); +if (!mpqemu_msg_send (msg, pdev->ioc, errp)) { +return ret; + } + +if (!mpqemu_msg_recv (&msg_reply, pdev->ioc, errp)) { +return ret; + } + +if (!mpqemu_msg_valid (&msg_reply)) { +error_setg (errp,"ERROR: Invalid reply received for command %d" , + msg->cmd); +return ret; + } + +return msg_reply.data.u64; +} +bool mpqemu_msg_valid (MPQemuMsg *msg) {if (msg->cmd >=MPQEMU_CMD_MAX && msg->cmd <0 ) {
Signature · 1 line
1.8.3.1
PATCH v13 16/19 multi-process: Synchronize remote memory
In reply to Jagannathan RamanAdd ProxyMemoryListener object which is used to keep the view of the RAM in sync between QEMU and remote process.
A MemoryListener is registered for system-memory AddressSpace. The listener sends SYNC_SYSMEM message to the remote process when memory listener commits the changes to memory, the remote process receives the message and processes it in the handler for SYNC_SYSMEM message.
--- include/hw/remote/proxy-memory-listener.h | 28 ++++ include/hw/remote/proxy.h | 2 + hw/remote/message.c | 4 + hw/remote/proxy-memory-listener.c | 227 ++++++++++++++++++++++++++++++ hw/remote/proxy.c | 6 + MAINTAINERS | 2 + hw/remote/meson.build | 1 + 7 files changed, 270 insertions(+) create mode 100644 include/hw/remote/proxy-memory-listener.h create mode 100644 hw/remote/proxy-memory-listener.c
include/hw/remote/proxy-memory-listener.h new file +28 -0
@@ -0,0 +1,28 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef PROXY_MEMORY_LISTENER_H +#define PROXY_MEMORY_LISTENER_H + +#include "exec/memory.h" +#include "io/channel.h" + +typedef struct ProxyMemoryListener { +MemoryListener listener; + +int n_mr_sections; +MemoryRegionSection *mr_sections; + +QIOChannel *ioc; +}ProxyMemoryListener ; + +void proxy_memory_listener_configure (ProxyMemoryListener *proxy_listener, +QIOChannel *ioc); +void proxy_memory_listener_deconfigure (ProxyMemoryListener *proxy_listener); + +#endif
include/hw/remote/proxy.h +2 -0
@@ -11,6 +11,7 @@#include "hw/pci/pci.h" #include "io/channel.h" +#include "hw/remote/proxy-memory-listener.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"
@@ -40,6 +41,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; +ProxyMemoryListener proxy_listener;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };
hw/remote/message.c +4 -0
@@ -17,6 +17,7 @@#include "sysemu/runstate.h" #include "hw/pci/pci.h" #include "exec/memattrs.h" +#include "hw/remote/memory.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg,Error **errp);
@@ -61,6 +62,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case MPQEMU_CMD_BAR_READ :process_bar_read (com->ioc, &msg, &local_err);break ; +case MPQEMU_CMD_SYNC_SYSMEM : +remote_sysmem_reconfig (&msg, &local_err); +break ;default :error_setg (&local_err, "Unknown command (%d) received for device %s"
hw/remote/proxy-memory-listener.c new file +227 -0
227 lines not on this page · raw message
hw/remote/proxy.c +6 -0
@@ -18,6 +18,8 @@#include "migration/blocker.h" #include "hw/remote/mpqemu-link.h" #include "qemu/error-report.h" +#include "hw/remote/proxy-memory-listener.h" +#include "qom/object.h" static void pci_proxy_dev_realize (PCIDevice *device,Error **errp) {
@@ -44,6 +46,8 @@ static void pci_proxy_dev_realize(PCIDevice *device, Error **errp)qemu_mutex_init (&dev->io_mutex);qio_channel_set_blocking (dev->ioc, true,NULL ); + +proxy_memory_listener_configure (&dev->proxy_listener, dev->ioc); }static void pci_proxy_dev_exit (PCIDevice *pdev)
@@ -57,6 +61,8 @@ static void pci_proxy_dev_exit(PCIDevice *pdev)migrate_del_blocker (dev->migration_blocker);error_free (dev->migration_blocker); + +proxy_memory_listener_deconfigure (&dev->proxy_listener); } static void config_op_send(PCIProxyDev *pdev, uint32_t addr, uint32_t *val,
MAINTAINERS +2 -0
@@ -3165,6 +3165,8 @@ F: include/hw/remote/memory.h F: hw/remote/memory.c F: hw/remote/proxy.c F: include/hw/remote/proxy.h +F: hw/remote/proxy-memory-listener.c +F: include/hw/remote/proxy-memory-listener.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -7,5 +7,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy-memory-listener.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v13 11/19 multi-process: setup memory manager for remote device
In reply to Jagannathan RamanSyncSysMemMsg message format is defined. It is used to send file descriptors of the RAM regions to remote device.
RAM on the remote device is configured with a set of file descriptors. Old RAM regions are deleted and new regions, each with an fd, is added to the RAM.
--- include/hw/remote/memory.h | 19 ++++++++++++ include/hw/remote/mpqemu-link.h | 10 +++++++ hw/remote/memory.c | 65 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 11 +++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 2 ++ 6 files changed, 109 insertions(+) create mode 100644 include/hw/remote/memory.h create mode 100644 hw/remote/memory.c
include/hw/remote/memory.h new file +19 -0
@@ -0,0 +1,19 @@ +/* +* Memory manager for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MEMORY_H +#define REMOTE_MEMORY_H + +#include "exec/hwaddr.h" +#include "hw/remote/mpqemu-link.h" + +void remote_sysmem_reconfig (MPQemuMsg *msg,Error **errp); + +#endif
include/hw/remote/mpqemu-link.h +10 -0
@@ -14,6 +14,7 @@#include "qom/object.h" #include "qemu/thread.h" #include "io/channel.h" +#include "exec/hwaddr.h" #define REMOTE_MAX_FDS 8
@@ -30,9 +31,16 @@* */ typedef enum { +MPQEMU_CMD_SYNC_SYSMEM ,MPQEMU_CMD_MAX , }MPQemuCmd ; +typedef struct { +hwaddr gpas[REMOTE_MAX_FDS ]; +uint64_t sizes[REMOTE_MAX_FDS ]; +off_t offsets[REMOTE_MAX_FDS ]; +}SyncSysmemMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -43,12 +51,14 @@ typedef enum {* MPQemuMsg Format of the message sent to the remote device from QEMU. * */ + typedef struct { int cmd; size_t size; union { uint64_t u64; + SyncSysmemMsg sync_sysmem; } data; int fds[REMOTE_MAX_FDS];
hw/remote/memory.c new file +65 -0
65 lines not on this page · raw message
hw/remote/mpqemu-link.c +11 -0
@@ -201,5 +201,16 @@ bool mpqemu_msg_valid(MPQemuMsg *msg) } } +/* Verify message specific fields. */ +switch (msg->cmd) { +case MPQEMU_CMD_SYNC_SYSMEM : +if (msg->num_fds ==0 || msg->size !=sizeof (SyncSysmemMsg)) { +return false; + } +break ; +default : +break ; + } +return true; }
MAINTAINERS +2 -0
@@ -3161,6 +3161,8 @@ F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c F: hw/remote/remote-obj.c +F: include/hw/remote/memory.h +F: hw/remote/memory.c Build and test automation -------------------------
hw/remote/meson.build +2 -0
@@ -5,4 +5,6 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) + softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v13 14/19 multi-process: Forward PCI config space acceses to the remote process
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
The Proxy Object sends the PCI config space accesses as messages to the remote process over the communication channel
--- include/hw/remote/mpqemu-link.h | 10 +++++++ hw/remote/message.c | 60 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 8 +++++- hw/remote/proxy.c | 55 +++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-)
include/hw/remote/mpqemu-link.h +10 -0
@@ -34,6 +34,9 @@*/ typedef enum {MPQEMU_CMD_SYNC_SYSMEM , +MPQEMU_CMD_RET , +MPQEMU_CMD_PCI_CFGWRITE , +MPQEMU_CMD_PCI_CFGREAD ,MPQEMU_CMD_MAX , }MPQemuCmd ;
@@ -43,6 +46,12 @@ typedef struct {off_t offsets[REMOTE_MAX_FDS ]; } SyncSysmemMsg; +typedef struct { +uint32_t addr; +uint32_t val; +int len; +}PciConfDataMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -60,6 +69,7 @@ typedef struct {union {uint64_t u64; +PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; } data;
hw/remote/message.c +60 -0
75 lines not on this page · raw message
hw/remote/mpqemu-link.c +7 -1
@@ -207,7 +207,7 @@ uint64_t mpqemu_msg_send_and_await_reply(MPQemuMsg *msg, PCIProxyDev *pdev,return ret; } -if (!mpqemu_msg_valid (&msg_reply)) { +if (!mpqemu_msg_valid (&msg_reply) || msg_reply.cmd !=MPQEMU_CMD_RET ) {error_setg (errp,"ERROR: Invalid reply received for command %d" , msg->cmd);return ret;
@@ -242,6 +242,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case MPQEMU_CMD_PCI_CFGWRITE : +case MPQEMU_CMD_PCI_CFGREAD : +if (msg->size !=sizeof (PciConfDataMsg)) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +55 -0
73 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v13 5/19 multi-process: setup PCI host bridge for remote device
In reply to Jagannathan RamanPCI host bridge is setup for the remote device process. It is implemented using remote-pcihost object. It is an extension of the PCI host bridge setup by QEMU.
Remote-pcihost configures a PCI bus which could be used by the remote PCI device to latch on to.
--- include/hw/pci-host/remote.h | 30 ++++++++++++++++++ hw/pci-host/remote.c | 75 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/pci-host/Kconfig | 3 ++ hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 1 + 6 files changed, 112 insertions(+) create mode 100644 include/hw/pci-host/remote.h create mode 100644 hw/pci-host/remote.c
include/hw/pci-host/remote.h new file +30 -0
@@ -0,0 +1,30 @@ +/* +* PCI Host for remote device +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_PCIHOST_H +#define REMOTE_PCIHOST_H + +#include "exec/memory.h" +#include "hw/pci/pcie_host.h" + +#define TYPE_REMOTE_PCIHOST "remote-pcihost" +#define REMOTE_PCIHOST (obj) \ + OBJECT_CHECK(RemotePCIHost, (obj), TYPE_REMOTE_PCIHOST) + +typedef struct RemotePCIHost { +/*< private >*/ +PCIExpressHost parent_obj; +/*< public >*/ + +MemoryRegion *mr_pci_mem; +MemoryRegion *mr_sys_io; +}RemotePCIHost ; + +#endif
hw/pci-host/remote.c new file +75 -0
75 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3153,6 +3153,8 @@ M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst F: docs/multi-process.rst +F: hw/pci-host/remote.c +F: include/hw/pci-host/remote.h Build and test automation -------------------------
hw/pci-host/Kconfig +3 -0
@@ -60,3 +60,6 @@ config PCI_BONITO select PCI select UNIMP bool + +config REMOTE_PCIHOST + bool
hw/pci-host/meson.build +1 -0
@@ -9,6 +9,7 @@ pci_ss.add(when: 'CONFIG_PCI_EXPRESS_XILINX', if_true: files('xilinx-pcie.c')) pci_ss.add(when: 'CONFIG_PCI_I440FX', if_true: files('i440fx.c')) pci_ss.add(when: 'CONFIG_PCI_SABRE', if_true: files('sabre.c')) pci_ss.add(when: 'CONFIG_XEN_IGD_PASSTHROUGH', if_true: files('xen_igd_pt.c')) +pci_ss.add(when: 'CONFIG_REMOTE_PCIHOST', if_true: files('remote.c')) # PPC devices pci_ss.add(when: 'CONFIG_PREP_PCI', if_true: files('prep.c'))
hw/remote/Kconfig +1 -0
@@ -1,3 +1,4 @@ config MULTIPROCESS bool depends on PCI && KVM + select REMOTE_PCIHOST
Signature · 1 line
1.8.3.1
PATCH v13 1/19 multi-process: add the concept description to docs/devel/qemu-multiprocess
In reply to Jagannathan RamanFrom: John G Johnson <[email protected]>
--- docs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 +++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 7 + 3 files changed, 974 insertions(+) create mode 100644 docs/devel/multi-process.rst
docs/devel/index.rst +1 -0
@@ -35,3 +35,4 @@ Contents: clocks qom block-coroutine-wrapper + multi-process
docs/devel/multi-process.rst new file +966 -0
966 lines not on this page · raw message
MAINTAINERS +7 -0
@@ -3146,6 +3146,13 @@ S: Maintained F: hw/semihosting/ F: include/hw/semihosting/ +Multi-process QEMU +M: Elena Ufimtseva <[email protected]> +M: Jagannathan Raman <[email protected]> +M: John G Johnson <[email protected]> +S: Maintained +F: docs/devel/multi-process.rst + Build and test automation ------------------------- Build and test automation
Signature · 1 line
1.8.3.1
PATCH v13 17/19 multi-process: create IOHUB object to handle irq
In reply to Jagannathan RamanIOHUB object is added to manage PCI IRQs. It uses KVM_IRQFD ioctl to create irqfd to injecting PCI interrupts to the guest. IOHUB object forwards the irqfd to the remote process. Remote process uses this fd to directly send interrupts to the guest, bypassing QEMU.
--- include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++++++++++++++ include/hw/remote/machine.h | 3 + include/hw/remote/mpqemu-link.h | 1 + include/hw/remote/proxy.h | 4 ++ hw/remote/iohub.c | 119 ++++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 10 ++++ hw/remote/message.c | 4 ++ hw/remote/mpqemu-link.c | 5 ++ hw/remote/proxy.c | 56 +++++++++++++++++++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + 12 files changed, 250 insertions(+) create mode 100644 include/hw/remote/iohub.h create mode 100644 hw/remote/iohub.c
include/hw/pci/pci_ids.h +3 -0
@@ -192,6 +192,9 @@#define PCI_DEVICE_ID_SUN_SIMBA 0x5000#define PCI_DEVICE_ID_SUN_SABRE 0xa000 +#define PCI_VENDOR_ID_ORACLE 0x108e +#define PCI_DEVICE_ID_REMOTE_IOHUB 0xb000 +#define PCI_VENDOR_ID_CMD 0x1095#define PCI_DEVICE_ID_CMD_646 0x0646
include/hw/remote/iohub.h new file +42 -0
42 lines not on this page · raw message
include/hw/remote/machine.h +3 -0
@@ -15,11 +15,14 @@#include "hw/boards.h" #include "hw/pci-host/remote.h" #include "io/channel.h" +#include "hw/remote/iohub.h" typedef struct RemoteMachineState {MachineState parent_obj;RemotePCIHost *host; + +RemoteIOHubState iohub; }RemoteMachineState ;/* Used to pass to co-routine device and ioc. */
include/hw/remote/mpqemu-link.h +1 -0
@@ -39,6 +39,7 @@ typedef enum {MPQEMU_CMD_PCI_CFGREAD ,MPQEMU_CMD_BAR_WRITE ,MPQEMU_CMD_BAR_READ , +MPQEMU_CMD_SET_IRQFD ,MPQEMU_CMD_MAX , } MPQemuCmd;
include/hw/remote/proxy.h +4 -0
@@ -12,6 +12,7 @@#include "hw/pci/pci.h" #include "io/channel.h" #include "hw/remote/proxy-memory-listener.h" +#include "qemu/event_notifier.h" #define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev"
@@ -42,6 +43,9 @@ struct PCIProxyDev {QIOChannel *ioc;Error *migration_blocker;ProxyMemoryListener proxy_listener; +int virq; +EventNotifier intr; +EventNotifier resample;ProxyMemoryRegion region[PCI_NUM_REGIONS ]; };
hw/remote/iohub.c new file +119 -0
119 lines not on this page · raw message
hw/remote/machine.c +10 -0
@@ -20,12 +20,15 @@#include "exec/address-spaces.h" #include "exec/memory.h" #include "qapi/error.h" +#include "hw/pci/pci_host.h" +#include "hw/remote/iohub.h" static void remote_machine_init (MachineState *machine) {MemoryRegion *system_memory, *system_io, *pci_memory;RemoteMachineState *s =REMOTE_MACHINE (machine);RemotePCIHost *rem_host; +PCIHostState *pci_host; system_memory =get_system_memory (); system_io =get_system_io ();
@@ -45,6 +48,13 @@ static void remote_machine_init(MachineState *machine)memory_region_add_subregion_overlap (system_memory,0x0 , pci_memory,-1 );qdev_realize (DEVICE (rem_host),sysbus_get_default (), &error_fatal); + + pci_host =PCI_HOST_BRIDGE (rem_host); + +remote_iohub_init (&s->iohub); + +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, + &s->iohub,REMOTE_IOHUB_NB_PIRQS ); }static void remote_machine_class_init (ObjectClass *oc,void *data)
hw/remote/message.c +4 -0
@@ -18,6 +18,7 @@#include "hw/pci/pci.h" #include "exec/memattrs.h" #include "hw/remote/memory.h" +#include "hw/remote/iohub.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg,Error **errp);
@@ -65,6 +66,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case MPQEMU_CMD_SYNC_SYSMEM :remote_sysmem_reconfig (&msg, &local_err);break ; +case MPQEMU_CMD_SET_IRQFD : +process_set_irqfd_msg (pci_dev, &msg); +break ;default :error_setg (&local_err, "Unknown command (%d) received for device %s"
hw/remote/mpqemu-link.c +5 -0
@@ -254,6 +254,11 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case MPQEMU_CMD_SET_IRQFD : +if (msg->size || (msg->num_fds !=2 )) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +56 -0
74 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3167,6 +3167,8 @@ F: hw/remote/proxy.c F: include/hw/remote/proxy.h F: hw/remote/proxy-memory-listener.c F: include/hw/remote/proxy-memory-listener.h +F: hw/remote/iohub.c +F: include/hw/remote/iohub.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -5,6 +5,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy-memory-listener.c'))
Signature · 1 line
1.8.3.1
PATCH v13 12/19 multi-process: introduce proxy object
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Defines a PCI Device proxy object as a child of TYPE_PCI_DEVICE.
--- include/hw/remote/proxy.h | 36 +++++++++++++++++++ hw/remote/proxy.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 1 + 4 files changed, 130 insertions(+) create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/remote/proxy.c
include/hw/remote/proxy.h new file +36 -0
@@ -0,0 +1,36 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef PROXY_H +#define PROXY_H + +#include "hw/pci/pci.h" +#include "io/channel.h" + +#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev" + +#define PCI_PROXY_DEV (obj) \ + OBJECT_CHECK(PCIProxyDev, (obj), TYPE_PCI_PROXY_DEV) +typedef struct PCIProxyDev PCIProxyDev ; + +struct PCIProxyDev { +PCIDevice parent_dev; +char *fd; + +/* +* Mutex used to protect the QIOChannel fd from +* the concurrent access by the VCPUs since proxy +* blocks while awaiting for the replies from the +* process remote. +*/ +QemuMutex io_mutex; +QIOChannel *ioc; +Error *migration_blocker; +}; + +#endif /* PROXY_H */
hw/remote/proxy.c new file +91 -0
91 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3163,6 +3163,8 @@ F: hw/remote/message.c F: hw/remote/remote-obj.c F: include/hw/remote/memory.h F: hw/remote/memory.c +F: hw/remote/proxy.c +F: include/hw/remote/proxy.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -4,6 +4,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c'))
Signature · 1 line
1.8.3.1
PATCH v13 2/19 multi-process: add configure and usage information
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Adds documentation explaining the command-line arguments needed to use multi-process.
--- docs/multi-process.rst | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 65 insertions(+) create mode 100644 docs/multi-process.rst
docs/multi-process.rst new file +64 -0
@@ -0,0 +1,64 @@ +Multi-process QEMU +================== + +This document describes how to configure and use multi-process qemu. +For the design document refer to docs/devel/qemu-multiprocess. + +1) Configuration +---------------- + +multi-process is enabled by default for targets that enable KVM + + +2) Usage +-------- + +Multi-process QEMU requires an orchestrator to launch. + +Following is a description of command-line used to launch mpqemu. + +* Orchestrator: + + - The Orchestrator creates a unix socketpair + + - It launches the remote process and passes one of the + sockets to it via command-line. + + - It then launches QEMU and specifies the other socket as an option + to the Proxy device object + +* Remote Process: + + - QEMU can enter remote process mode by using the "remote" machine + option. + + - The orchestrator creates a "remote-object" with details about + the device and the file descriptor for the device + + - The remaining options are no different from how one launches QEMU with + devices. + + - Example command-line for the remote process is as follows: + + /usr/bin/qemu-system-x86_64 \ + -machine x-remote \ + -device lsi53c895a,id=lsi0 \ + -drive id=drive_image2,file=/build/ol7-nvme-test-1.qcow2 \ + -device scsi-hd,id=drive2,drive=drive_image2,bus=lsi0.0,scsi-id=0 \ + -object x-remote-object,id=robj1,devid=lsi1,fd=4, + +* QEMU: + + - Since parts of the RAM are shared between QEMU & remote process, a + memory-backend-memfd is required to facilitate this, as follows: + + -object memory-backend-memfd,id=mem,size=2G + + - A "x-pci-proxy-dev" device is created for each of the PCI devices emulated + in the remote process. A "socket" sub-option specifies the other end of + unix channel created by orchestrator. The "id" sub-option must be specified + and should be the same as the "id" specified for the remote PCI device + + - Example commandline for QEMU is as follows: + + -device x-pci-proxy-dev,id=lsi0,socket=3
MAINTAINERS +1 -0
@@ -3152,6 +3152,7 @@ M: Jagannathan Raman <[email protected]> M: John G Johnson <[email protected]> S: Maintained F: docs/devel/multi-process.rst +F: docs/multi-process.rst Build and test automation -------------------------
Signature · 1 line
1.8.3.1
PATCH v13 9/19 multi-process: Initialize message handler in remote device
In reply to Jagannathan RamanInitializes the message handler function in the remote process. It is called whenever there's an event pending on QIOChannel that registers this function.
--- include/hw/remote/machine.h | 9 +++++++ hw/remote/message.c | 57 +++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 4 files changed, 68 insertions(+) create mode 100644 hw/remote/message.c
include/hw/remote/machine.h +9 -0
@@ -14,6 +14,7 @@#include "qom/object.h" #include "hw/boards.h" #include "hw/pci-host/remote.h" +#include "io/channel.h" typedef struct RemoteMachineState { MachineState parent_obj;
@@ -21,8 +22,16 @@ typedef struct RemoteMachineState {RemotePCIHost *host; } RemoteMachineState; +/* Used to pass to co-routine device and ioc. */ +typedef struct RemoteCommDev { +PCIDevice *dev; +QIOChannel *ioc; +}RemoteCommDev ; +#define TYPE_REMOTE_MACHINE "x-remote-machine"#define REMOTE_MACHINE (obj) \ OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) +void coroutine_fn mpqemu_remote_msg_loop_co(void *data); +#endif
hw/remote/message.c new file +57 -0
57 lines not on this page · raw message
MAINTAINERS +1 -0
@@ -3159,6 +3159,7 @@ F: hw/remote/machine.c F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h +F: hw/remote/message.c Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -2,5 +2,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v13 6/19 multi-process: setup a machine object for remote device process
In reply to Jagannathan Ramanx-remote-machine object sets up various subsystems of the remote device process. Instantiate PCI host bridge object and initialize RAM, IO & PCI memory regions.
--- include/hw/pci-host/remote.h | 1 + include/hw/remote/machine.h | 28 ++++++++++++++++++ hw/remote/machine.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/meson.build | 1 + hw/remote/meson.build | 5 ++++ 6 files changed, 107 insertions(+) create mode 100644 include/hw/remote/machine.h create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/meson.build
include/hw/pci-host/remote.h +1 -0
@@ -25,6 +25,7 @@ typedef struct RemotePCIHost {MemoryRegion *mr_pci_mem;MemoryRegion *mr_sys_io; + MemoryRegion *mr_sys_mem; } RemotePCIHost;#endif
include/hw/remote/machine.h new file +28 -0
@@ -0,0 +1,28 @@ +/* +* Remote machine configuration +* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_MACHINE_H +#define REMOTE_MACHINE_H + +#include "qom/object.h" +#include "hw/boards.h" +#include "hw/pci-host/remote.h" + +typedef struct RemoteMachineState { +MachineState parent_obj; + +RemotePCIHost *host; +}RemoteMachineState ; + +#define TYPE_REMOTE_MACHINE "x-remote-machine" +#define REMOTE_MACHINE (obj) \ + OBJECT_CHECK(RemoteMachineState, (obj), TYPE_REMOTE_MACHINE) + +#endif
hw/remote/machine.c new file +70 -0
70 lines not on this page · raw message
MAINTAINERS +2 -0
@@ -3155,6 +3155,8 @@ F: docs/devel/multi-process.rst F: docs/multi-process.rst F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h +F: hw/remote/machine.c +F: include/hw/remote/machine.h Build and test automation -------------------------
hw/meson.build +1 -0
@@ -56,6 +56,7 @@ subdir('moxie') subdir('nios2') subdir('openrisc') subdir('ppc') +subdir('remote') subdir('riscv') subdir('rx') subdir('s390x')
hw/remote/meson.build new file +5 -0
@@ -0,0 +1,5 @@ +remote_ss = ss.source_set() + +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) + +softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v13 10/19 multi-process: Associate fd of a PCIDevice with its object
In reply to Jagannathan RamanAssociate the file descriptor for a PCIDevice in remote process with DeviceState object.
--- hw/remote/remote-obj.c | 206 +++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 3 files changed, 208 insertions(+) create mode 100644 hw/remote/remote-obj.c
hw/remote/remote-obj.c new file +206 -0
@@ -0,0 +1,206 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later. +* +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/error-report.h" +#include "qemu/notify.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" +#include "qemu/sockets.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT (obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS (obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS (klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}RemoteObjectClass ; + +typedef struct { +/* private */ +Object parent; + +Notifier machine_done; + +int32_t fd; +char *devid; + +QIOChannel *ioc; + +DeviceState *dev; +DeviceListener listener; +}RemoteObject ; + +static void remote_object_set_fd (Object *obj,Visitor *v,const char *name, +void *opaque,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); +int32_t value; + +if (!visit_type_int32 (v, name, &value, errp)) { +return ; + } + +if (!fd_is_socket (value)) { + o->fd =-1 ; +error_setg (errp,"File descriptor %d is not a socket" , value); +return ; + } + + o->fd = value; +} + +static void remote_object_set_devid (Object *obj,const char *str,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); + + o->devid =g_strdup (str); +} + +static void remote_object_release (DeviceListener *listener,DeviceState *dev) +{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_release; +device_listener_register (&o->listener); + +/* co-routine should free this. */ + comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + }; + + co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev); +qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { +error_report ("Reached maximum number of devices: %u" , k->max_devs); +return ; + } + + o->ioc =NULL ; + o->fd =-1 ; + o->devid =NULL ; + + k->nr_devs++; + + o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +device_listener_unregister (&o->listener); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc)); + + k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + +/* +* Limit number of supported devices to 1. This is done to avoid devices +* from one VM accessing the RAM of another VM. This is done until we +* start using separate address spaces for individual devices. +*/ + k->max_devs =1 ; + k->nr_devs =0 ; + +object_class_property_add (klass,"fd" ,"int32_t" ,NULL , + remote_object_set_fd,NULL ,NULL ); +object_class_property_add_str (klass,"devid" ,NULL , + remote_object_set_devid); +} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);
MAINTAINERS +1 -0
@@ -3160,6 +3160,7 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: hw/remote/remote-obj.c Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
Signature · 1 line
1.8.3.1
PATCH v13 15/19 multi-process: PCI BAR read/write handling for proxy & remote endpoints
In reply to Jagannathan RamanProxy device object implements handler for PCI BAR writes and reads. The handler uses BAR_WRITE/BAR_READ message to communicate to the remote process with the BAR address and value to be written/read. The remote process implements handler for BAR_WRITE/BAR_READ message.
--- include/hw/remote/mpqemu-link.h | 10 +++++ include/hw/remote/proxy.h | 10 +++++ hw/remote/message.c | 83 +++++++++++++++++++++++++++++++++++++++++ hw/remote/mpqemu-link.c | 6 +++ hw/remote/proxy.c | 60 +++++++++++++++++++++++++++++ 5 files changed, 169 insertions(+)
include/hw/remote/mpqemu-link.h +10 -0
@@ -37,6 +37,8 @@ typedef enum {MPQEMU_CMD_RET ,MPQEMU_CMD_PCI_CFGWRITE ,MPQEMU_CMD_PCI_CFGREAD , +MPQEMU_CMD_BAR_WRITE , +MPQEMU_CMD_BAR_READ ,MPQEMU_CMD_MAX , } MPQemuCmd;
@@ -52,6 +54,13 @@ typedef struct {int len; } PciConfDataMsg; +typedef struct { +hwaddr addr; +uint64_t val; +unsigned size; +bool memory; +}BarAccessMsg ; +/** * MPQemuMsg: * @cmd: The remote command
@@ -71,6 +80,7 @@ typedef struct {uint64_t u64;PciConfDataMsg pci_conf_data;SyncSysmemMsg sync_sysmem; +BarAccessMsg bar_access; } data;int fds[REMOTE_MAX_FDS ];
include/hw/remote/proxy.h +10 -0
@@ -16,8 +16,17 @@#define PCI_PROXY_DEV (obj) \ OBJECT_CHECK(PCIProxyDev, (obj), TYPE_PCI_PROXY_DEV) +typedef struct PCIProxyDev PCIProxyDev ; +typedef struct ProxyMemoryRegion { +PCIProxyDev *dev; +MemoryRegion mr; +bool memory; +bool present; +uint8_t type; +}ProxyMemoryRegion ; +struct PCIProxyDev {PCIDevice parent_dev;char *fd;
@@ -31,6 +40,7 @@ struct PCIProxyDev {QemuMutex io_mutex;QIOChannel *ioc;Error *migration_blocker; + ProxyMemoryRegion region[PCI_NUM_REGIONS]; };#endif /* PROXY_H */
hw/remote/message.c +83 -0
103 lines not on this page · raw message
hw/remote/mpqemu-link.c +6 -0
@@ -248,6 +248,12 @@ bool mpqemu_msg_valid(MPQemuMsg *msg)return false; }break ; +case MPQEMU_CMD_BAR_WRITE : +case MPQEMU_CMD_BAR_READ : +if ((msg->size !=sizeof (BarAccessMsg)) || (msg->num_fds !=0 )) { +return false; + } +break ;default :break ; }
hw/remote/proxy.c +60 -0
63 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v13 8/19 multi-process: define MPQemuMsg format and transmission functions
In reply to Jagannathan RamanFrom: Elena Ufimtseva <[email protected]>
Defines MPQemuMsg, which is the message that is sent to the remote process. This message is sent over QIOChannel and is used to command the remote process to perform various tasks. Define transmission functions used by proxy and by remote.
--- meson.build | 1 + hw/remote/trace.h | 1 + include/hw/remote/mpqemu-link.h | 63 ++++++++++++ include/sysemu/iothread.h | 6 ++ hw/remote/mpqemu-link.c | 205 ++++++++++++++++++++++++++++++++++++++++ iothread.c | 6 ++ MAINTAINERS | 2 + hw/remote/meson.build | 1 + hw/remote/trace-events | 4 + 9 files changed, 289 insertions(+) create mode 100644 hw/remote/trace.h create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 hw/remote/mpqemu-link.c create mode 100644 hw/remote/trace-events
meson.build +1 -0
@@ -1483,6 +1483,7 @@ if have_system 'net', 'softmmu', 'ui', + 'hw/remote', ] endif trace_events_subdirs += [
hw/remote/trace.h new file +1 -0
@@ -0,0 +1 @@ +#include "trace/trace-hw_remote.h"
include/hw/remote/mpqemu-link.h new file +63 -0
63 lines not on this page · raw message
include/sysemu/iothread.h +6 -0
@@ -57,4 +57,10 @@ IOThread *iothread_create(const char *id, Error **errp);void iothread_stop (IOThread *iothread);void iothread_destroy (IOThread *iothread); +/* +* Returns true if executing withing IOThread context, +* false otherwise. +*/ +bool qemu_in_iothread (void ); +#endif /* IOTHREAD_H */
hw/remote/mpqemu-link.c new file +205 -0
205 lines not on this page · raw message
iothread.c +6 -0
@@ -375,3 +375,9 @@ IOThread *iothread_by_id(const char *id) {return IOTHREAD (object_resolve_path_type (id,TYPE_IOTHREAD ,NULL )); } + +bool qemu_in_iothread (void ) +{ +return qemu_get_current_aio_context () ==qemu_get_aio_context () ? + false : true; +}
MAINTAINERS +2 -0
@@ -3157,6 +3157,8 @@ F: hw/pci-host/remote.c F: include/hw/pci-host/remote.h F: hw/remote/machine.c F: include/hw/remote/machine.h +F: hw/remote/mpqemu-link.c +F: include/hw/remote/mpqemu-link.h Build and test automation -------------------------
hw/remote/meson.build +1 -0
@@ -1,5 +1,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
hw/remote/trace-events new file +4 -0
@@ -0,0 +1,4 @@ +# multi-process trace events + +mpqemu_send_io_error(int cmd, int size, int nfds) "send command %d size %d, %d file descriptors to remote process" +mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, %d file descriptors to remote process"
Signature · 1 line
1.8.3.1
v13 cover Initial support for multi-process Qemu
Hello,
This is the v13 of the patchset. Thank you very much for reviewing v12 and sharing your feedback.
We have addressed all the comments from the v12 review except for the following two items, which we will be addressing in subsequent patches:
- Prefix log messages with PID in the logging subsystem.
- Refactor vfio-user and multi-process code to avoid logic duplication.
To touch upon the history of this project, we posted the Proof Of Concept patches before the BoF session in 2018. Subsequently, we have posted 12 versions on the qemu-devel mailing list. You can find them by following the links below ([1] - [12]). Following people contributed to the design and implementation of this project:
Jagannathan Raman <[email protected]>
Elena Ufimtseva <[email protected]>
John G Johnson <[email protected]>
Stefan Hajnoczi <[email protected]>
Konrad Wilk <[email protected]>
Kanth Ghatraju <[email protected]>
We would like to thank the QEMU community for your feedback in the design and implementation of this project. Qemu wiki page:
https://wiki.qemu.org/Features/MultiProcessQEMU
For the full concept writeup about QEMU multi-process, please refer to docs/devel/qemu-multiprocess.rst. Also, see docs/qemu-multiprocess.txt for usage information.
We have retained Stefan's "Reviewed-by" in the commit messages. Thank you very much for your time in reviewing these patches!
[POC]: https://www.mail-archive.com/[email protected]/msg566538.html
[1]: https://www.mail-archive.com/[email protected]/msg602285.html
[2]: https://www.mail-archive.com/[email protected]/msg624877.html
[3]: https://www.mail-archive.com/[email protected]/msg642000.html
[4]: https://www.mail-archive.com/[email protected]/msg655118.html
[5]: https://www.mail-archive.com/[email protected]/msg682429.html
[6]: https://www.mail-archive.com/[email protected]/msg697484.html
[7]: https://patchew.org/QEMU/[email protected]/
[8]: https://www.mail-archive.com/[email protected]/msg727007.html
[9]: https://www.mail-archive.com/[email protected]/msg734275.html
[10]: https://www.mail-archive.com/[email protected]/msg747638.html
[11]: https://www.mail-archive.com/[email protected]/msg750972.html
[12]: https://patchew.org/QEMU/[email protected]/
Elena Ufimtseva (7):
multi-process: add configure and usage information
multi-process: add qio channel function to transmit data and fds
multi-process: define MPQemuMsg format and transmission functions
multi-process: introduce proxy object
multi-process: add proxy communication functions
multi-process: Forward PCI config space acceses to the remote process
multi-process: perform device reset in the remote process
Jagannathan Raman (11):
memory: alloc RAM from file at offset
multi-process: Add config option for multi-process QEMU
multi-process: setup PCI host bridge for remote device
multi-process: setup a machine object for remote device process
multi-process: Initialize message handler in remote device
multi-process: Associate fd of a PCIDevice with its object
multi-process: setup memory manager for remote device
multi-process: PCI BAR read/write handling for proxy & remote
endpoints
multi-process: Synchronize remote memory
multi-process: create IOHUB object to handle irq
multi-process: Retrieve PCI info from remote process
John G Johnson (1):
multi-process: add the concept description to
docs/devel/qemu-multiprocess
docs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 ++++++++++++++++++++++++++++++ docs/multi-process.rst | 64 ++ meson.build | 1 + hw/remote/trace.h | 1 + include/exec/memory.h | 2 + include/exec/ram_addr.h | 2 +- include/hw/pci-host/remote.h | 31 + include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++ include/hw/remote/machine.h | 40 ++ include/hw/remote/memory.h | 19 + include/hw/remote/mpqemu-link.h | 99 +++ include/hw/remote/proxy-memory-listener.h | 28 + include/hw/remote/proxy.h | 52 ++ include/io/channel.h | 50 ++ include/qemu/mmap-alloc.h | 4 +- include/sysemu/iothread.h | 6 + backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 +- hw/pci-host/remote.c | 75 +++ hw/remote/iohub.c | 119 ++++ hw/remote/machine.c | 80 +++ hw/remote/memory.c | 65 ++ hw/remote/message.c | 230 +++++++ hw/remote/mpqemu-link.c | 267 +++++++++ hw/remote/proxy-memory-listener.c | 227 +++++++ hw/remote/proxy.c | 371 ++++++++++++ hw/remote/remote-obj.c | 206 +++++++ io/channel.c | 63 +- iothread.c | 6 + softmmu/memory.c | 3 +- softmmu/physmem.c | 11 +- util/mmap-alloc.c | 7 +- util/oslib-posix.c | 2 +- MAINTAINERS | 24 + accel/Kconfig | 1 + hw/Kconfig | 1 + hw/meson.build | 1 + hw/pci-host/Kconfig | 3 + hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 4 + hw/remote/meson.build | 13 + hw/remote/trace-events | 4 + 44 files changed, 3186 insertions(+), 14 deletions(-) create mode 100644 docs/devel/multi-process.rst create mode 100644 docs/multi-process.rst create mode 100644 hw/remote/trace.h create mode 100644 include/hw/pci-host/remote.h create mode 100644 include/hw/remote/iohub.h create mode 100644 include/hw/remote/machine.h create mode 100644 include/hw/remote/memory.h create mode 100644 include/hw/remote/mpqemu-link.h create mode 100644 include/hw/remote/proxy-memory-listener.h create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/pci-host/remote.c create mode 100644 hw/remote/iohub.c create mode 100644 hw/remote/machine.c create mode 100644 hw/remote/memory.c create mode 100644 hw/remote/message.c create mode 100644 hw/remote/mpqemu-link.c create mode 100644 hw/remote/proxy-memory-listener.c create mode 100644 hw/remote/proxy.c create mode 100644 hw/remote/remote-obj.c create mode 100644 hw/remote/Kconfig create mode 100644 hw/remote/meson.build create mode 100644 hw/remote/trace-events
Signature · 1 line
1.8.3.1
PATCH v13 19/19 multi-process: perform device reset in the remote process
From: Elena Ufimtseva <[email protected]>
Perform device reset in the remote process when QEMU performs device reset. This is required to reset the internal state (like registers, etc...) of emulated devices
--- include/hw/remote/mpqemu-link.h | 1 + hw/remote/message.c | 22 ++++++++++++++++++++++ hw/remote/proxy.c | 19 +++++++++++++++++++ 3 files changed, 42 insertions(+)
include/hw/remote/mpqemu-link.h +1 -0
@@ -40,6 +40,7 @@ typedef enum {MPQEMU_CMD_BAR_WRITE ,MPQEMU_CMD_BAR_READ ,MPQEMU_CMD_SET_IRQFD , +MPQEMU_CMD_DEVICE_RESET ,MPQEMU_CMD_MAX , } MPQemuCmd;
hw/remote/message.c +22 -0
@@ -19,6 +19,7 @@#include "exec/memattrs.h" #include "hw/remote/memory.h" #include "hw/remote/iohub.h" +#include "sysemu/reset.h" static void process_config_write (QIOChannel *ioc,PCIDevice *dev,MPQemuMsg *msg,Error **errp);
@@ -26,6 +27,8 @@ static void process_config_read(QIOChannel *ioc, PCIDevice *dev, MPQemuMsg *msg, Error **errp);static void process_bar_write (QIOChannel *ioc,MPQemuMsg *msg,Error **errp);static void process_bar_read (QIOChannel *ioc,MPQemuMsg *msg,Error **errp); +static void process_device_reset_msg (QIOChannel *ioc,PCIDevice *dev, +Error **errp); void coroutine_fn mpqemu_remote_msg_loop_co(void *data) {
@@ -69,6 +72,9 @@ void coroutine_fn mpqemu_remote_msg_loop_co(void *data)case MPQEMU_CMD_SET_IRQFD :process_set_irqfd_msg (pci_dev, &msg);break ; +case MPQEMU_CMD_DEVICE_RESET : +process_device_reset_msg (com->ioc, pci_dev, &local_err); +break ;default :error_setg (&local_err, "Unknown command (%d) received for device %s"
@@ -206,3 +212,19 @@ fail: getpid()); } } + +static void process_device_reset_msg (QIOChannel *ioc,PCIDevice *dev, +Error **errp) +{ +DeviceClass *dc =DEVICE_GET_CLASS (dev); +DeviceState *s =DEVICE (dev); +MPQemuMsg ret = {0 }; + +if (dc->reset) { + dc->reset (s); + } + + ret.cmd =MPQEMU_CMD_RET ; + +mpqemu_msg_send (&ret, ioc, errp); +}
hw/remote/proxy.c +19 -0
@@ -25,6 +25,7 @@#include "util/event_notifier-posix.c" static void probe_pci_info (PCIDevice *dev,Error **errp); +static void proxy_device_reset (DeviceState *dev);static void proxy_intx_update (PCIDevice *pci_dev) {
@@ -194,6 +195,8 @@ static void pci_proxy_dev_class_init(ObjectClass *klass, void *data) k->config_read = pci_proxy_read_config; k->config_write = pci_proxy_write_config; + dc->reset = proxy_device_reset; +device_class_set_props (dc, proxy_properties); }
@@ -350,3 +353,19 @@ static void probe_pci_info(PCIDevice *dev, Error **errp) } } } + +static void proxy_device_reset (DeviceState *dev) +{ +PCIProxyDev *pdev =PCI_PROXY_DEV (dev); +MPQemuMsg msg = {0 }; +Error *local_err =NULL ; + + msg.cmd =MPQEMU_CMD_DEVICE_RESET ; + msg.size =0 ; + +mpqemu_msg_send_and_await_reply (&msg, pdev, &local_err); +if (local_err) { +error_report_err (local_err); + } + +}
Signature · 1 line
1.8.3.1
Hi,
In this series, we have incorporated the changes Marc-Andre proposed to improve the code in v12. Following is a changelog that outlines the differences between v12 and v13.
CHANGELOG:
--------------------
[PATCH v13 02/19] multi-process: add configure and usage information
- Dropped “tests/multiprocess/multiprocess-lsi53c895a.py” based on feedback. We are working on acceptance tests instead.
[PATCH v13 03/19] memory: alloc RAM from file at offset
- Changed the name of argument “start” to “map_offset”
[PATCH v13 05/19] multi-process: setup PCI host bridge for remote device
- Renamed MULTIPROCESS_HOST as REMOTE_PCIHOST
- Also renamed TYPE_REMOTE_HOST_DEVICE and REMOTE_HOST_DEVICE as TYPE_REMOTE_PCIHOST and REMOTE_PCIHOST respectively
[PATCH v13 06/19] multi-process: setup a machine object for remote device
process
- Renamed “remote-device” as “remote-pcihost”
- Set a description for the machine class
[PATCH v13 07/19] multi-process: add qio channel function to transmit data
and fds
- Refactored qio_channel_writev_all() and qio_channel_writev_full_all() to avoid logic duplication
[PATCH v13 08/19] multi-process: define MPQemuMsg format and transmission
functions
- Introduced qemu_in_iothread() helper function
- Using ERRP_GUARD to handle input errp parameter
- Addressed redundant typecasts to “void”
- mpqemu_msg_send() and mpqemu_msg_send() returns true/false to indicate success/failure
- Added a comment to explain that the messaging protocol used would eventually be replaced by vfio-user
- Prefixed the command enums with “MPQEMU_CMD_”.
- Skip calling mpqemu_read() if msg->size is zero.
[PATCH v13 09/19] multi-process: Initialize message handler in remote device
- Check for NULL IOChannel converted to an assert, and moved outside for loop
- Moved error handling outside for loop
- Shutdown with SHUTDOWN_CAUSE_HOST_ERROR with error in case of failure
- Moved exit condition into the for loop statement
[PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object
- Removed a separate header file and merged it into the “.c” file
- Replaced Object properties with Object Class properties
- Accept the file descriptor as an integer property, and check if it is a valid socket using fd_is_socket()
- Replaced runtime/dynamic property with a DeviceListener
- Added a comment to explain the temporary limitation for one device per remote process
[PATCH v13 11/19] multi-process: setup memory manager for remote device
- Dropped the calls to memory_region_transaction_begin() and memory_region_transaction_commit()
- Using ERRP_GUARD to handle input errp parameter
- Using g_autofree to automatically allocate the “name” variable
- Moved RET_MSG command definition to the patch that starts to use it
[PATCH v13 12/19] multi-process: introduce proxy object
- device class property is defined just above the class_init function
- Using ERRP_GUARD to handle input errp parameter
- Check if IOChannel is NULL before closing in the exit function, to avoid crash
- Early return if fd is not set
- dev->migration_blocker is free’d by the exit function alone
[PATCH v13 13/19] multi-process: add proxy communication functions
- Asset that mpqemu_msg_send_and_await_reply() thread is not in co-routine context
- Using ERRP_GUARD to handle input errp parameter
[PATCH v13 14/19] multi-process: Forward PCI config space acceses to the
remote process
- process_config_write() & process_config_read() take an Error** argument to return error
- Use FMT_pid to format messages with PID
- Use error_prepend to avoid leak of Error*
- Check if return value from remote indicates failure in config_op_send()
- Dropped typecasting return values of config_op_send to void as it’s not necessary
[PATCH v13 16/19] multi-process: Synchronize remote memory
- Renamed RemoteMemSync as ProxyMemoryListener
- Prefixed all related functions with “proxy_memory_listener_” for consistency
- Simplify logic in proxy_mrs_can_merge() to return efficiently and avoid unnecessary definition of local variable.
[PATCH v13 17/19] multi-process: create IOHUB object to handle irq
- Replace calls to qemu_mutex_lock() and qemu_mutex_unlock() with QEMU_LOCK_GUARD
Thank you very much!
Quoted 147 lines · Jagannathan Raman
Hello,
This is the v13 of the patchset. Thank you very much for reviewing v12 and sharing your feedback.
We have addressed all the comments from the v12 review except for the following two items, which we will be addressing in subsequent patches:
- Prefix log messages with PID in the logging subsystem.
- Refactor vfio-user and multi-process code to avoid logic duplication.To touch upon the history of this project, we posted the Proof Of Concept patches before the BoF session in 2018. Subsequently, we have posted 12 versions on the qemu-devel mailing list. You can find them by following the links below ([1] - [12]). Following people contributed to the design and implementation of this project:
Jagannathan Raman <[email protected]>
Elena Ufimtseva <[email protected]>
John G Johnson <[email protected]>
Stefan Hajnoczi <[email protected]>
Konrad Wilk <[email protected]>
Kanth Ghatraju <[email protected]>We would like to thank the QEMU community for your feedback in the design and implementation of this project. Qemu wiki page:
https://wiki.qemu.org/Features/MultiProcessQEMUFor the full concept writeup about QEMU multi-process, please refer to
docs/devel/qemu-multiprocess.rst. Also, seedocs/qemu-multiprocess.txtfor usage information.We have retained Stefan's "Reviewed-by" in the commit messages. Thank you very much for your time in reviewing these patches!
[POC]: https://www.mail-archive.com/[email protected]/msg566538.html
[1]: https://www.mail-archive.com/[email protected]/msg602285.html
[2]: https://www.mail-archive.com/[email protected]/msg624877.html
[3]: https://www.mail-archive.com/[email protected]/msg642000.html
[4]: https://www.mail-archive.com/[email protected]/msg655118.html
[5]: https://www.mail-archive.com/[email protected]/msg682429.html
[6]: https://www.mail-archive.com/[email protected]/msg697484.html
[7]: https://patchew.org/QEMU/[email protected]/
[8]: https://www.mail-archive.com/[email protected]/msg727007.html
[9]: https://www.mail-archive.com/[email protected]/msg734275.html
[10]: https://www.mail-archive.com/[email protected]/msg747638.html
[11]: https://www.mail-archive.com/[email protected]/msg750972.html
[12]: https://patchew.org/QEMU/[email protected]/Elena Ufimtseva (7):
multi-process: add configure and usage information
multi-process: add qio channel function to transmit data and fds
multi-process: define MPQemuMsg format and transmission functions
multi-process: introduce proxy object
multi-process: add proxy communication functions
multi-process: Forward PCI config space acceses to the remote process
multi-process: perform device reset in the remote processJagannathan Raman (11):
memory: alloc RAM from file at offset
multi-process: Add config option for multi-process QEMU
multi-process: setup PCI host bridge for remote device
multi-process: setup a machine object for remote device process
multi-process: Initialize message handler in remote device
multi-process: Associate fd of a PCIDevice with its object
multi-process: setup memory manager for remote device
multi-process: PCI BAR read/write handling for proxy & remote
endpoints
multi-process: Synchronize remote memory
multi-process: create IOHUB object to handle irq
multi-process: Retrieve PCI info from remote processJohn G Johnson (1):
multi-process: add the concept description to
docs/devel/qemu-multiprocessdocs/devel/index.rst | 1 + docs/devel/multi-process.rst | 966 ++++++++++++++++++++++++++++++ docs/multi-process.rst | 64 ++ meson.build | 1 + hw/remote/trace.h | 1 + include/exec/memory.h | 2 + include/exec/ram_addr.h | 2 +- include/hw/pci-host/remote.h | 31 + include/hw/pci/pci_ids.h | 3 + include/hw/remote/iohub.h | 42 ++ include/hw/remote/machine.h | 40 ++ include/hw/remote/memory.h | 19 + include/hw/remote/mpqemu-link.h | 99 +++ include/hw/remote/proxy-memory-listener.h | 28 + include/hw/remote/proxy.h | 52 ++ include/io/channel.h | 50 ++ include/qemu/mmap-alloc.h | 4 +- include/sysemu/iothread.h | 6 + backends/hostmem-memfd.c | 2 +- hw/misc/ivshmem.c | 3 +- hw/pci-host/remote.c | 75 +++ hw/remote/iohub.c | 119 ++++ hw/remote/machine.c | 80 +++ hw/remote/memory.c | 65 ++ hw/remote/message.c | 230 +++++++ hw/remote/mpqemu-link.c | 267 +++++++++ hw/remote/proxy-memory-listener.c | 227 +++++++ hw/remote/proxy.c | 371 ++++++++++++ hw/remote/remote-obj.c | 206 +++++++ io/channel.c | 63 +- iothread.c | 6 + softmmu/memory.c | 3 +- softmmu/physmem.c | 11 +- util/mmap-alloc.c | 7 +- util/oslib-posix.c | 2 +- MAINTAINERS | 24 + accel/Kconfig | 1 + hw/Kconfig | 1 + hw/meson.build | 1 + hw/pci-host/Kconfig | 3 + hw/pci-host/meson.build | 1 + hw/remote/Kconfig | 4 + hw/remote/meson.build | 13 + hw/remote/trace-events | 4 + 44 files changed, 3186 insertions(+), 14 deletions(-)create mode 100644
docs/devel/multi-process.rstcreate mode 100644docs/multi-process.rst
create mode 100644hw/remote/trace.h
create mode 100644include/hw/pci-host/remote.hcreate mode 100644include/hw/remote/iohub.hcreate mode 100644include/hw/remote/machine.hcreate mode 100644include/hw/remote/memory.hcreate mode 100644include/hw/remote/mpqemu-link.hcreate mode 100644include/hw/remote/proxy-memory-listener.hcreate mode 100644include/hw/remote/proxy.hcreate mode 100644hw/pci-host/remote.c
create mode 100644hw/remote/iohub.c
create mode 100644hw/remote/machine.c
create mode 100644hw/remote/memory.c
create mode 100644hw/remote/message.c
create mode 100644hw/remote/mpqemu-link.c
create mode 100644hw/remote/proxy-memory-listener.ccreate mode 100644hw/remote/proxy.c
create mode 100644hw/remote/remote-obj.c
create mode 100644 hw/remote/Kconfig
create mode 100644 hw/remote/meson.build
create mode 100644 hw/remote/trace-eventsSignature · 1 line
1.8.3.1
Quoted 4 lines · Jagannathan Raman
Hello,
This is the v13 of the patchset. Thank you very much for reviewing v12 and sharing your feedback.
As mentioned on IRC, please add a ./configure option so package maintainers can easily disable multi-process QEMU without patching the source code.
Thanks,
Stefan
Re: [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object
In reply to Jagannathan RamanQuoted 25 lines · Jagannathan Raman
hw/remote/remote-obj.c new file +25
-0+static void remote_object_release (DeviceListener *listener,DeviceState *dev) +{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err);
hw/remote/remote-obj.c line 118
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_release; +device_listener_register (&o->listener);
The remote_object_release() name is outdated. How about remote_object_unrealize_listener()?
Quoted 7 lines · Jagannathan Raman
This is the v13 of the patchset. Thank you very much for reviewing v12 and sharing your feedback.
We have addressed all the comments from the v12 review except for the following two items, which we will be addressing in subsequent patches:
- Prefix log messages with PID in the logging subsystem.
- Refactor vfio-user and multi-process code to avoid logic duplication.
Aside from the missing ./configure option needed to easily compile out this feature, I'm happy with this series. I think that should be solved before merging this patch series.
The series passes Gitlab CI.
I have not run multi-process QEMU since there is no test case or launcher. Please send a test case as a separate patch series.
Stefan
Re: [PATCH v13 07/19] multi-process: add qio channel function to transmit data and fds
In reply to Jagannathan RamanHi
Quoted 119 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds QIO channel functions that transmit and receive iovs along with fds.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/io/channel.h | 50 +++++++++++++++++++++++++++++++++++++++++ io/channel.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-)include/io/channel.h +50
-0@@ -774,4 +774,54 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_write, void *opaque); +/** +* qio_channel_readv_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to read data to +* @niov: the length of the @iov array +* @fds: an array of file handles to read +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_readvv_full but will attempt +* to read all data specified (file handles and memory regions). +* The function will wait for all requested data +* to be read, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were read, or -1 on error +*/ + +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp); + +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp); +#endif /* QIO_CHANNEL_H */ io/channel.c +37
-0@@ -152,15 +152,72 @@ int qio_channel_readv_all(QIOChannel *ioc,return ret; } +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp) +{ +int ret =-1 ; + struct iovec *local_iov = g_new(struct iovec, niov); +struct iovec *local_iov_head = local_iov; +unsigned int nlocal_iov = niov; +int **local_fds = fds; +size_t *local_nfds = nfds; + + nlocal_iov =iov_copy (local_iov, nlocal_iov, + iov, niov, +0 ,iov_size (iov, niov)); + +while (nlocal_iov >0 ) { +ssize_t len; + len =qio_channel_readv_full (ioc, local_iov, nlocal_iov, local_fds, + local_nfds, errp); +if (len ==QIO_CHANNEL_ERR_BLOCK ) { +if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_OUT ); + }else { +qio_channel_wait (ioc,G_IO_OUT ); + } +continue ; + } +if (len <=0 ) { + ret = len; + goto cleanup; + } + +iov_discard_front (&local_iov, &nlocal_iov, len); + + local_fds =NULL ;
io/channel.c int qio_channel_readv_all(QIOChannel *ioc, · line 202
Quoted 10 lines
io/channel.c +9
-0+ local_nfds =0 ; + } + + ret =1 ; + cleanup: +g_free (local_iov_head); +return ret; +} +
I suggest to adapt the code so qio_channel_readv_all_eof() calls qio_channel_readv_full_all().
You may want to split this patch in 2 parts: the read and write parts.
Quoted 48 lines
io/channel.c +11
-0int qio_channel_writev_all (QIOChannel *ioc,const struct iovec *iov,size_t niov,Error **errp) { +return qio_channel_writev_full_all (ioc, iov, niov,NULL ,0 , errp); +} + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp) +{int ret =-1 ; struct iovec *local_iov = g_new(struct iovec, niov);struct iovec *local_iov_head = local_iov;unsigned int nlocal_iov = niov; +int *local_fds = fds; +size_t local_nfds = nfds;
nlocal_iov= iov_copy(local_iov,nlocal_iov,
iov, niov,io/channel.c +2
-1@@ -168,7 +225,8 @@ int qio_channel_writev_all(QIOChannel *ioc,while (nlocal_iov >0 ) {ssize_t len; - len =qio_channel_writev (ioc, local_iov, nlocal_iov, errp); + len =qio_channel_writev_full (ioc, local_iov, nlocal_iov, local_fds, + local_nfds, errp);if (len ==QIO_CHANNEL_ERR_BLOCK ) {if (qemu_in_coroutine ()) {qio_channel_yield(ioc, G_IO_OUT);io/channel.c +3
-1@@ -182,6 +240,9 @@ int qio_channel_writev_all(QIOChannel *ioc, }iov_discard_front (&local_iov, &nlocal_iov, len); + + local_fds =NULL ; + local_nfds =0 ; } ret =0 ; --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v13 07/19] multi-process: add qio channel function to transmit data and fds
Quoted 138 lines · Marc-André Lureau
Hi
Quoted 119 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds QIO channel functions that transmit and receive iovs along with fds.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/io/channel.h | 50 +++++++++++++++++++++++++++++++++++++++++ io/channel.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-)include/io/channel.h +50
-0@@ -774,4 +774,54 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_write, void *opaque); +/** +* qio_channel_readv_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to read data to +* @niov: the length of the @iov array +* @fds: an array of file handles to read +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_readvv_full but will attempt +* to read all data specified (file handles and memory regions). +* The function will wait for all requested data +* to be read, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were read, or -1 on error +*/ + +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp); + +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp); +#endif /* QIO_CHANNEL_H */ io/channel.c +37
-0@@ -152,15 +152,72 @@ int qio_channel_readv_all(QIOChannel *ioc,return ret; } +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp) +{ +int ret =-1 ; + struct iovec *local_iov = g_new(struct iovec, niov); +struct iovec *local_iov_head = local_iov; +unsigned int nlocal_iov = niov; +int **local_fds = fds; +size_t *local_nfds = nfds; + + nlocal_iov =iov_copy (local_iov, nlocal_iov, + iov, niov, +0 ,iov_size (iov, niov)); + +while (nlocal_iov >0 ) { +ssize_t len; + len =qio_channel_readv_full (ioc, local_iov, nlocal_iov, local_fds, + local_nfds, errp); +if (len ==QIO_CHANNEL_ERR_BLOCK ) { +if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_OUT ); + }else { +qio_channel_wait (ioc,G_IO_OUT ); + } +continue ; + } +if (len <=0 ) { + ret = len; + goto cleanup; + } + +iov_discard_front (&local_iov, &nlocal_iov, len); + + local_fds =NULL ;io/channel.c int qio_channel_readv_all(QIOChannel *ioc, · line 202
Quoted 10 lines
io/channel.c +9
-0+ local_nfds =0 ; + } + + ret =1 ; + cleanup: +g_free (local_iov_head); +return ret; +} +I suggest to adapt the code so
qio_channel_readv_all_eof()callsqio_channel_readv_full_all().
Quoted 3 lines
You may want to split this patch in 2 parts: the read and write parts.
Thank you Marc-Andre, will do.
Elena
Quoted 51 lines
Quoted 48 lines
io/channel.c +11
-0int qio_channel_writev_all (QIOChannel *ioc,const struct iovec *iov,size_t niov,Error **errp) { +return qio_channel_writev_full_all (ioc, iov, niov,NULL ,0 , errp); +} + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp) +{int ret =-1 ; struct iovec *local_iov = g_new(struct iovec, niov);struct iovec *local_iov_head = local_iov;unsigned int nlocal_iov = niov; +int *local_fds = fds; +size_t local_nfds = nfds;
nlocal_iov= iov_copy(local_iov,nlocal_iov,
iov, niov,io/channel.c +2
-1@@ -168,7 +225,8 @@ int qio_channel_writev_all(QIOChannel *ioc,while (nlocal_iov >0 ) {ssize_t len; - len =qio_channel_writev (ioc, local_iov, nlocal_iov, errp); + len =qio_channel_writev_full (ioc, local_iov, nlocal_iov, local_fds, + local_nfds, errp);if (len ==QIO_CHANNEL_ERR_BLOCK ) {if (qemu_in_coroutine ()) {qio_channel_yield(ioc, G_IO_OUT);io/channel.c +3
-1@@ -182,6 +240,9 @@ int qio_channel_writev_all(QIOChannel *ioc, }iov_discard_front (&local_iov, &nlocal_iov, len); + + local_fds =NULL ; + local_nfds =0 ; } ret =0 ; --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v13 07/19] multi-process: add qio channel function to transmit data and fds
In reply to Marc-André LureauQuoted 138 lines · Marc-André Lureau
Hi
Quoted 119 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Adds QIO channel functions that transmit and receive iovs along with fds.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/io/channel.h | 50 +++++++++++++++++++++++++++++++++++++++++ io/channel.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-)include/io/channel.h +50
-0@@ -774,4 +774,54 @@ void qio_channel_set_aio_fd_handler(QIOChannel *ioc, IOHandler *io_write, void *opaque); +/** +* qio_channel_readv_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to read data to +* @niov: the length of the @iov array +* @fds: an array of file handles to read +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_readvv_full but will attempt +* to read all data specified (file handles and memory regions). +* The function will wait for all requested data +* to be read, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were read, or -1 on error +*/ + +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp); + +/** +* qio_channel_writev_full_all: +* @ioc: the channel object +* @iov: the array of memory regions to write data from +* @niov: the length of the @iov array +* @fds: an array of file handles to send +* @nfds: number of file handles in @fds +* @errp: pointer to a NULL-initialized error object +* +* +* Behaves like qio_channel_writev_full but will attempt +* to send all data passed (file handles and memory regions). +* The function will wait for all requested data +* to be written, yielding from the current coroutine +* if required. +* +* Returns: 0 if all bytes were written, or -1 on error +*/ + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp); +#endif /* QIO_CHANNEL_H */ io/channel.c +37
-0@@ -152,15 +152,72 @@ int qio_channel_readv_all(QIOChannel *ioc,return ret; } +int qio_channel_readv_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int **fds,size_t *nfds, +Error **errp) +{ +int ret =-1 ; + struct iovec *local_iov = g_new(struct iovec, niov); +struct iovec *local_iov_head = local_iov; +unsigned int nlocal_iov = niov; +int **local_fds = fds; +size_t *local_nfds = nfds; + + nlocal_iov =iov_copy (local_iov, nlocal_iov, + iov, niov, +0 ,iov_size (iov, niov)); + +while (nlocal_iov >0 ) { +ssize_t len; + len =qio_channel_readv_full (ioc, local_iov, nlocal_iov, local_fds, + local_nfds, errp); +if (len ==QIO_CHANNEL_ERR_BLOCK ) { +if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_OUT ); + }else { +qio_channel_wait (ioc,G_IO_OUT ); + } +continue ; + } +if (len <=0 ) { + ret = len; + goto cleanup; + } + +iov_discard_front (&local_iov, &nlocal_iov, len); + + local_fds =NULL ;io/channel.c int qio_channel_readv_all(QIOChannel *ioc, · line 202
Quoted 10 lines
io/channel.c +9
-0+ local_nfds =0 ; + } + + ret =1 ; + cleanup: +g_free (local_iov_head); +return ret; +} +I suggest to adapt the code so
qio_channel_readv_all_eof()callsqio_channel_readv_full_all().
Quoted 2 lines
You may want to split this patch in 2 parts: the read and write parts.
And should we drop the Reviewed-by: Stefan Hajnoczi then?
Thanks!
Elena
Quoted 52 lines
Quoted 49 lines
io/channel.c +11
-0int qio_channel_writev_all (QIOChannel *ioc,const struct iovec *iov,size_t niov,Error **errp) { +return qio_channel_writev_full_all (ioc, iov, niov,NULL ,0 , errp); +} + +int qio_channel_writev_full_all (QIOChannel *ioc, +const struct iovec *iov, +size_t niov, +int *fds,size_t nfds, +Error **errp) +{int ret =-1 ; struct iovec *local_iov = g_new(struct iovec, niov);struct iovec *local_iov_head = local_iov;unsigned int nlocal_iov = niov; +int *local_fds = fds; +size_t local_nfds = nfds;
nlocal_iov= iov_copy(local_iov,nlocal_iov,
iov, niov,io/channel.c +2
-1@@ -168,7 +225,8 @@ int qio_channel_writev_all(QIOChannel *ioc,while (nlocal_iov >0 ) {ssize_t len; - len =qio_channel_writev (ioc, local_iov, nlocal_iov, errp); + len =qio_channel_writev_full (ioc, local_iov, nlocal_iov, local_fds, + local_nfds, errp);if (len ==QIO_CHANNEL_ERR_BLOCK ) {if (qemu_in_coroutine ()) {qio_channel_yield(ioc, G_IO_OUT);io/channel.c +3
-1@@ -182,6 +240,9 @@ int qio_channel_writev_all(QIOChannel *ioc, }iov_discard_front (&local_iov, &nlocal_iov, len); + + local_fds =NULL ; + local_nfds =0 ; } ret =0 ; --1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v13 07/19] multi-process: add qio channel function to transmit data and fds
Quoted 8 lines · Elena Ufimtseva
Quoted 5 lines · Marc-André Lureau
I suggest to adapt the code so
qio_channel_readv_all_eof()callsqio_channel_readv_full_all().You may want to split this patch in 2 parts: the read and write parts.
And should we drop the Reviewed-by: Stefan Hajnoczi then?
Yes, please drop my R-b and I'll review again.
Stefan
Re: [PATCH v13 12/19] multi-process: introduce proxy object
In reply to Jagannathan RamanQuoted 95 lines · Jagannathan Raman
From: Elena Ufimtseva <[email protected]>
Defines a PCI Device proxy object as a child of
TYPE_PCI_DEVICE.Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Signed-off-by John G Johnson <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- include/hw/remote/proxy.h | 36 +++++++++++++++++++ hw/remote/proxy.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 2 ++ hw/remote/meson.build | 1 + 4 files changed, 130 insertions(+) create mode 100644 include/hw/remote/proxy.h create mode 100644 hw/remote/proxy.cinclude/hw/remote/proxy.h new file +4
-0@@ -0,0 +1,36 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */include/hw/remote/proxy.h new file +29
-0+ +#ifndef PROXY_H +#define PROXY_H + +#include "hw/pci/pci.h" +#include "io/channel.h" + +#define TYPE_PCI_PROXY_DEV "x-pci-proxy-dev" + +#define PCI_PROXY_DEV (obj) \ + OBJECT_CHECK(PCIProxyDev, (obj), TYPE_PCI_PROXY_DEV) +typedef struct PCIProxyDev PCIProxyDev ; + +struct PCIProxyDev { +PCIDevice parent_dev; +char *fd; + +/* +* Mutex used to protect the QIOChannel fd from +* the concurrent access by the VCPUs since proxy +* blocks while awaiting for the replies from the +* process remote. +*/ +QemuMutex io_mutex; +QIOChannel *ioc; +Error *migration_blocker; +}; + +#endif /* PROXY_H */ hw/remote/proxy.c new file +4
-0@@ -0,0 +1,91 @@ +/* +* Copyright © 2018, 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/proxy.c new file +19
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "hw/remote/proxy.h" +#include "hw/pci/pci.h" +#include "qapi/error.h" +#include "io/channel-util.h" +#include "hw/qdev-properties.h" +#include "monitor/monitor.h" +#include "migration/blocker.h" + +static void pci_proxy_dev_realize (PCIDevice *device,Error **errp) +{ +ERRP_GUARD (); +PCIProxyDev *dev =PCI_PROXY_DEV (device); +int fd; + +if (!dev->fd) {
hw/remote/proxy.c line 36
Quoted 10 lines
hw/remote/proxy.c new file +10
-0+error_setg (errp,"fd parameter not specified for %s" , +DEVICE (device)->id); +return ; + } + + fd =monitor_fd_param (monitor_cur (), dev->fd, errp); +if (fd ==-1 ) { +error_prepend (errp,"proxy: unable to parse fd: " ); +return ; + }
Let's add a fd_is_socket() check.
+ dev->ioc = qio_channel_new_fd(fd, errp);
Quoted 85 lines
hw/remote/proxy.c new file +54
-0+ +error_setg (&dev->migration_blocker,"%s does not support migration" , +TYPE_PCI_PROXY_DEV ); +migrate_add_blocker (dev->migration_blocker, errp); + +qemu_mutex_init (&dev->io_mutex); +qio_channel_set_blocking (dev->ioc, true,NULL ); +} + +static void pci_proxy_dev_exit (PCIDevice *pdev) +{ +PCIProxyDev *dev =PCI_PROXY_DEV (pdev); + +if (dev->ioc) { +qio_channel_close (dev->ioc,NULL ); + } + +migrate_del_blocker (dev->migration_blocker); + +error_free (dev->migration_blocker); +} + +static Property proxy_properties[] = { +DEFINE_PROP_STRING ("fd" , PCIProxyDev, fd), +DEFINE_PROP_END_OF_LIST (), +}; + +static void pci_proxy_dev_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +PCIDeviceClass *k =PCI_DEVICE_CLASS (klass); + + k->realize = pci_proxy_dev_realize; + k->exit = pci_proxy_dev_exit; +device_class_set_props (dc, proxy_properties); +} + +static const TypeInfo pci_proxy_dev_type_info = { + .name =TYPE_PCI_PROXY_DEV , + .parent =TYPE_PCI_DEVICE , + .instance_size =sizeof (PCIProxyDev), + .class_init = pci_proxy_dev_class_init, + .interfaces = (InterfaceInfo []) { + {INTERFACE_CONVENTIONAL_PCI_DEVICE }, + { }, + }, +}; + +static void pci_proxy_dev_register_types (void ) +{ +type_register_static (&pci_proxy_dev_type_info); +} + +type_init (pci_proxy_dev_register_types)MAINTAINERS +2
-0@@ -3163,6 +3163,8 @@ F: hw/remote/message.c F: hw/remote/remote-obj.c F: include/hw/remote/memory.h F: hw/remote/memory.c +F: hw/remote/proxy.c +F: include/hw/remote/proxy.h Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -4,6 +4,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c'))
specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c'))--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object
In reply to Jagannathan RamanQuoted 202 lines · Jagannathan Raman
Associate the file descriptor for a PCIDevice in remote process with DeviceState object.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- hw/remote/remote-obj.c | 206 +++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 3 files changed, 208 insertions(+) create mode 100644 hw/remote/remote-obj.chw/remote/remote-obj.c new file +4
-0@@ -0,0 +1,206 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
+ *
+ * See the COPYING file in the top-level directory.
+ *
+ */hw/remote/remote-obj.c new file +48
-0+ +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/error-report.h" +#include "qemu/notify.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" +#include "qemu/sockets.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT (obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS (obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS (klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}RemoteObjectClass ; + +typedef struct { +/* private */ +Object parent; + +Notifier machine_done; + +int32_t fd; +char *devid; + +QIOChannel *ioc; + +DeviceState *dev; +DeviceListener listener; +}RemoteObject ; + +static void remote_object_set_fd(Object *obj, Visitor *v, const char*name,
hw/remote/remote-obj.c new file +19
-0+ void *opaque, Error **errp) +{ + RemoteObject *o = REMOTE_OBJECT(obj); +int32_t value; + +if (!visit_type_int32 (v, name, &value, errp)) { +return ; + } + +if (!fd_is_socket (value)) { + o->fd =-1 ; +error_setg (errp,"File descriptor %d is not a socket" , value); +return ; + } + + o->fd = value; +} + +static void remote_object_set_devid(Object *obj, const char *str, Error**errp)
hw/remote/remote-obj.c new file +9
-0+{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); + + o->devid =g_strdup (str); +} + +static void remote_object_release(DeviceListener *listener, DeviceState*dev)
hw/remote/remote-obj.c new file +53
-0+{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_release; +device_listener_register (&o->listener); + +/* co-routine should free this. */ + comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + }; + + co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev); +qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { + error_report("Reached maximum number of devices: %u",
k->max_devs);hw/remote/remote-obj.c new file +37
-0+return ; + } + + o->ioc =NULL ; + o->fd =-1 ; + o->devid =NULL ; + + k->nr_devs++; + + o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +device_listener_unregister (&o->listener); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc)); + + k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + +/* +* Limit number of supported devices to 1. This is done to avoid devices
hw/remote/remote-obj.c line 182
Quoted 8 lines
hw/remote/remote-obj.c new file +8
-0+* from one VM accessing the RAM of another VM. This is done until we +* start using separate address spaces for individual devices. +*/ + k->max_devs =1 ; + k->nr_devs =0 ; + +object_class_property_add (klass,"fd" ,"int32_t" ,NULL , + remote_object_set_fd,NULL ,NULL );
Let's be consistent with pci-proxy-dev, and make it a string property that is parsed with monitor_fd_param()
+ object_class_property_add_str(klass, "devid", NULL,
Quoted 51 lines
hw/remote/remote-obj.c new file +23
-0+ remote_object_set_devid); +} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);MAINTAINERS +1
-0@@ -3160,6 +3160,7 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: hw/remote/remote-obj.c Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
--
1.8.3.1
Signature · 1 line
Marc-André Lureau
Re: [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object
Quoted 210 lines · Marc-André Lureau
On Mon, Dec 14, 2020 at 9:15 AM Jagannathan Raman <[email protected]> wrote:
Associate the file descriptor for a PCIDevice in remote process with DeviceState object.Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- hw/remote/remote-obj.c | 206 +++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 3 files changed, 208 insertions(+) create mode 100644 hw/remote/remote-obj.chw/remote/remote-obj.c new file +182
-0@@ -0,0 +1,206 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later. +* +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/error-report.h" +#include "qemu/notify.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" +#include "qemu/sockets.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT (obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS (obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS (klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}RemoteObjectClass ; + +typedef struct { +/* private */ +Object parent; + +Notifier machine_done; + +int32_t fd; +char *devid; + +QIOChannel *ioc; + +DeviceState *dev; +DeviceListener listener; +}RemoteObject ; + +static void remote_object_set_fd (Object *obj,Visitor *v,const char *name, +void *opaque,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); +int32_t value; + +if (!visit_type_int32 (v, name, &value, errp)) { +return ; + } + +if (!fd_is_socket (value)) { + o->fd =-1 ; +error_setg (errp,"File descriptor %d is not a socket" , value); +return ; + } + + o->fd = value; +} + +static void remote_object_set_devid (Object *obj,const char *str,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); + + o->devid =g_strdup (str); +} + +static void remote_object_release (DeviceListener *listener,DeviceState *dev) +{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_release; +device_listener_register (&o->listener); + +/* co-routine should free this. */ + comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + }; + + co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev); +qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { +error_report ("Reached maximum number of devices: %u" , k->max_devs); +return ; + } + + o->ioc =NULL ; + o->fd =-1 ; + o->devid =NULL ; + + k->nr_devs++; + + o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +device_listener_unregister (&o->listener); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc)); + + k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + +/* +* Limit number of supported devices to 1. This is done to avoid devices +* from one VM accessing the RAM of another VM. This is done until we +* start using separate address spaces for individual devices. +*/ + k->max_devs =1 ; + k->nr_devs =0 ; + +object_class_property_add (klass,"fd" ,"int32_t" ,NULL , + remote_object_set_fd,NULL ,NULL );
Quoted 1 line
Let's be consistent with pci-proxy-dev, and make it a string property that is parsed with
monitor_fd_param()
Thank you for the feedback, Marc-Andre!
Would it be alright if we added this to util/qemu-sockets.c since this logic is used more than once, or is it an overkill? I saw that socket_get_fd() in qemu-sockets.c is doing something similar, but using monitor_get_fd() instead.
--
Jag
Quoted 55 lines
hw/remote/remote-obj.c new file +24
-0+object_class_property_add_str (klass,"devid" ,NULL , + remote_object_set_devid); +} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);MAINTAINERS +1
-0@@ -3160,6 +3160,7 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: hw/remote/remote-obj.c Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)Signature · 6 lines
1.8.3.1 -- Marc-André Lureau
Re: [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object
Hi
Quoted 224 lines · Jag Raman
Quoted 3 lines
On Dec 16, 2020, at 8:43 AM, Marc-André Lureau <
Quoted 4 lines · [email protected]> wrote:
On Mon, Dec 14, 2020 at 9:15 AM Jagannathan Raman <[email protected]>
Quoted 9 lines · wrote:
Associate the file descriptor for a PCIDevice in remote process with DeviceState object.
Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- hw/remote/remote-obj.c | 206+++++++++++++++++++++++++++++++++++++++++++++++++
Quoted 11 lines
MAINTAINERS | 1 + hw/remote/meson.build | 1 + 3 files changed, 208 insertions(+) create mode 100644 hw/remote/remote-obj.chw/remote/remote-obj.c new file +0
-0hw/remote/remote-obj.c line 4
Quoted 4 lines
hw/remote/remote-obj.c new file +4
-0@@ -0,0 +1,206 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later.
Quoted 42 lines
hw/remote/remote-obj.c new file +42
-0+* +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/error-report.h" +#include "qemu/notify.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" +#include "qemu/sockets.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT (obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS (obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS (klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}RemoteObjectClass ; + +typedef struct { + /* private */ + Object parent; + + Notifier machine_done; +hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+int32_t fd; +char *devid; + +QIOChannel *ioc; + +DeviceState *dev; +DeviceListener listener; +} RemoteObject; + +static void remote_object_set_fd(Object *obj, Visitor *v, const char*name,
Quoted 9 lines
hw/remote/remote-obj.c new file +9
-0+ void *opaque, Error **errp) +{ + RemoteObject *o = REMOTE_OBJECT(obj); +int32_t value; + +if (!visit_type_int32 (v, name, &value, errp)) { +return ; + } +hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+if (!fd_is_socket (value)) { + o->fd =-1 ; +error_setg (errp,"File descriptor %d is not a socket" , value); +return ; + } + + o->fd = value; +} + +static void remote_object_set_devid(Object *obj, const char *str, Error**errp)
hw/remote/remote-obj.c
Quoted 9 lines
hw/remote/remote-obj.c new file +9
-0+{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); + + o->devid =g_strdup (str); +} + +static void remote_object_release(DeviceListener *listener, DeviceState*dev)
hw/remote/remote-obj.c
Quoted 11 lines
hw/remote/remote-obj.c new file +11
-0+{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ + RemoteObject *o = container_of(notifier, RemoteObject,
machine_done);Quoted 32 lines
hw/remote/remote-obj.c new file +32
-0+DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_release; +device_listener_register (&o->listener); + +/* co-routine should free this. */ + comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + }; + + co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev);hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { + error_report("Reached maximum number of devices: %u",
k->max_devs);Quoted 27 lines
hw/remote/remote-obj.c new file +27
-0+return ; + } + + o->ioc =NULL ; + o->fd =-1 ; + o->devid =NULL ; + + k->nr_devs++; + + o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +device_listener_unregister (&o->listener); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc)); +hw/remote/remote-obj.c
Quoted 10 lines
hw/remote/remote-obj.c new file +10
-0+ k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + +/* +* Limit number of supported devices to 1. This is done to avoid devices
hw/remote/remote-obj.c line 175
Quoted 1 line
hw/remote/remote-obj.c new file +1
-0+* from one VM accessing the RAM of another VM. This is done until we
Quoted 9 lines
hw/remote/remote-obj.c new file +7
-0+* start using separate address spaces for individual devices. +*/ + k->max_devs =1 ; + k->nr_devs =0 ; + +object_class_property_add (klass,"fd" ,"int32_t" ,NULL , + remote_object_set_fd,NULL ,NULL );Let's be consistent with pci-proxy-dev, and make it a string property
that is parsed with
monitor_fd_param()
Quoted 9 lines
Thank you for the feedback, Marc-Andre!
Would it be alright if we added this to
util/qemu-sockets.csince this logic is used more than once, or is it
an overkill? I saw thatsocket_get_fd()inqemu-sockets.cis doing something similar, but usingmonitor_get_fd()instead.
That sounds like a good idea. socket_get_fd() could become an exported function.
You should move the num != 1 check to socket_listen(), that should be fine.
thanks
Quoted 62 lines
--
JagQuoted 42 lines
hw/remote/remote-obj.c new file +24
-0+object_class_property_add_str (klass,"devid" ,NULL , + remote_object_set_devid); +} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);MAINTAINERS +1
-0@@ -3160,6 +3160,7 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: hw/remote/remote-obj.c Build and test automation -------------------------hw/remote/meson.build +0
-0hw/remote/meson.build remote_ss = ss.source_set() · line 4
Quoted 2 lines
hw/remote/meson.build +0
-0@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:files('mpqemu-link.c'))
hw/remote/meson.build remote_ss = ss.source_set() · line 6
Quoted 2 lines
hw/remote/meson.build +1
-0remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true:files('remote-obj.c'))
Quoted 9 lines
hw/remote/meson.build +0
-1softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss) --1.8.3.1
--
Marc-André Lureau
Signature · 1 line
Marc-André Lureau
Re: [PATCH v13 10/19] multi-process: Associate fd of a PCIDevice with its object
Quoted 221 lines · Marc-André Lureau
Hi
Quoted 208 lines · Jag Raman
On Dec 16, 2020, at 8:43 AM, Marc-André Lureau <[email protected]> wrote:
On Mon, Dec 14, 2020 at 9:15 AM Jagannathan Raman <[email protected]> wrote:
Associate the file descriptor for a PCIDevice in remote process with DeviceState object.Signed-off-by Elena Ufimtseva <[email protected]>Signed-off-by John G Johnson <[email protected]>Signed-off-by Jagannathan Raman <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- hw/remote/remote-obj.c | 206 +++++++++++++++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + 3 files changed, 208 insertions(+) create mode 100644 hw/remote/remote-obj.chw/remote/remote-obj.c new file +182
-0@@ -0,0 +1,206 @@ +/* +* Copyright © 2020 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL-v2, version 2 or later. +* +* See the COPYING file in the top-level directory. +* +*/ + +#include "qemu/osdep.h" +#include "qemu-common.h" + +#include "qemu/error-report.h" +#include "qemu/notify.h" +#include "qom/object_interfaces.h" +#include "hw/qdev-core.h" +#include "io/channel.h" +#include "hw/qdev-core.h" +#include "hw/remote/machine.h" +#include "io/channel-util.h" +#include "qapi/error.h" +#include "sysemu/sysemu.h" +#include "hw/pci/pci.h" +#include "qemu/sockets.h" + +#define TYPE_REMOTE_OBJECT "x-remote-object" +#define REMOTE_OBJECT (obj) \ + OBJECT_CHECK(RemoteObject, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_GET_CLASS (obj) \ + OBJECT_GET_CLASS(RemoteObjectClass, (obj), TYPE_REMOTE_OBJECT) +#define REMOTE_OBJECT_CLASS (klass) \ + OBJECT_CLASS_CHECK(RemoteObjectClass, (klass), TYPE_REMOTE_OBJECT) + +typedef struct { +ObjectClass parent_class; + +unsigned int nr_devs; +unsigned int max_devs; +}RemoteObjectClass ; + +typedef struct { +/* private */ +Object parent; + +Notifier machine_done; + +int32_t fd; +char *devid; + +QIOChannel *ioc; + +DeviceState *dev; +DeviceListener listener; +}RemoteObject ; + +static void remote_object_set_fd (Object *obj,Visitor *v,const char *name, +void *opaque,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); +int32_t value; + +if (!visit_type_int32 (v, name, &value, errp)) { +return ; + } + +if (!fd_is_socket (value)) { + o->fd =-1 ; +error_setg (errp,"File descriptor %d is not a socket" , value); +return ; + } + + o->fd = value; +} + +static void remote_object_set_devid (Object *obj,const char *str,Error **errp) +{ +RemoteObject *o =REMOTE_OBJECT (obj); + +g_free (o->devid); + + o->devid =g_strdup (str); +} + +static void remote_object_release (DeviceListener *listener,DeviceState *dev) +{ +RemoteObject *o =container_of (listener, RemoteObject, listener); + +if (o->dev == dev) { +object_unref (OBJECT (o)); + } +} + +static void remote_object_machine_done (Notifier *notifier,void *data) +{ +RemoteObject *o =container_of (notifier, RemoteObject, machine_done); +DeviceState *dev =NULL ; +QIOChannel *ioc =NULL ; +Coroutine *co =NULL ; +RemoteCommDev *comdev =NULL ; +Error *err =NULL ; + + dev =qdev_find_recursive (sysbus_get_default (), o->devid); +if (!dev || !object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_report ("%s is not a PCI device" , o->devid); +return ; + } + + ioc =qio_channel_new_fd (o->fd, &err); +if (!ioc) { +error_report_err (err); +return ; + } +qio_channel_set_blocking (ioc, false,NULL ); + + o->dev = dev; + + o->listener.unrealize = remote_object_release; +device_listener_register (&o->listener); + +/* co-routine should free this. */ + comdev =g_new0 (RemoteCommDev,1 ); + *comdev = (RemoteCommDev ) { + .ioc = ioc, + .dev =PCI_DEVICE (dev), + }; + + co =qemu_coroutine_create (mpqemu_remote_msg_loop_co, comdev); +qemu_coroutine_enter (co); +} + +static void remote_object_init (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +if (k->nr_devs >= k->max_devs) { +error_report ("Reached maximum number of devices: %u" , k->max_devs); +return ; + } + + o->ioc =NULL ; + o->fd =-1 ; + o->devid =NULL ; + + k->nr_devs++; + + o->machine_done.notify = remote_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); +} + +static void remote_object_finalize (Object *obj) +{ +RemoteObjectClass *k =REMOTE_OBJECT_GET_CLASS (obj); +RemoteObject *o =REMOTE_OBJECT (obj); + +device_listener_unregister (&o->listener); + +if (o->ioc) { +qio_channel_shutdown (o->ioc,QIO_CHANNEL_SHUTDOWN_BOTH ,NULL ); +qio_channel_close (o->ioc,NULL ); + } + +object_unref (OBJECT (o->ioc)); + + k->nr_devs--; +g_free (o->devid); +} + +static void remote_object_class_init (ObjectClass *klass,void *data) +{ +RemoteObjectClass *k =REMOTE_OBJECT_CLASS (klass); + +/* +* Limit number of supported devices to 1. This is done to avoid devices +* from one VM accessing the RAM of another VM. This is done until we +* start using separate address spaces for individual devices. +*/ + k->max_devs =1 ; + k->nr_devs =0 ; + +object_class_property_add (klass,"fd" ,"int32_t" ,NULL , + remote_object_set_fd,NULL ,NULL );Quoted 1 line
Let's be consistent with pci-proxy-dev, and make it a string property that is parsed with
monitor_fd_param()Thank you for the feedback, Marc-Andre!
Quoted 8 lines
Would it be alright if we added this to
util/qemu-sockets.csince this logic is used more than once, or is it an overkill? I saw thatsocket_get_fd()inqemu-sockets.cis doing something similar, but usingmonitor_get_fd()instead.That sounds like a good idea.
socket_get_fd()could become an exported function.You should move the num != 1 check to
socket_listen(), that should be fine.
Thank you for confirming!
—
Jag
Quoted 66 lines
thanks
--
JagQuoted 55 lines
hw/remote/remote-obj.c new file +24
-0+object_class_property_add_str (klass,"devid" ,NULL , + remote_object_set_devid); +} + +static const TypeInfo remote_object_info = { + .name =TYPE_REMOTE_OBJECT , + .parent =TYPE_OBJECT , + .instance_size =sizeof (RemoteObject), + .instance_init = remote_object_init, + .instance_finalize = remote_object_finalize, + .class_size =sizeof (RemoteObjectClass), + .class_init = remote_object_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + { } + } +}; + +static void register_types (void ) +{ +type_register_static (&remote_object_info); +} + +type_init (register_types);MAINTAINERS +1
-0@@ -3160,6 +3160,7 @@ F: include/hw/remote/machine.h F: hw/remote/mpqemu-link.c F: include/hw/remote/mpqemu-link.h F: hw/remote/message.c +F: hw/remote/remote-obj.c Build and test automation -------------------------hw/remote/meson.build +1
-0@@ -3,5 +3,6 @@ remote_ss = ss.source_set() remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)Signature · 6 lines
1.8.3.1 -- Marc-André LureauSignature · 1 line
Marc-André Lureau