vfio-user server in QEMU
v11 cover vfio-user server in QEMU
Hi,
This is v11 of the server side changes to enable vfio-user in QEMU.
Thank you for reviewing and sharing your feedback for the previous revision. We have addressed your comments in this revision.
We made the following changes in this series:
[PATCH v11 13/14] vfio-user: handle device interrupts
- Added msi_set_irq_state() and msix_set_irq_state() to mask and unmask individual MSI(x) vectors
- Implement callbacks to handle the MASK/UNMASK actions initiated by SET_IRQS message
- vfu_object_set_bus_irq() sets the maximum number of IRQS to max BDF. This only affects devices using INTx - allows multiple devices to use INTx
Thank you very much!
Jagannathan Raman (14):
qdev: unplug blocker for devices remote/machine: add HotplugHandler for remote machine remote/machine: add vfio-user property
vfio-user: build library
vfio-user: define vfio-user-server object
vfio-user: instantiate vfio-user context
vfio-user: find and init PCI device
vfio-user: run vfio-user context
vfio-user: handle PCI config space accesses
vfio-user: IOMMU support for remote device
vfio-user: handle DMA mappings
vfio-user: handle PCI BAR accesses
vfio-user: handle device interrupts
vfio-user: handle reset of remote device
configure | 17 + meson.build | 23 +- qapi/misc.json | 31 + qapi/qom.json | 20 +- include/exec/memory.h | 3 + include/hw/pci/msi.h | 1 + include/hw/pci/msix.h | 1 + include/hw/pci/pci.h | 13 + include/hw/qdev-core.h | 29 + include/hw/remote/iommu.h | 40 + include/hw/remote/machine.h | 4 + include/hw/remote/vfio-user-obj.h | 6 + hw/core/qdev.c | 24 + hw/pci/msi.c | 48 +- hw/pci/msix.c | 35 +- hw/pci/pci.c | 13 + hw/remote/iommu.c | 131 ++++ hw/remote/machine.c | 88 ++- hw/remote/vfio-user-obj.c | 958 ++++++++++++++++++++++++ softmmu/physmem.c | 4 +- softmmu/qdev-monitor.c | 4 + stubs/vfio-user-obj.c | 6 + tests/qtest/fuzz/generic_fuzz.c | 9 +- .gitlab-ci.d/buildtest.yml | 1 + .gitmodules | 3 + Kconfig.host | 4 + MAINTAINERS | 5 + hw/remote/Kconfig | 4 + hw/remote/meson.build | 4 + hw/remote/trace-events | 11 + meson_options.txt | 2 + stubs/meson.build | 1 + subprojects/libvfio-user | 1 + tests/docker/dockerfiles/centos8.docker | 2 + 34 files changed, 1527 insertions(+), 19 deletions(-) create mode 100644 include/hw/remote/iommu.h create mode 100644 include/hw/remote/vfio-user-obj.h create mode 100644 hw/remote/iommu.c create mode 100644 hw/remote/vfio-user-obj.c create mode 100644 stubs/vfio-user-obj.c create mode 160000 subprojects/libvfio-user
Signature · 1 line
2.20.1
PATCH v11 1/14 qdev: unplug blocker for devices
Add blocker to prevent hot-unplug of devices
TYPE_VFIO_USER_SERVER, which is introduced shortly, attaches itself to a PCIDevice on which it depends. If the attached PCIDevice gets removed while the server in use, it could cause it crash. To prevent this, TYPE_VFIO_USER_SERVER adds an unplug blocker for the PCIDevice.
--- include/hw/qdev-core.h | 29 +++++++++++++++++++++++++++++ hw/core/qdev.c | 24 ++++++++++++++++++++++++ softmmu/qdev-monitor.c | 4 ++++ 3 files changed, 57 insertions(+)
include/hw/qdev-core.h +29 -0
@@ -193,6 +193,7 @@ struct DeviceState {int instance_id_alias;int alias_required_for_version;ResettableState reset; +GSList *unplug_blockers; };struct DeviceListener {
@@ -419,6 +420,34 @@ void qdev_simple_device_unplug_cb(HotplugHandler *hotplug_dev,void qdev_machine_creation_done (void );bool qdev_machine_modified (void ); +/** +* qdev_add_unplug_blocker: Add an unplug blocker to a device +* +* @dev: Device to be blocked from unplug +* @reason: Reason for blocking +*/ +void qdev_add_unplug_blocker (DeviceState *dev,Error *reason); + +/** +* qdev_del_unplug_blocker: Remove an unplug blocker from a device +* +* @dev: Device to be unblocked +* @reason: Pointer to the Error used with qdev_add_unplug_blocker. +* Used as a handle to lookup the blocker for deletion. +*/ +void qdev_del_unplug_blocker (DeviceState *dev,Error *reason); + +/** +* qdev_unplug_blocked: Confirm if a device is blocked from unplug +* +* @dev: Device to be tested +* @reason: Returns one of the reasons why the device is blocked, +* if any +* +* Returns: true if device is blocked from unplug, false otherwise +*/ +bool qdev_unplug_blocked (DeviceState *dev,Error **errp); +/** * GpioPolarity: Polarity of a GPIO line *
hw/core/qdev.c +24 -0
@@ -468,6 +468,28 @@ char *qdev_get_dev_path(DeviceState *dev)return NULL ; } +void qdev_add_unplug_blocker (DeviceState *dev,Error *reason) +{ + dev->unplug_blockers =g_slist_prepend (dev->unplug_blockers, reason); +} + +void qdev_del_unplug_blocker (DeviceState *dev,Error *reason) +{ + dev->unplug_blockers =g_slist_remove (dev->unplug_blockers, reason); +} + +bool qdev_unplug_blocked (DeviceState *dev,Error **errp) +{ +ERRP_GUARD (); + +if (dev->unplug_blockers) { +error_propagate (errp,error_copy (dev->unplug_blockers->data)); +return true; + } + +return false; +} +static bool device_get_realized (Object *obj,Error **errp) {DeviceState *dev =DEVICE (obj);
@@ -704,6 +726,8 @@ static void device_finalize(Object *obj)DeviceState *dev =DEVICE (obj); +g_assert (!dev->unplug_blockers); + QLIST_FOREACH_SAFE(ngl, &dev->gpios, node, next) {QLIST_REMOVE (ngl, node);qemu_free_irqs (ngl->in, ngl->num_in);
softmmu/qdev-monitor.c +4 -0
@@ -899,6 +899,10 @@ void qdev_unplug(DeviceState *dev, Error **errp)HotplugHandlerClass *hdc;Error *local_err =NULL ; +if (qdev_unplug_blocked (dev, errp)) { +return ; + } +if (dev->parent_bus && !qbus_is_hotpluggable (dev->parent_bus)) {error_setg (errp,QERR_BUS_NO_HOTPLUG , dev->parent_bus->name);return ;
Signature · 1 line
2.20.1
PATCH v11 2/14 remote/machine: add HotplugHandler for remote machine
In reply to Jagannathan RamanAllow hotplugging of PCI(e) devices to remote machine
--- hw/remote/machine.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
hw/remote/machine.c +10 -0
@@ -20,6 +20,7 @@#include "qapi/error.h" #include "hw/pci/pci_host.h" #include "hw/remote/iohub.h" +#include "hw/qdev-core.h" static void remote_machine_init (MachineState *machine) {
@@ -53,14 +54,19 @@ static void remote_machine_init(MachineState *machine)pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, &s->iohub,REMOTE_IOHUB_NB_PIRQS ); + +qbus_set_hotplug_handler (BUS (pci_host->bus),OBJECT (s)); }static void remote_machine_class_init (ObjectClass *oc,void *data) {MachineClass *mc =MACHINE_CLASS (oc); +HotplugHandlerClass *hc =HOTPLUG_HANDLER_CLASS (oc); mc->init = remote_machine_init; mc->desc ="Experimental remote machine" ; + + hc->unplug = qdev_simple_device_unplug_cb; }static const TypeInfo remote_machine = {
@@ -68,6 +74,10 @@ static const TypeInfo remote_machine = { .parent = TYPE_MACHINE, .instance_size = sizeof(RemoteMachineState), .class_init = remote_machine_class_init, + .interfaces = (InterfaceInfo []) { + {TYPE_HOTPLUG_HANDLER }, + { } + } };static void remote_machine_register_types (void )
Signature · 1 line
2.20.1
PATCH v11 3/14 remote/machine: add vfio-user property
In reply to Jagannathan RamanAdd vfio-user to x-remote machine. It is a boolean, which indicates if the machine supports vfio-user protocol. The machine configures the bus differently vfio-user and multiprocess protocols, so this property informs it on how to configure the bus.
This property should be short lived. Once vfio-user fully replaces multiprocess, this property could be removed.
--- include/hw/remote/machine.h | 2 ++ hw/remote/machine.c | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+)
include/hw/remote/machine.h +2 -0
@@ -22,6 +22,8 @@ struct RemoteMachineState {RemotePCIHost *host;RemoteIOHubState iohub; + +bool vfio_user; };/* Used to pass to co-routine device and ioc. */
hw/remote/machine.c +23 -0
@@ -58,6 +58,25 @@ static void remote_machine_init(MachineState *machine)qbus_set_hotplug_handler (BUS (pci_host->bus),OBJECT (s)); } +static bool remote_machine_get_vfio_user (Object *obj,Error **errp) +{ +RemoteMachineState *s =REMOTE_MACHINE (obj); + +return s->vfio_user; +} + +static void remote_machine_set_vfio_user (Object *obj,bool value,Error **errp) +{ +RemoteMachineState *s =REMOTE_MACHINE (obj); + +if (phase_check (PHASE_MACHINE_CREATED )) { +error_setg (errp,"Error enabling vfio-user - machine already created" ); +return ; + } + + s->vfio_user = value; +} +static void remote_machine_class_init (ObjectClass *oc,void *data) {MachineClass *mc =MACHINE_CLASS (oc);
@@ -67,6 +86,10 @@ static void remote_machine_class_init(ObjectClass *oc, void *data) mc->desc ="Experimental remote machine" ; hc->unplug = qdev_simple_device_unplug_cb; + +object_class_property_add_bool (oc,"vfio-user" , + remote_machine_get_vfio_user, + remote_machine_set_vfio_user); }static const TypeInfo remote_machine = {
Signature · 1 line
2.20.1
PATCH v11 4/14 vfio-user: build library
In reply to Jagannathan Ramanadd the libvfio-user library as a submodule. build it as a meson subproject.
libvfio-user is distributed with BSD 3-Clause license and json-c with MIT (Expat) license
--- configure | 17 +++++++++++++++++ meson.build | 23 ++++++++++++++++++++++- .gitlab-ci.d/buildtest.yml | 1 + .gitmodules | 3 +++ Kconfig.host | 4 ++++ MAINTAINERS | 1 + hw/remote/Kconfig | 4 ++++ hw/remote/meson.build | 2 ++ meson_options.txt | 2 ++ subprojects/libvfio-user | 1 + tests/docker/dockerfiles/centos8.docker | 2 ++ 11 files changed, 59 insertions(+), 1 deletion(-) create mode 160000 subprojects/libvfio-user
configure +17 -0
@@ -315,6 +315,7 @@ meson_args="" ninja="" bindir="bin" skip_meson=no +vfio_user_server="disabled" # The following Meson options are handled manually (still they # are included in the automatically generated help message)
@@ -909,6 +910,10 @@ for opt do ;; --disable-blobs) meson_option_parse --disable-install-blobs "" ;; + --enable-vfio-user-server) vfio_user_server="enabled" + ;; + --disable-vfio-user-server) vfio_user_server="disabled" + ;; --enable-tcmalloc) meson_option_parse --enable-malloc=tcmalloc tcmalloc ;; --enable-jemalloc) meson_option_parse --enable-malloc=jemalloc jemalloc
@@ -2133,6 +2138,17 @@ write_container_target_makefile() { +########################################## +# check for vfio_user_server + +case "$vfio_user_server" in + enabled ) + if test "$git_submodules_action" != "ignore"; then + git_submodules="${git_submodules} subprojects/libvfio-user" + fi + ;; +esac + ########################################## # End of CC checks # After here, no more $cc or $ld runs
@@ -2669,6 +2685,7 @@ if test "$skip_meson" = no; then test "$slirp" != auto && meson_option_add "-Dslirp=$slirp" test "$smbd" != '' && meson_option_add "-Dsmbd=$smbd" test "$tcg" != enabled && meson_option_add "-Dtcg=$tcg" + test "$vfio_user_server" != auto && meson_option_add "-Dvfio_user_server=$vfio_user_server" run_meson() { NINJA=$ninja $meson setup --prefix "$prefix" "$@" $cross_arg "$PWD" "$source_path" }
meson.build +22 -1
@@ -308,6 +308,10 @@ multiprocess_allowed = get_option('multiprocess') \ .require(targetos == 'linux', error_message: 'Multiprocess QEMU is supported only on Linux') \ .allowed() +vfio_user_server_allowed = get_option('vfio_user_server') \ + .require(targetos == 'linux', error_message: 'vfio-user server is supported only on Linux') \ + .allowed() + have_tpm = get_option('tpm') \ .require(targetos != 'windows', error_message: 'TPM emulation only available on POSIX systems') \ .allowed()
@@ -2373,7 +2377,8 @@ host_kconfig = \ (have_virtfs ? ['CONFIG_VIRTFS=y'] : []) + \ ('CONFIG_LINUX' in config_host ? ['CONFIG_LINUX=y'] : []) + \ (have_pvrdma ? ['CONFIG_PVRDMA=y'] : []) + \ - (multiprocess_allowed ? ['CONFIG_MULTIPROCESS_ALLOWED=y'] : []) + (multiprocess_allowed ? ['CONFIG_MULTIPROCESS_ALLOWED=y'] : []) + \ + (vfio_user_server_allowed ? ['CONFIG_VFIO_USER_SERVER_ALLOWED=y'] : []) ignored = [ 'TARGET_XML_FILES', 'TARGET_ABI_DIR', 'TARGET_ARCH' ]
@@ -2665,6 +2670,21 @@ if have_system endif endif +libvfio_user_dep = not_found +if have_system and vfio_user_server_allowed + have_internal = fs.exists(meson.current_source_dir() / 'subprojects/libvfio-user/meson.build') + + if not have_internal + error('libvfio-user source not found - please pull git submodule') + endif + + libvfio_user_proj = subproject('libvfio-user') + + libvfio_user_lib = libvfio_user_proj.get_variable('libvfio_user_dep') + + libvfio_user_dep = declare_dependency(dependencies: [libvfio_user_lib]) +endif + fdt = not_found if have_system fdt_opt = get_option('fdt')
@@ -3783,6 +3803,7 @@ summary_info += {'target list': ' '.join(target_dirs)} if have_system summary_info += {'default devices': get_option('default_devices')} summary_info += {'out of process emulation': multiprocess_allowed} + summary_info += {'vfio-user server': vfio_user_server_allowed} endif summary(summary_info, bool_yn: true, section: 'Targets and accelerators')
.gitlab-ci.d/buildtest.yml +1 -0
@@ -168,6 +168,7 @@ build-system-centos: IMAGE: centos8 CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-fdt=system --enable-modules --enable-trace-backends=dtrace --enable-docs + --enable-vfio-user-server TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu MAKE_CHECK_ARGS: check-build
.gitmodules +3 -0
@@ -64,3 +64,6 @@ [submodule "tests/lcitool/libvirt-ci"] path = tests/lcitool/libvirt-ci url = https://gitlab.com/libvirt/libvirt-ci.git +[submodule "subprojects/libvfio-user"] + path = subprojects/libvfio-user + url = https://github.com/nutanix/libvfio-user.git
Kconfig.host +4 -0
@@ -42,3 +42,7 @@ config MULTIPROCESS_ALLOWED config FUZZ bool select SPARSE_MEM + +config VFIO_USER_SERVER_ALLOWED + bool + imply VFIO_USER_SERVER
MAINTAINERS +1 -0
@@ -3631,6 +3631,7 @@ F: hw/remote/proxy-memory-listener.c F: include/hw/remote/proxy-memory-listener.h F: hw/remote/iohub.c F: include/hw/remote/iohub.h +F: subprojects/libvfio-user EBPF: M: Jason Wang <[email protected]>
hw/remote/Kconfig +4 -0
@@ -2,3 +2,7 @@ config MULTIPROCESS bool depends on PCI && PCI_EXPRESS && KVM select REMOTE_PCIHOST + +config VFIO_USER_SERVER + bool + depends on MULTIPROCESS
hw/remote/meson.build +2 -0
@@ -7,6 +7,8 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) +remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: libvfio_user_dep) + specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('memory.c')) specific_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy-memory-listener.c'))
meson_options.txt +2 -0
@@ -88,6 +88,8 @@ option('cfi_debug', type: 'boolean', value: 'false', description: 'Verbose errors in case of CFI violation') option('multiprocess', type: 'feature', value: 'auto', description: 'Out of process device emulation support') +option('vfio_user_server', type: 'feature', value: 'disabled', + description: 'vfio-user server support') option('dbus_display', type: 'feature', value: 'auto', description: '-display dbus support') option('tpm', type : 'feature', value : 'auto',
subprojects/libvfio-user new file +1 -0
@@ -0,0 +1 @@ +Subproject commit 0b28d205572c80b568a1003db2c8f37ca333e4d7
tests/docker/dockerfiles/centos8.docker +2 -0
@@ -51,6 +51,7 @@ RUN dnf update -y && \ libbpf-devel \ libcacard-devel \ libcap-ng-devel \ + libcmocka-devel \ libcurl-devel \ libdrm-devel \ libepoxy-devel \
@@ -59,6 +60,7 @@ RUN dnf update -y && \ libgcrypt-devel \ libiscsi-devel \ libjpeg-devel \ + json-c-devel \ libnfs-devel \ libpmem-devel \ libpng-devel \
Signature · 1 line
2.20.1
PATCH v11 5/14 vfio-user: define vfio-user-server object
In reply to Jagannathan RamanDefine vfio-user object which is remote process server for QEMU. Setup object initialization functions and properties necessary to instantiate the object
--- qapi/qom.json | 20 +++- include/hw/remote/machine.h | 2 + hw/remote/machine.c | 27 +++++ hw/remote/vfio-user-obj.c | 210 ++++++++++++++++++++++++++++++++++++ MAINTAINERS | 1 + hw/remote/meson.build | 1 + hw/remote/trace-events | 3 + 7 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 hw/remote/vfio-user-obj.c
qapi/qom.json +18 -2
@@ -734,6 +734,20 @@ { 'struct': 'RemoteObjectProperties', 'data': { 'fd': 'str', 'devid': 'str' } } +## +# @VfioUserServerProperties: +# +# Properties for x-vfio-user-server objects. +# +# @socket: socket to be used by the libvfio-user library +# +# @device: the ID of the device to be emulated at the server +# +# Since: 7.1 +## +{ 'struct': 'VfioUserServerProperties', + 'data': { 'socket': 'SocketAddress', 'device': 'str' } } + ## # @RngProperties: #
@@ -874,7 +888,8 @@ 'tls-creds-psk', 'tls-creds-x509', 'tls-cipher-suites', - { 'name': 'x-remote-object', 'features': [ 'unstable' ] } + { 'name': 'x-remote-object', 'features': [ 'unstable' ] }, + { 'name': 'x-vfio-user-server', 'features': [ 'unstable' ] } ] } ##
@@ -938,7 +953,8 @@ 'tls-creds-psk': 'TlsCredsPskProperties', 'tls-creds-x509': 'TlsCredsX509Properties', 'tls-cipher-suites': 'TlsCredsProperties', - 'x-remote-object': 'RemoteObjectProperties' + 'x-remote-object': 'RemoteObjectProperties', + 'x-vfio-user-server': 'VfioUserServerProperties' } } ##
include/hw/remote/machine.h +2 -0
@@ -24,6 +24,8 @@ struct RemoteMachineState {RemoteIOHubState iohub;bool vfio_user; + +bool auto_shutdown; };/* Used to pass to co-routine device and ioc. */
hw/remote/machine.c +27 -0
@@ -77,6 +77,28 @@ static void remote_machine_set_vfio_user(Object *obj, bool value, Error **errp) s->vfio_user = value; } +static bool remote_machine_get_auto_shutdown (Object *obj,Error **errp) +{ +RemoteMachineState *s =REMOTE_MACHINE (obj); + +return s->auto_shutdown; +} + +static void remote_machine_set_auto_shutdown (Object *obj,bool value, +Error **errp) +{ +RemoteMachineState *s =REMOTE_MACHINE (obj); + + s->auto_shutdown = value; +} + +static void remote_machine_instance_init (Object *obj) +{ +RemoteMachineState *s =REMOTE_MACHINE (obj); + + s->auto_shutdown = true; +} +static void remote_machine_class_init (ObjectClass *oc,void *data) {MachineClass *mc =MACHINE_CLASS (oc);
@@ -90,12 +112,17 @@ static void remote_machine_class_init(ObjectClass *oc, void *data)object_class_property_add_bool (oc,"vfio-user" , remote_machine_get_vfio_user, remote_machine_set_vfio_user); + +object_class_property_add_bool (oc,"auto-shutdown" , + remote_machine_get_auto_shutdown, + remote_machine_set_auto_shutdown); }static const TypeInfo remote_machine = { .name =TYPE_REMOTE_MACHINE , .parent =TYPE_MACHINE , .instance_size =sizeof (RemoteMachineState), + .instance_init = remote_machine_instance_init, .class_init = remote_machine_class_init, .interfaces = (InterfaceInfo []) { {TYPE_HOTPLUG_HANDLER },
hw/remote/vfio-user-obj.c new file +210 -0
210 lines not on this page · raw message
MAINTAINERS +1 -0
@@ -3632,6 +3632,7 @@ F: include/hw/remote/proxy-memory-listener.h F: hw/remote/iohub.c F: include/hw/remote/iohub.h F: subprojects/libvfio-user +F: hw/remote/vfio-user-obj.c EBPF: M: Jason Wang <[email protected]>
hw/remote/meson.build +1 -0
@@ -6,6 +6,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) +remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: files('vfio-user-obj.c')) remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: libvfio_user_dep)
hw/remote/trace-events +3 -0
@@ -2,3 +2,6 @@ mpqemu_send_io_error(int cmd, int size, int nfds) "send command %d size %d, %d file descriptors to remote process" mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, %d file descriptors to remote process" + +# vfio-user-obj.c +vfu_prop(const char *prop, const char *val) "vfu: setting %s as %s"
Signature · 1 line
2.20.1
PATCH v11 6/14 vfio-user: instantiate vfio-user context
In reply to Jagannathan Ramancreate a context with the vfio-user library to run a PCI device
--- hw/remote/vfio-user-obj.c | 82 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+)
hw/remote/vfio-user-obj.c +82 -0
@@ -40,6 +40,9 @@#include "hw/remote/machine.h" #include "qapi/error.h" #include "qapi/qapi-visit-sockets.h" +#include "qemu/notify.h" +#include "sysemu/sysemu.h" +#include "libvfio-user.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )
@@ -73,8 +76,14 @@ struct VfuObject {char *device;Error *err; + +Notifier machine_done; + +vfu_ctx_t *vfu_ctx; }; +static void vfu_object_init_ctx (VfuObject *o,Error **errp); +static bool vfu_object_auto_shutdown (void ) {bool auto_shutdown = true;
@@ -107,6 +116,11 @@ static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, {VfuObject *o =VFU_OBJECT (obj); +if (o->vfu_ctx) { +error_setg (errp,"vfu: Unable to set socket property - server busy" ); +return ; + } +qapi_free_SocketAddress (o->socket); o->socket =NULL ;
@@ -122,17 +136,69 @@ static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, }trace_vfu_prop ("socket" , o->socket->u.q_unix.path); + +vfu_object_init_ctx (o, errp); }static void vfu_object_set_device (Object *obj,const char *str,Error **errp) {VfuObject *o =VFU_OBJECT (obj); +if (o->vfu_ctx) { +error_setg (errp,"vfu: Unable to set device property - server busy" ); +return ; + } +g_free (o->device); o->device =g_strdup (str);trace_vfu_prop ("device" , str); + +vfu_object_init_ctx (o, errp); +} + +/* +* TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' +* properties. It also depends on devices instantiated in QEMU. These +* dependencies are not available during the instance_init phase of this +* object's life-cycle. As such, the server is initialized after the +* machine is setup. machine_init_done_notifier notifies TYPE_VFU_OBJECT +* when the machine is setup, and the dependencies are available. +*/ +static void vfu_object_machine_done (Notifier *notifier,void *data) +{ +VfuObject *o =container_of (notifier, VfuObject, machine_done); +Error *err =NULL ; + +vfu_object_init_ctx (o, &err); + +if (err) { +error_propagate (&error_abort, err); + } +} + +static void vfu_object_init_ctx (VfuObject *o,Error **errp) +{ +ERRP_GUARD (); + +if (o->vfu_ctx || !o->socket || !o->device || + !phase_check (PHASE_MACHINE_READY )) { +return ; + } + +if (o->err) { +error_propagate (errp, o->err); + o->err =NULL ; +return ; + } + + o->vfu_ctx =vfu_create_ctx (VFU_TRANS_SOCK , o->socket->u.q_unix.path,0 , + o,VFU_DEV_TYPE_PCI ); +if (o->vfu_ctx ==NULL ) { +error_setg (errp,"vfu: Failed to create context - %s" ,strerror (errno)); +return ; + } }static void vfu_object_init (Object *obj)
@@ -147,6 +213,12 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE);return ; } + +if (!phase_check (PHASE_MACHINE_READY )) { + o->machine_done.notify = vfu_object_machine_done; +qemu_add_machine_init_done_notifier (&o->machine_done); + } + }static void vfu_object_finalize (Object *obj)
@@ -160,6 +232,11 @@ static void vfu_object_finalize(Object *obj) o->socket =NULL ; +if (o->vfu_ctx) { +vfu_destroy_ctx (o->vfu_ctx); + o->vfu_ctx =NULL ; + } +g_free (o->device); o->device =NULL ;
@@ -167,6 +244,11 @@ static void vfu_object_finalize(Object *obj)if (!k->nr_devs &&vfu_object_auto_shutdown ()) {qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); } + +if (o->machine_done.notify) { +qemu_remove_machine_init_done_notifier (&o->machine_done); + o->machine_done.notify =NULL ; + } }static void vfu_object_class_init (ObjectClass *klass,void *data)
Signature · 1 line
2.20.1
PATCH v11 7/14 vfio-user: find and init PCI device
In reply to Jagannathan RamanFind the PCI device with specified id. Initialize the device context with the QEMU PCI device
--- hw/remote/vfio-user-obj.c | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+)
hw/remote/vfio-user-obj.c +67 -0
@@ -43,6 +43,8 @@#include "qemu/notify.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" +#include "hw/qdev-core.h" +#include "hw/pci/pci.h" #define TYPE_VFU_OBJECT "x-vfio-user-server"OBJECT_DECLARE_TYPE (VfuObject, VfuObjectClass,VFU_OBJECT )
@@ -80,6 +82,10 @@ struct VfuObject {Notifier machine_done;vfu_ctx_t *vfu_ctx; + +PCIDevice *pci_dev; + +Error *unplug_blocker; };static void vfu_object_init_ctx (VfuObject *o,Error **errp);
@@ -181,6 +187,9 @@ static void vfu_object_machine_done(Notifier *notifier, void *data)static void vfu_object_init_ctx (VfuObject *o,Error **errp) {ERRP_GUARD (); +DeviceState *dev =NULL ; +vfu_pci_type_t pci_type =VFU_PCI_TYPE_CONVENTIONAL ; +int ret;if (o->vfu_ctx || !o->socket || !o->device || !phase_check (PHASE_MACHINE_READY )) {
@@ -199,6 +208,53 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp)error_setg (errp,"vfu: Failed to create context - %s" ,strerror (errno));return ; } + + dev =qdev_find_recursive (sysbus_get_default (), o->device); +if (dev ==NULL ) { +error_setg (errp,"vfu: Device %s not found" , o->device); + goto fail; + } + +if (!object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +error_setg (errp,"vfu: %s not a PCI device" , o->device); + goto fail; + } + + o->pci_dev =PCI_DEVICE (dev); + +object_ref (OBJECT (o->pci_dev)); + +if (pci_is_express (o->pci_dev)) { + pci_type =VFU_PCI_TYPE_EXPRESS ; + } + + ret =vfu_pci_init (o->vfu_ctx, pci_type,PCI_HEADER_TYPE_NORMAL ,0 ); +if (ret <0 ) { +error_setg (errp, +"vfu: Failed to attach PCI device %s to context - %s" , + o->device,strerror (errno)); + goto fail; + } + +error_setg (&o->unplug_blocker, +"vfu: %s for %s must be deleted before unplugging" , +TYPE_VFU_OBJECT , o->device); +qdev_add_unplug_blocker (DEVICE (o->pci_dev), o->unplug_blocker); + +return ; + +fail: +vfu_destroy_ctx (o->vfu_ctx); +if (o->unplug_blocker && o->pci_dev) { +qdev_del_unplug_blocker (DEVICE (o->pci_dev), o->unplug_blocker); +error_free (o->unplug_blocker); + o->unplug_blocker =NULL ; + } +if (o->pci_dev) { +object_unref (OBJECT (o->pci_dev)); + o->pci_dev =NULL ; + } + o->vfu_ctx =NULL ; }static void vfu_object_init (Object *obj)
@@ -241,6 +297,17 @@ static void vfu_object_finalize(Object *obj) o->device =NULL ; +if (o->unplug_blocker && o->pci_dev) { +qdev_del_unplug_blocker (DEVICE (o->pci_dev), o->unplug_blocker); +error_free (o->unplug_blocker); + o->unplug_blocker =NULL ; + } + +if (o->pci_dev) { +object_unref (OBJECT (o->pci_dev)); + o->pci_dev =NULL ; + } +if (!k->nr_devs &&vfu_object_auto_shutdown ()) {qemu_system_shutdown_request (SHUTDOWN_CAUSE_GUEST_SHUTDOWN ); }
Signature · 1 line
2.20.1
PATCH v11 8/14 vfio-user: run vfio-user context
In reply to Jagannathan RamanSetup a handler to run vfio-user context. The context is driven by messages to the file descriptor associated with it - get the fd for the context and hook up the handler with it
--- qapi/misc.json | 31 ++++++++++ hw/remote/vfio-user-obj.c | 118 +++++++++++++++++++++++++++++++++++++- 2 files changed, 148 insertions(+), 1 deletion(-)
qapi/misc.json +31 -0
@@ -553,3 +553,34 @@ ## { 'event': 'RTC_CHANGE', 'data': { 'offset': 'int', 'qom-path': 'str' } } + +## +# @VFU_CLIENT_HANGUP: +# +# Emitted when the client of a TYPE_VFIO_USER_SERVER closes the +# communication channel +# +# @vfu-id: ID of the TYPE_VFIO_USER_SERVER object. It is the last component +# of @vfu-qom-path referenced below +# +# @vfu-qom-path: path to the TYPE_VFIO_USER_SERVER object in the QOM tree +# +# @dev-id: ID of attached PCI device +# +# @dev-qom-path: path to attached PCI device in the QOM tree +# +# Since: 7.1 +# +# Example: +# +# <- { "event": "VFU_CLIENT_HANGUP", +# "data": { "vfu-id": "vfu1", +# "vfu-qom-path": "/objects/vfu1", +# "dev-id": "sas1", +# "dev-qom-path": "/machine/peripheral/sas1" }, +# "timestamp": { "seconds": 1265044230, "microseconds": 450486 } } +# +## +{ 'event': 'VFU_CLIENT_HANGUP', + 'data': { 'vfu-id': 'str', 'vfu-qom-path': 'str', + 'dev-id': 'str', 'dev-qom-path': 'str' } }
hw/remote/vfio-user-obj.c +117 -1
177 lines not on this page · raw message
Signature · 1 line
2.20.1
PATCH v11 9/14 vfio-user: handle PCI config space accesses
In reply to Jagannathan RamanDefine and register handlers for PCI config space accesses
--- hw/remote/vfio-user-obj.c | 51 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 2 ++ 2 files changed, 53 insertions(+)
hw/remote/vfio-user-obj.c +51 -0
@@ -46,6 +46,7 @@#include "qapi/qapi-events-misc.h" #include "qemu/notify.h" #include "qemu/thread.h" +#include "qemu/main-loop.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" #include "hw/qdev-core.h"
@@ -244,6 +245,45 @@ retry_attach:qemu_set_fd_handler (o->vfu_poll_fd, vfu_object_ctx_run,NULL , o); } +static ssize_t vfu_object_cfg_access (vfu_ctx_t *vfu_ctx,char *const buf, +size_t count,loff_t offset, +const bool is_write) +{ +VfuObject *o =vfu_get_private (vfu_ctx); +uint32_t pci_access_width =sizeof (uint32_t ); +size_t bytes = count; +uint32_t val =0 ; +char *ptr = buf; +int len; + +/* +* Writes to the BAR registers would trigger an update to the +* global Memory and IO AddressSpaces. But the remote device +* never uses the global AddressSpaces, therefore overlapping +* memory regions are not a problem +*/ +while (bytes >0 ) { + len = (bytes > pci_access_width) ? pci_access_width : bytes; +if (is_write) { +memcpy (&val, ptr, len); +pci_host_config_write_common (o->pci_dev, offset, +pci_config_size (o->pci_dev), + val, len); +trace_vfu_cfg_write (offset, val); + }else { + val =pci_host_config_read_common (o->pci_dev, offset, +pci_config_size (o->pci_dev), len); +memcpy (ptr, &val, len); +trace_vfu_cfg_read (offset, val); + } + offset += len; + ptr += len; + bytes -= len; + } + +return count; +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
@@ -336,6 +376,17 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) TYPE_VFU_OBJECT, o->device);qdev_add_unplug_blocker (DEVICE (o->pci_dev), o->unplug_blocker); + ret =vfu_setup_region (o->vfu_ctx,VFU_PCI_DEV_CFG_REGION_IDX , +pci_config_size (o->pci_dev), &vfu_object_cfg_access, +VFU_REGION_FLAG_RW |VFU_REGION_FLAG_ALWAYS_CB , +NULL ,0 ,-1 ,0 ); +if (ret <0 ) { +error_setg (errp, +"vfu: Failed to setup config space handlers for %s- %s" , + o->device,strerror (errno)); + goto fail; + } + ret =vfu_realize_ctx (o->vfu_ctx);if (ret <0 ) { error_setg(errp, "vfu: Failed to realize device %s- %s",
hw/remote/trace-events +2 -0
@@ -5,3 +5,5 @@ mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, # vfio-user-obj.c vfu_prop(const char *prop, const char *val) "vfu: setting %s as %s" +vfu_cfg_read(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u -> 0x%x" +vfu_cfg_write(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u <- 0x%x"
Signature · 1 line
2.20.1
PATCH v11 10/14 vfio-user: IOMMU support for remote device
In reply to Jagannathan RamanAssign separate address space for each device in the remote processes.
--- include/hw/remote/iommu.h | 40 ++++++++++++ hw/remote/iommu.c | 131 ++++++++++++++++++++++++++++++++++++++ hw/remote/machine.c | 13 +++- MAINTAINERS | 2 + hw/remote/meson.build | 1 + 5 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 include/hw/remote/iommu.h create mode 100644 hw/remote/iommu.c
include/hw/remote/iommu.h new file +40 -0
@@ -0,0 +1,40 @@ +/** +* Copyright © 2022 Oracle and/or its affiliates. +* +* This work is licensed under the terms of the GNU GPL, version 2 or later. +* See the COPYING file in the top-level directory. +* +*/ + +#ifndef REMOTE_IOMMU_H +#define REMOTE_IOMMU_H + +#include "hw/pci/pci_bus.h" +#include "hw/pci/pci.h" + +#ifndef INT2VOIDP +#define INT2VOIDP (i) (void *)(uintptr_t)(i) +#endif + +typedef struct RemoteIommuElem { +MemoryRegion *mr; + +AddressSpace as; +}RemoteIommuElem ; + +#define TYPE_REMOTE_IOMMU "x-remote-iommu" +OBJECT_DECLARE_SIMPLE_TYPE(RemoteIommu, REMOTE_IOMMU) + +struct RemoteIommu { +Object parent; + +GHashTable *elem_by_devfn; + +QemuMutex lock; +}; + +void remote_iommu_setup (PCIBus *pci_bus); + +void remote_iommu_unplug_dev (PCIDevice *pci_dev); + +#endif
hw/remote/iommu.c new file +131 -0
131 lines not on this page · raw message
hw/remote/machine.c +12 -1
@@ -20,6 +20,7 @@#include "qapi/error.h" #include "hw/pci/pci_host.h" #include "hw/remote/iohub.h" +#include "hw/remote/iommu.h" #include "hw/qdev-core.h" static void remote_machine_init (MachineState *machine)
@@ -99,6 +100,16 @@ static void remote_machine_instance_init(Object *obj) s->auto_shutdown = true; } +static void remote_machine_dev_unplug_cb (HotplugHandler *hotplug_dev, +DeviceState *dev,Error **errp) +{ +qdev_unrealize (dev); + +if (object_dynamic_cast (OBJECT (dev),TYPE_PCI_DEVICE )) { +remote_iommu_unplug_dev (PCI_DEVICE (dev)); + } +} +static void remote_machine_class_init (ObjectClass *oc,void *data) {MachineClass *mc =MACHINE_CLASS (oc);
@@ -107,7 +118,7 @@ static void remote_machine_class_init(ObjectClass *oc, void *data) mc->init = remote_machine_init; mc->desc ="Experimental remote machine" ; - hc->unplug = qdev_simple_device_unplug_cb; + hc->unplug = remote_machine_dev_unplug_cb; object_class_property_add_bool(oc, "vfio-user", remote_machine_get_vfio_user,
MAINTAINERS +2 -0
@@ -3633,6 +3633,8 @@ F: hw/remote/iohub.c F: include/hw/remote/iohub.h F: subprojects/libvfio-user F: hw/remote/vfio-user-obj.c +F: hw/remote/iommu.c +F: include/hw/remote/iommu.h EBPF: M: Jason Wang <[email protected]>
hw/remote/meson.build +1 -0
@@ -6,6 +6,7 @@ remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('message.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('remote-obj.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('proxy.c')) remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iohub.c')) +remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('iommu.c')) remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: files('vfio-user-obj.c')) remote_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_true: libvfio_user_dep)
Signature · 1 line
2.20.1
PATCH v11 11/14 vfio-user: handle DMA mappings
In reply to Jagannathan RamanDefine and register callbacks to manage the RAM regions used for device DMA
--- hw/remote/machine.c | 5 ++++ hw/remote/vfio-user-obj.c | 55 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 2 ++ 3 files changed, 62 insertions(+)
hw/remote/machine.c +5 -0
@@ -22,6 +22,7 @@#include "hw/remote/iohub.h" #include "hw/remote/iommu.h" #include "hw/qdev-core.h" +#include "hw/remote/iommu.h" static void remote_machine_init (MachineState *machine) {
@@ -51,6 +52,10 @@ static void remote_machine_init(MachineState *machine) pci_host =PCI_HOST_BRIDGE (rem_host); +if (s->vfio_user) { +remote_iommu_setup (pci_host->bus); + } +remote_iohub_init (&s->iohub); pci_bus_irqs(pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq,
hw/remote/vfio-user-obj.c +55 -0
67 lines not on this page · raw message
hw/remote/trace-events +2 -0
@@ -7,3 +7,5 @@ mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, vfu_prop(const char *prop, const char *val) "vfu: setting %s as %s" vfu_cfg_read(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u -> 0x%x" vfu_cfg_write(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u <- 0x%x" +vfu_dma_register(uint64_t gpa, size_t len) "vfu: registering GPA 0x%"PRIx64", %zu bytes" +vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64""
Signature · 1 line
2.20.1
PATCH v11 12/14 vfio-user: handle PCI BAR accesses
In reply to Jagannathan RamanDetermine the BARs used by the PCI device and register handlers to manage the access to the same.
--- include/exec/memory.h | 3 + hw/remote/vfio-user-obj.c | 190 ++++++++++++++++++++++++++++++++ softmmu/physmem.c | 4 +- tests/qtest/fuzz/generic_fuzz.c | 9 +- hw/remote/trace-events | 3 + 5 files changed, 203 insertions(+), 6 deletions(-)
include/exec/memory.h +3 -0
@@ -2810,6 +2810,9 @@ MemTxResult address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr, const void *buf, hwaddr len); +int memory_access_size (MemoryRegion *mr,unsigned l,hwaddr addr); +bool prepare_mmio_access (MemoryRegion *mr); +static inline bool memory_access_is_direct (MemoryRegion *mr,bool is_write) {if (is_write) {
hw/remote/vfio-user-obj.c +190 -0
208 lines not on this page · raw message
softmmu/physmem.c +2 -2
@@ -2719,7 +2719,7 @@ void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)invalidate_and_set_dirty (mr, addr, size); } -static int memory_access_size (MemoryRegion *mr,unsigned l,hwaddr addr) +int memory_access_size (MemoryRegion *mr,unsigned l,hwaddr addr) {unsigned access_size_max = mr->ops->valid.max_access_size;
@@ -2746,7 +2746,7 @@ static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)return l; } -static bool prepare_mmio_access (MemoryRegion *mr) +bool prepare_mmio_access (MemoryRegion *mr) {bool release_lock = false;
tests/qtest/fuzz/generic_fuzz.c +5 -4
@@ -144,7 +144,7 @@ static void *pattern_alloc(pattern p, size_t len)return buf; } -static int memory_access_size (MemoryRegion *mr,unsigned l,hwaddr addr) +static int fuzz_memory_access_size (MemoryRegion *mr,unsigned l,hwaddr addr) {unsigned access_size_max = mr->ops->valid.max_access_size;
@@ -242,11 +242,12 @@ void fuzz_dma_read_cb(size_t addr, size_t len, MemoryRegion *mr)/* * If mr1 isn't RAM, address_space_translate doesn't update l. Use -* memory_access_size to identify the number of bytes that it is safe -* to write without accidentally writing to another MemoryRegion. +* fuzz_memory_access_size to identify the number of bytes that it +* is safe to write without accidentally writing to another +* MemoryRegion. */ if (!memory_region_is_ram (mr1)) { - l =memory_access_size (mr1, l, addr1); + l =fuzz_memory_access_size (mr1, l, addr1); } if (memory_region_is_ram(mr1) || memory_region_is_romd(mr1) ||
hw/remote/trace-events +3 -0
@@ -9,3 +9,6 @@ vfu_cfg_read(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u -> 0x%x" vfu_cfg_write(uint32_t offset, uint32_t val) "vfu: cfg: 0x%u <- 0x%x" vfu_dma_register(uint64_t gpa, size_t len) "vfu: registering GPA 0x%"PRIx64", %zu bytes" vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64"" +vfu_bar_register(int i, uint64_t addr, uint64_t size) "vfu: BAR %d: addr 0x%"PRIx64" size 0x%"PRIx64"" +vfu_bar_rw_enter(const char *op, uint64_t addr) "vfu: %s request for BAR address 0x%"PRIx64"" +vfu_bar_rw_exit(const char *op, uint64_t addr) "vfu: Finished %s of BAR address 0x%"PRIx64""
Signature · 1 line
2.20.1
PATCH v11 13/14 vfio-user: handle device interrupts
In reply to Jagannathan RamanForward remote device's interrupts to the guest
--- include/hw/pci/msi.h | 1 + include/hw/pci/msix.h | 1 + include/hw/pci/pci.h | 13 +++ include/hw/remote/vfio-user-obj.h | 6 ++ hw/pci/msi.c | 48 +++++++-- hw/pci/msix.c | 35 ++++++- hw/pci/pci.c | 13 +++ hw/remote/machine.c | 14 ++- hw/remote/vfio-user-obj.c | 167 ++++++++++++++++++++++++++++++ stubs/vfio-user-obj.c | 6 ++ MAINTAINERS | 1 + hw/remote/trace-events | 1 + stubs/meson.build | 1 + 13 files changed, 296 insertions(+), 11 deletions(-) create mode 100644 include/hw/remote/vfio-user-obj.h create mode 100644 stubs/vfio-user-obj.c
include/hw/pci/msi.h +1 -0
@@ -43,6 +43,7 @@ void msi_notify(PCIDevice *dev, unsigned int vector);void msi_send_message (PCIDevice *dev,MSIMessage msg);void msi_write_config (PCIDevice *dev,uint32_t addr,uint32_t val,int len);unsigned int msi_nr_vectors_allocated (const PCIDevice *dev); +void msi_set_irq_state (PCIDevice *dev,int vector,bool mask,Error **errp);static inline bool msi_present (const PCIDevice *dev) {
include/hw/pci/msix.h +1 -0
@@ -36,6 +36,7 @@ void msix_clr_pending(PCIDevice *dev, int vector);int msix_vector_use (PCIDevice *dev,unsigned vector);void msix_vector_unuse (PCIDevice *dev,unsigned vector);void msix_unuse_all_vectors (PCIDevice *dev); +void msix_set_irq_state (PCIDevice *dev,int vector,bool mask,Error **errp);void msix_notify (PCIDevice *dev,unsigned vector);
include/hw/pci/pci.h +13 -0
@@ -16,6 +16,7 @@ extern bool pci_available;#define PCI_SLOT (devfn) (((devfn) >> 3) & 0x1f)#define PCI_FUNC (devfn) ((devfn) & 0x07)#define PCI_BUILD_BDF (bus, devfn) ((bus << 8) | (devfn)) +#define PCI_BDF_TO_DEVFN (x) ((x) & 0xff)#define PCI_BUS_MAX 256#define PCI_DEVFN_MAX 256#define PCI_SLOT_MAX 32
@@ -127,6 +128,10 @@ typedef void PCIMapIORegionFunc(PCIDevice *pci_dev, int region_num, pcibus_t addr, pcibus_t size, int type);typedef void PCIUnregisterFunc (PCIDevice *pci_dev); +typedef void MSITriggerFunc (PCIDevice *dev,MSIMessage msg); +typedef MSIMessage MSIPrepareMessageFunc (PCIDevice *dev,unsigned vector); +typedef MSIMessage MSIxPrepareMessageFunc (PCIDevice *dev,unsigned vector); + typedef struct PCIIORegion { pcibus_t addr; /* current PCI mapping address. -1 means not mapped */ #define PCI_BAR_UNMAPPED (~(pcibus_t)0)
@@ -329,6 +334,14 @@ struct PCIDevice {/* Space to store MSIX table & pending bit array */ uint8_t *msix_table;uint8_t *msix_pba; + +/* May be used by INTx or MSI during interrupt notification */ +void *irq_opaque; + +MSITriggerFunc *msi_trigger; +MSIPrepareMessageFunc *msi_prepare_message; +MSIxPrepareMessageFunc *msix_prepare_message; +/* MemoryRegion container for msix exclusive BAR setup */ MemoryRegion msix_exclusive_bar;/* Memory Regions for MSIX table and pending bit entries. */
include/hw/remote/vfio-user-obj.h new file +6 -0
@@ -0,0 +1,6 @@ +#ifndef VFIO_USER_OBJ_H +#define VFIO_USER_OBJ_H + +void vfu_object_set_bus_irq (PCIBus *pci_bus); + +#endif
hw/pci/msi.c +42 -6
84 lines not on this page · raw message
hw/pci/msix.c +34 -1
@@ -31,7 +31,7 @@#define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)#define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8) -MSIMessage msix_get_message (PCIDevice *dev,unsigned vector) +static MSIMessage msix_prepare_message (PCIDevice *dev,unsigned vector) {uint8_t *table_entry = dev->msix_table + vector *PCI_MSIX_ENTRY_SIZE ;MSIMessage msg;
@@ -41,6 +41,11 @@ MSIMessage msix_get_message(PCIDevice *dev, unsigned vector)return msg; } +MSIMessage msix_get_message (PCIDevice *dev,unsigned vector) +{ +return dev->msix_prepare_message (dev, vector); +} +/* * Special API for POWER to configure the vectors through * a side channel. Should never be used by devices.
@@ -131,6 +136,31 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) } } +void msix_set_irq_state (PCIDevice *dev,int vector,bool mask,Error **errp) +{ +ERRP_GUARD (); +unsigned offset; +bool was_masked; + +if (vector > dev->msix_entries_nr) { +error_setg (errp,"msix: vector %d not allocated. max vector is %d" , + vector, dev->msix_entries_nr); +return ; + } + + offset = vector *PCI_MSIX_ENTRY_SIZE +PCI_MSIX_ENTRY_VECTOR_CTRL ; + + was_masked =msix_is_masked (dev, vector); + +if (mask) { + dev->msix_table[offset] |=PCI_MSIX_ENTRY_CTRL_MASKBIT ; + }else { + dev->msix_table[offset] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT ; + } + +msix_handle_mask_update (dev, vector, was_masked); +} +static bool msix_masked (PCIDevice *dev) {return dev->config[dev->msix_cap +MSIX_CONTROL_OFFSET ] &MSIX_MASKALL_MASK ;
@@ -344,6 +374,8 @@ int msix_init(struct PCIDevice *dev, unsigned short nentries, "msix-pba", pba_size);memory_region_add_subregion (pba_bar, pba_offset, &dev->msix_pba_mmio); + dev->msix_prepare_message = msix_prepare_message; +return 0 ; }
@@ -429,6 +461,7 @@ void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)g_free (dev->msix_entry_used); dev->msix_entry_used =NULL ; dev->cap_present &= ~QEMU_PCI_CAP_MSIX ; + dev->msix_prepare_message =NULL ; }void msix_uninit_exclusive_bar (PCIDevice *dev)
hw/pci/pci.c +13 -0
@@ -317,6 +317,15 @@ void pci_device_deassert_intx(PCIDevice *dev) } } +static void pci_msi_trigger (PCIDevice *dev,MSIMessage msg) +{ +MemTxAttrs attrs = {}; + + attrs.requester_id =pci_requester_id (dev); +address_space_stl_le (&dev->bus_master_as, msg.address, msg.data, + attrs,NULL ); +} +static void pci_reset_regions (PCIDevice *dev) {int r;
@@ -1212,6 +1221,8 @@ static void pci_qdev_unrealize(DeviceState *dev)pci_device_deassert_intx (pci_dev);do_pci_unregister_device (pci_dev); + + pci_dev->msi_trigger =NULL ; } void pci_register_bar(PCIDevice *pci_dev, int region_num,
@@ -2251,6 +2262,8 @@ static void pci_qdev_realize(DeviceState *qdev, Error **errp) }pci_set_power (pci_dev, true); + + pci_dev->msi_trigger = pci_msi_trigger; } PCIDevice *pci_new_multifunction(int devfn, bool multifunction,
hw/remote/machine.c +10 -4
@@ -23,6 +23,8 @@#include "hw/remote/iommu.h" #include "hw/qdev-core.h" #include "hw/remote/iommu.h" +#include "hw/remote/vfio-user-obj.h" +#include "hw/pci/msi.h" static void remote_machine_init (MachineState *machine) {
@@ -54,12 +56,16 @@ static void remote_machine_init(MachineState *machine)if (s->vfio_user) {remote_iommu_setup (pci_host->bus); - } -remote_iohub_init (&s->iohub); + msi_nonbroken = true; + +vfu_object_set_bus_irq (pci_host->bus); + }else { +remote_iohub_init (&s->iohub); -pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, - &s->iohub,REMOTE_IOHUB_NB_PIRQS ); +pci_bus_irqs (pci_host->bus, remote_iohub_set_irq, remote_iohub_map_irq, + &s->iohub,REMOTE_IOHUB_NB_PIRQS ); + }qbus_set_hotplug_handler (BUS (pci_host->bus),OBJECT (s)); }
hw/remote/vfio-user-obj.c +167 -0
203 lines not on this page · raw message
stubs/vfio-user-obj.c new file +6 -0
@@ -0,0 +1,6 @@ +#include "qemu/osdep.h" +#include "hw/remote/vfio-user-obj.h" + +void vfu_object_set_bus_irq (PCIBus *pci_bus) +{ +}
MAINTAINERS +1 -0
@@ -3633,6 +3633,7 @@ F: hw/remote/iohub.c F: include/hw/remote/iohub.h F: subprojects/libvfio-user F: hw/remote/vfio-user-obj.c +F: include/hw/remote/vfio-user-obj.h F: hw/remote/iommu.c F: include/hw/remote/iommu.h
hw/remote/trace-events +1 -0
@@ -12,3 +12,4 @@ vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64"" vfu_bar_register(int i, uint64_t addr, uint64_t size) "vfu: BAR %d: addr 0x%"PRIx64" size 0x%"PRIx64"" vfu_bar_rw_enter(const char *op, uint64_t addr) "vfu: %s request for BAR address 0x%"PRIx64"" vfu_bar_rw_exit(const char *op, uint64_t addr) "vfu: Finished %s of BAR address 0x%"PRIx64"" +vfu_interrupt(int pirq) "vfu: sending interrupt to device - PIRQ %d"
stubs/meson.build +1 -0
@@ -60,3 +60,4 @@ if have_system else stub_ss.add(files('qdev.c')) endif +stub_ss.add(when: 'CONFIG_VFIO_USER_SERVER', if_false: files('vfio-user-obj.c'))
Signature · 1 line
2.20.1
Re: [PATCH v11 13/14] vfio-user: handle device interrupts
hw/pci/msi.c bool msi_is_masked(const PCIDevice *dev, unsigned int vector) · line 318
Quoted 5 lines · Jagannathan Raman
hw/pci/msi.c +1
-0@@ -307,6 +315,38 @@ bool msi_is_masked(const PCIDevice *dev, unsigned int vector)return mask & (1U << vector); } +void msi_set_irq_state (PCIDevice *dev,int vector,bool mask,Error **errp)
The function name is non-obvious. This function masks or unmasks an MSI vector. Maybe call it msi_set_mask()?
hw/pci/msi.c bool msi_is_masked(const PCIDevice *dev, unsigned int vector) · line 328
Quoted 10 lines
hw/pci/msi.c +10
-0+{ +ERRP_GUARD (); +uint16_t flags =pci_get_word (dev->config +msi_flags_off (dev)); +bool msi64bit = flags &PCI_MSI_FLAGS_64BIT ; +uint32_t irq_state, vector_mask, pending; + +if (vector >PCI_MSI_VECTORS_MAX ) { +error_setg (errp,"msi: vector %d not allocated. max vector is %d" , + vector,PCI_MSI_VECTORS_MAX ); + }
Missing return statement?
hw/pci/msix.c static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) · line 139
Quoted 5 lines
hw/pci/msix.c +1
-0@@ -131,6 +136,31 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) } } +void msix_set_irq_state (PCIDevice *dev,int vector,bool mask,Error **errp)
Same naming question here.
PATCH v11 14/14 vfio-user: handle reset of remote device
In reply to Jagannathan RamanAdds handler to reset a remote device
--- hw/remote/vfio-user-obj.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+)
hw/remote/vfio-user-obj.c +20 -0
@@ -676,6 +676,20 @@ void vfu_object_set_bus_irq(PCIBus *pci_bus) max_bdf); } +static int vfu_object_device_reset (vfu_ctx_t *vfu_ctx,vfu_reset_type_t type) +{ +VfuObject *o =vfu_get_private (vfu_ctx); + +/* vfu_object_ctx_run() handles lost connection */ +if (type ==VFU_RESET_LOST_CONN ) { +return 0 ; + } + +qdev_reset_all (DEVICE (o->pci_dev)); + +return 0 ; +} +/* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These
@@ -795,6 +809,12 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + ret =vfu_setup_device_reset_cb (o->vfu_ctx, &vfu_object_device_reset); +if (ret <0 ) { +error_setg (errp,"vfu: Failed to setup reset callback" ); + goto fail; + } + ret =vfu_realize_ctx (o->vfu_ctx);if (ret <0 ) { error_setg(errp, "vfu: Failed to realize device %s- %s",
Signature · 1 line
2.20.1