add generic vDPA device support
v4 cover add generic vDPA device support
From: Longpeng <[email protected]>
Hi guys,
With the generic vDPA device, QEMU won't need to touch the device types any more, such like vfio.
We can use the generic vDPA device as follow:
-device vhost-vdpa-device-pci,vhostdev=/dev/vhost-vdpa-X
I've done some simple tests on Huawei's offloading card (net, 0.95).
Changes v3 -> v4:
v3: https://www.mail-archive.com/[email protected]/msg877015.html
- reorganize the series [Stefano]
- fix some typos [Stefano]
- fix logical error in vhost_vdpa_device_realize [Stefano]
Changes v2 -> v3
Patch 4 & 5:
- only call vdpa ioctls in vdpa-dev.c [Stefano, Longpeng]
- s/VQS_NUM/VQS_COUNT [Stefano]
- check both vdpa_dev_fd and vdpa_dev [Stefano]
Patch 6:
- move all steps into vhost_vdpa_device_unrealize. [Stefano]
Changes RFC -> v2
Patch 1:
- rename 'pdev_id' to 'trans_devid' [Michael]
- only use transitional device id for the devices listed in the spec [Michael]
- use macros to make the id_info table clearer [Longpeng]
- add some modern devices in the id_info table [Longpeng]
Patch 2:
- remove the GET_VECTORS_NUM command [Jason]
Patch 4:
- expose vdpa_dev_fd as a QOM preperty [Stefan]
- introduce vhost_vdpa_device_get_u32 as a common function to make the code clearer [Stefan]
- fix the misleading description of 'dc->desc' [Stefano]
Patch 5:
- check returned number of virtqueues [Stefan]
Patch 6:
- init s->num_queues [Stefano]
- free s->dev.vqs [Stefano]
Longpeng (Mike) (4):
linux-headers: Update headers to Linux 5.18-rc6
virtio: get class_id and pci device id by the virtio id
vdpa: add vdpa-dev support
vdpa: add vdpa-dev-pci support
hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 2 + hw/virtio/vdpa-dev-pci.c | 101 +++++++++ hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ hw/virtio/virtio-pci.c | 77 +++++++ hw/virtio/virtio-pci.h | 5 + include/hw/virtio/vdpa-dev.h | 43 ++++ linux-headers/linux/vhost.h | 7 + 8 files changed, 625 insertions(+) create mode 100644 hw/virtio/vdpa-dev-pci.c create mode 100644 hw/virtio/vdpa-dev.c create mode 100644 include/hw/virtio/vdpa-dev.h
Signature · 1 line
2.23.0
PATCH v4 1/4 linux-headers: Update headers to Linux 5.18-rc6
From: Longpeng <[email protected]>
Update headers to 5.18-rc6. I need latest vhost changes.
--- linux-headers/linux/vhost.h | 7 +++++++ 1 file changed, 7 insertions(+)
linux-headers/linux/vhost.h +7 -0
@@ -150,4 +150,11 @@/* Get the valid iova range */ #define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \ struct vhost_vdpa_iova_range) + +/* Get the config size */ +#define VHOST_VDPA_GET_CONFIG_SIZE _IOR(VHOST_VIRTIO, 0x79, __u32) + +/* Get the count of all virtqueues */ +#define VHOST_VDPA_GET_VQS_COUNT _IOR(VHOST_VIRTIO, 0x80, __u32) +#endif
Signature · 1 line
2.23.0
PATCH v4 2/4 virtio: get class_id and pci device id by the virtio id
In reply to Longpeng(Mike) viaFrom: Longpeng <[email protected]>
Add helpers to get the "Transitional PCI Device ID" and "class_id" of the device specified by the "Virtio Device ID".
These helpers will be used to build the generic vDPA device later.
--- hw/virtio/virtio-pci.c | 77 ++++++++++++++++++++++++++++++++++++++++++ hw/virtio/virtio-pci.h | 5 +++ 2 files changed, 82 insertions(+)
hw/virtio/virtio-pci.c +77 -0
@@ -19,6 +19,7 @@#include "exec/memop.h" #include "standard-headers/linux/virtio_pci.h" +#include "standard-headers/linux/virtio_ids.h" #include "hw/boards.h" #include "hw/virtio/virtio.h" #include "migration/qemu-file-types.h"
@@ -212,6 +213,79 @@ static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f)return 0 ; } +typedef struct VirtIOPCIIDInfo { +/* virtio id */ +uint16_t vdev_id; +/* pci device id for the transitional device */ +uint16_t trans_devid; +uint16_t class_id; +}VirtIOPCIIDInfo ; + +#define VIRTIO_TRANS_DEV_ID_INFO (name, class) \ + { \ + .vdev_id = VIRTIO_ID_##name, \ + .trans_devid = PCI_DEVICE_ID_VIRTIO_##name, \ + .class_id = class, \ + } + +#define VIRTIO_MODERN_DEV_ID_NFO (name, class) \ + { \ + .vdev_id = VIRTIO_ID_##name, \ + .class_id = class, \ + } + +static const VirtIOPCIIDInfo virtio_pci_id_info[] = { +/* Non-transitional devices */ +VIRTIO_MODERN_DEV_ID_NFO (CRYPTO ,PCI_CLASS_OTHERS ), +VIRTIO_MODERN_DEV_ID_NFO (FS ,PCI_CLASS_STORAGE_OTHER ), +/* Transitional devices */ +VIRTIO_TRANS_DEV_ID_INFO (NET ,PCI_CLASS_NETWORK_ETHERNET ), +VIRTIO_TRANS_DEV_ID_INFO (BLOCK ,PCI_CLASS_STORAGE_SCSI ), +VIRTIO_TRANS_DEV_ID_INFO (CONSOLE ,PCI_CLASS_COMMUNICATION_OTHER ), +VIRTIO_TRANS_DEV_ID_INFO (SCSI ,PCI_CLASS_STORAGE_SCSI ), +VIRTIO_TRANS_DEV_ID_INFO (9P ,PCI_BASE_CLASS_NETWORK ), +VIRTIO_TRANS_DEV_ID_INFO (BALLOON ,PCI_CLASS_OTHERS ), +VIRTIO_TRANS_DEV_ID_INFO (RNG ,PCI_CLASS_OTHERS ), +}; + +static const VirtIOPCIIDInfo *virtio_pci_get_id_info (uint16_t vdev_id) +{ +const VirtIOPCIIDInfo *info =NULL ; +int i; + +for (i =0 ; i <ARRAY_SIZE (virtio_pci_id_info); i++) { +if (virtio_pci_id_info[i].vdev_id == vdev_id) { + info = &virtio_pci_id_info[i]; +break ; + } + } + +if (!info) { +/* The device id is invalid or not added to the id_info yet. */ +error_report ("Invalid virtio device(id %u)" , vdev_id); +abort (); + } + +return info; +} + +/* +* Get the Transitional Device ID for the specific device, return +* zero if the device is non-transitional. +*/ +uint16_t virtio_pci_get_trans_devid (uint16_t device_id) +{ +return virtio_pci_get_id_info (device_id)->trans_devid; +} + +/* +* Get the Class ID for the specific device. +*/ +uint16_t virtio_pci_get_class_id (uint16_t device_id) +{ +return virtio_pci_get_id_info (device_id)->class_id; +} +static bool virtio_pci_ioeventfd_enabled (DeviceState *d) {VirtIOPCIProxy *proxy =to_virtio_pci_proxy (d);
@@ -1675,6 +1749,9 @@ static void virtio_pci_device_plugged(DeviceState *d, Error **errp)* is set to PCI_SUBVENDOR_ID_REDHAT_QUMRANET by default. */ pci_set_word (config +PCI_SUBSYSTEM_ID ,virtio_bus_get_vdev_id (bus)); +if (proxy->trans_devid) { +pci_config_set_device_id (config, proxy->trans_devid); + } }else {/* pure virtio-1.0 */ pci_set_word(config + PCI_VENDOR_ID,
hw/virtio/virtio-pci.h +5 -0
@@ -146,6 +146,8 @@ struct VirtIOPCIProxy {bool disable_modern;bool ignore_backend_features;OnOffAuto disable_legacy; +/* Transitional device id */ +uint16_t trans_devid;uint32_t class_code;uint32_t nvectors;uint32_t dfselect;
@@ -158,6 +160,9 @@ struct VirtIOPCIProxy {VirtioBusState bus; }; +uint16_t virtio_pci_get_trans_devid (uint16_t device_id); +uint16_t virtio_pci_get_class_id (uint16_t device_id); +static inline bool virtio_pci_modern (VirtIOPCIProxy *proxy) {return !proxy->disable_modern;
Signature · 1 line
2.23.0
PATCH v4 3/4 vdpa: add vdpa-dev support
In reply to Longpeng(Mike) viaFrom: Longpeng <[email protected]>
Supports vdpa-dev.
--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+) create mode 100644 hw/virtio/vdpa-dev.c create mode 100644 include/hw/virtio/vdpa-dev.h
hw/virtio/Kconfig +5 -0
@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCI + depends on VIRTIO && VHOST_VDPA && LINUX
hw/virtio/meson.build +1 -0
@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))
hw/virtio/vdpa-dev.c new file +385 -0
385 lines not on this page · raw message
include/hw/virtio/vdpa-dev.h new file +43 -0
43 lines not on this page · raw message
Signature · 1 line
2.23.0
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
Quoted 20 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+) create mode 100644 hw/virtio/vdpa-dev.c create mode 100644 include/hw/virtio/vdpa-dev.hhw/virtio/Kconfig +0
-0
hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 7 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCI
Do we have the plan to add VIRTIO_MMIO support?
Quoted 80 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +61
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ;
hw/virtio/vdpa-dev.c line 71
Quoted 10 lines
hw/virtio/vdpa-dev.c new file +10
-0+ } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ; + } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {
Is this better to set error messages for all the possible failures during realization?
Quoted 250 lines
hw/virtio/vdpa-dev.c new file +250
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) {
hw/virtio/vdpa-dev.c line 331
Quoted 10 lines
hw/virtio/vdpa-dev.c new file +10
-0+error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev); + } +} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),
This is probably not needed since we can "abuse" /dev/fd/X for vhostdev.
Thanks
Quoted 106 lines
hw/virtio/vdpa-dev.c new file +54
-0+DEFINE_PROP_UINT16 ("queue-size" , VhostVdpaDevice, queue_size,0 ), + DEFINE_PROP_END_OF_LIST(), +}; + +static const VMStateDescription vmstate_vhost_vdpa_device = { + .name ="vhost-vdpa-device" , + .unmigratable =1 , + .minimum_version_id =1 , + .version_id =1 , + .fields = (VMStateField []) { +VMSTATE_VIRTIO_DEVICE , +VMSTATE_END_OF_LIST () + }, +}; + +static void vhost_vdpa_device_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +VirtioDeviceClass *vdc =VIRTIO_DEVICE_CLASS (klass); + +device_class_set_props (dc, vhost_vdpa_device_properties); + dc->desc ="VDPA-based generic device assignment" ; + dc->vmsd = &vmstate_vhost_vdpa_device; +set_bit (DEVICE_CATEGORY_MISC , dc->categories); + vdc->realize = vhost_vdpa_device_realize; + vdc->unrealize = vhost_vdpa_device_unrealize; + vdc->get_config = vhost_vdpa_device_get_config; + vdc->set_config = vhost_vdpa_device_set_config; + vdc->get_features = vhost_vdpa_device_get_features; + vdc->set_status = vhost_vdpa_device_set_status; +} + +static void vhost_vdpa_device_instance_init (Object *obj) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (obj); + +device_add_bootindex_property (obj, &s->bootindex,"bootindex" , +NULL ,DEVICE (obj)); +} + +static const TypeInfo vhost_vdpa_device_info = { + .name =TYPE_VHOST_VDPA_DEVICE , + .parent =TYPE_VIRTIO_DEVICE , + .instance_size =sizeof (VhostVdpaDevice), + .class_init = vhost_vdpa_device_class_init, + .instance_init = vhost_vdpa_device_instance_init, +}; + +static void register_vhost_vdpa_device_type (void ) +{ +type_register_static (&vhost_vdpa_device_info); +} + +type_init (register_vhost_vdpa_device_type);include/hw/virtio/vdpa-dev.h new file +43
-1@@ -0,0 +1,43 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk.h" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#ifndef _VHOST_VDPA_DEVICE_H +#define _VHOST_VDPA_DEVICE_H + +#include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-vdpa.h" +#include "qom/object.h" + + +#define TYPE_VHOST_VDPA_DEVICE "vhost-vdpa-device" +OBJECT_DECLARE_SIMPLE_TYPE(VhostVdpaDevice, VHOST_VDPA_DEVICE) + +struct VhostVdpaDevice { +VirtIODevice parent_obj; +char *vhostdev; +int vhostfd; +int32_t bootindex; +uint32_t vdev_id; +uint32_t num_queues; +struct vhost_dev dev; +struct vhost_vdpa vdpa; +VirtQueue **virtqs; +uint8_t *config; +int config_size; +uint16_t queue_size; +bool started; +int (*post_init)(VhostVdpaDevice *v,Error **errp); +}; + +#endif --2.23.0
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
Quoted 16 lines · Jason Wang
Quoted 15 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+)create mode 100644
hw/virtio/vdpa-dev.c
create mode 100644include/hw/virtio/vdpa-dev.h
Quoted 15 lines
hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 12 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCIDo we have the plan to add
VIRTIO_MMIOsupport?
There is currently no plan. Maybe we can try to support it in the next step.
Quoted 85 lines
Quoted 85 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +66
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ; + } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ;
Quoted 9 lines
hw/virtio/vdpa-dev.c line 71
Quoted 5 lines
hw/virtio/vdpa-dev.c new file +5
-0+ } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {Is this better to set error messages for all the possible failures during realization?
I think the error messages already set in *errp, right?
Quoted 254 lines
Quoted 254 lines
hw/virtio/vdpa-dev.c new file +254
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) { +error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev);
Quoted 9 lines
hw/virtio/vdpa-dev.c line 331
Quoted 6 lines
hw/virtio/vdpa-dev.c new file +6
-0+ } +} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),This is probably not needed since we can "abuse"
/dev/fd/Xfor vhostdev.
It seems "vhostdev" is enough. I prefer to remove "vhostfd" if there is no objection.
Thanks.
Quoted 110 lines
Thanks
Quoted 104 lines
hw/virtio/vdpa-dev.c new file +54
-0+DEFINE_PROP_UINT16 ("queue-size" , VhostVdpaDevice, queue_size,0 ), + DEFINE_PROP_END_OF_LIST(), +}; + +static const VMStateDescription vmstate_vhost_vdpa_device = { + .name ="vhost-vdpa-device" , + .unmigratable =1 , + .minimum_version_id =1 , + .version_id =1 , + .fields = (VMStateField []) { +VMSTATE_VIRTIO_DEVICE , +VMSTATE_END_OF_LIST () + }, +}; + +static void vhost_vdpa_device_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +VirtioDeviceClass *vdc =VIRTIO_DEVICE_CLASS (klass); + +device_class_set_props (dc, vhost_vdpa_device_properties); + dc->desc ="VDPA-based generic device assignment" ; + dc->vmsd = &vmstate_vhost_vdpa_device; +set_bit (DEVICE_CATEGORY_MISC , dc->categories); + vdc->realize = vhost_vdpa_device_realize; + vdc->unrealize = vhost_vdpa_device_unrealize; + vdc->get_config = vhost_vdpa_device_get_config; + vdc->set_config = vhost_vdpa_device_set_config; + vdc->get_features = vhost_vdpa_device_get_features; + vdc->set_status = vhost_vdpa_device_set_status; +} + +static void vhost_vdpa_device_instance_init (Object *obj) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (obj); + +device_add_bootindex_property (obj, &s->bootindex,"bootindex" , +NULL ,DEVICE (obj)); +} + +static const TypeInfo vhost_vdpa_device_info = { + .name =TYPE_VHOST_VDPA_DEVICE , + .parent =TYPE_VIRTIO_DEVICE , + .instance_size =sizeof (VhostVdpaDevice), + .class_init = vhost_vdpa_device_class_init, + .instance_init = vhost_vdpa_device_instance_init, +}; + +static void register_vhost_vdpa_device_type (void ) +{ +type_register_static (&vhost_vdpa_device_info); +} + +type_init (register_vhost_vdpa_device_type);include/hw/virtio/vdpa-dev.h new file +43
-1@@ -0,0 +1,43 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk.h" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#ifndef _VHOST_VDPA_DEVICE_H +#define _VHOST_VDPA_DEVICE_H + +#include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-vdpa.h" +#include "qom/object.h" + + +#define TYPE_VHOST_VDPA_DEVICE "vhost-vdpa-device" +OBJECT_DECLARE_SIMPLE_TYPE(VhostVdpaDevice, VHOST_VDPA_DEVICE) + +struct VhostVdpaDevice { +VirtIODevice parent_obj; +char *vhostdev; +int vhostfd; +int32_t bootindex; +uint32_t vdev_id; +uint32_t num_queues; +struct vhost_dev dev; +struct vhost_vdpa vdpa; +VirtQueue **virtqs; +uint8_t *config; +int config_size; +uint16_t queue_size; +bool started; +int (*post_init)(VhostVdpaDevice *v,Error **errp); +}; + +#endif --Quoted 2 lines
2.23.0
.
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
Quoted 35 lines · Cloud Infrastructure Service Product Dept.)
Quoted 16 lines · Jason Wang
Quoted 15 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+)create mode 100644
hw/virtio/vdpa-dev.c
create mode 100644include/hw/virtio/vdpa-dev.hQuoted 15 lines
hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 12 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCIDo we have the plan to add
VIRTIO_MMIOsupport?
Quoted 1 line
There is currently no plan. Maybe we can try to support it in the next step.
That would be better, allowing MMIO on top of vhost-vDPA is considered to be one of the important advantages.
Quoted 95 lines
Quoted 86 lines
Quoted 86 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +66
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ; + } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ;Quoted 9 lines
hw/virtio/vdpa-dev.c line 71
Quoted 5 lines
hw/virtio/vdpa-dev.c new file +5
-0+ } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {Is this better to set error messages for all the possible failures during realization?
Quoted 1 line
I think the error messages already set in *errp, right?
Oh right.
Quoted 264 lines
Quoted 255 lines
Quoted 255 lines
hw/virtio/vdpa-dev.c new file +254
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) { +error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev);Quoted 9 lines
hw/virtio/vdpa-dev.c line 331
Quoted 6 lines
hw/virtio/vdpa-dev.c new file +6
-0+ } +} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),This is probably not needed since we can "abuse"
/dev/fd/Xfor vhostdev.
Quoted 2 lines
It seems "vhostdev" is enough. I prefer to remove "vhostfd" if there is no objection.
Yes, it is the way the current vhost-vDPA networking backend did.
Thanks
Quoted 114 lines
Thanks.
Quoted 110 lines
Thanks
Quoted 104 lines
hw/virtio/vdpa-dev.c new file +54
-0+DEFINE_PROP_UINT16 ("queue-size" , VhostVdpaDevice, queue_size,0 ), + DEFINE_PROP_END_OF_LIST(), +}; + +static const VMStateDescription vmstate_vhost_vdpa_device = { + .name ="vhost-vdpa-device" , + .unmigratable =1 , + .minimum_version_id =1 , + .version_id =1 , + .fields = (VMStateField []) { +VMSTATE_VIRTIO_DEVICE , +VMSTATE_END_OF_LIST () + }, +}; + +static void vhost_vdpa_device_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +VirtioDeviceClass *vdc =VIRTIO_DEVICE_CLASS (klass); + +device_class_set_props (dc, vhost_vdpa_device_properties); + dc->desc ="VDPA-based generic device assignment" ; + dc->vmsd = &vmstate_vhost_vdpa_device; +set_bit (DEVICE_CATEGORY_MISC , dc->categories); + vdc->realize = vhost_vdpa_device_realize; + vdc->unrealize = vhost_vdpa_device_unrealize; + vdc->get_config = vhost_vdpa_device_get_config; + vdc->set_config = vhost_vdpa_device_set_config; + vdc->get_features = vhost_vdpa_device_get_features; + vdc->set_status = vhost_vdpa_device_set_status; +} + +static void vhost_vdpa_device_instance_init (Object *obj) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (obj); + +device_add_bootindex_property (obj, &s->bootindex,"bootindex" , +NULL ,DEVICE (obj)); +} + +static const TypeInfo vhost_vdpa_device_info = { + .name =TYPE_VHOST_VDPA_DEVICE , + .parent =TYPE_VIRTIO_DEVICE , + .instance_size =sizeof (VhostVdpaDevice), + .class_init = vhost_vdpa_device_class_init, + .instance_init = vhost_vdpa_device_instance_init, +}; + +static void register_vhost_vdpa_device_type (void ) +{ +type_register_static (&vhost_vdpa_device_info); +} + +type_init (register_vhost_vdpa_device_type);include/hw/virtio/vdpa-dev.h new file +43
-1@@ -0,0 +1,43 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk.h" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#ifndef _VHOST_VDPA_DEVICE_H +#define _VHOST_VDPA_DEVICE_H + +#include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-vdpa.h" +#include "qom/object.h" + + +#define TYPE_VHOST_VDPA_DEVICE "vhost-vdpa-device" +OBJECT_DECLARE_SIMPLE_TYPE(VhostVdpaDevice, VHOST_VDPA_DEVICE) + +struct VhostVdpaDevice { +VirtIODevice parent_obj; +char *vhostdev; +int vhostfd; +int32_t bootindex; +uint32_t vdev_id; +uint32_t num_queues; +struct vhost_dev dev; +struct vhost_vdpa vdpa; +VirtQueue **virtqs; +uint8_t *config; +int config_size; +uint16_t queue_size; +bool started; +int (*post_init)(VhostVdpaDevice *v,Error **errp); +}; + +#endif --Quoted 2 lines
2.23.0
.
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
Quoted 37 lines · Jason Wang
Quoted 35 lines · Cloud Infrastructure Service Product Dept.)
Quoted 31 lines · Jason Wang
Quoted 20 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+)create mode 100644
hw/virtio/vdpa-dev.ccreate mode 100644include/hw/virtio/vdpa-dev.hhw/virtio/Kconfig +0
-0hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 7 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCIDo we have the plan to add
VIRTIO_MMIOsupport?
Quoted 5 lines
Quoted 1 line
There is currently no plan. Maybe we can try to support it in the next step.
That would be better, allowing MMIO on top of vhost-vDPA is considered to be one of the important advantages.
OK, I've done some simple tests, and it works fine. I'll send v5 later.
Quoted 486 lines
Quoted 95 lines
Quoted 86 lines
Quoted 86 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +66
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ; + } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ;Quoted 9 lines
hw/virtio/vdpa-dev.c line 71
Quoted 5 lines
hw/virtio/vdpa-dev.c new file +5
-0+ } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {Is this better to set error messages for all the possible failures during realization?
Quoted 1 line
I think the error messages already set in *errp, right?
Oh right.
Quoted 264 lines
Quoted 255 lines
Quoted 255 lines
hw/virtio/vdpa-dev.c new file +254
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) { +error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev);Quoted 9 lines
hw/virtio/vdpa-dev.c line 331
Quoted 6 lines
hw/virtio/vdpa-dev.c new file +6
-0+ } +} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),This is probably not needed since we can "abuse"
/dev/fd/Xfor vhostdev.Quoted 2 lines
It seems "vhostdev" is enough. I prefer to remove "vhostfd" if there is no objection.
Yes, it is the way the current vhost-vDPA networking backend did.
Thanks
Quoted 114 lines
Thanks.
Quoted 110 lines
Thanks
Quoted 104 lines
hw/virtio/vdpa-dev.c new file +54
-0+DEFINE_PROP_UINT16 ("queue-size" , VhostVdpaDevice, queue_size,0 ), + DEFINE_PROP_END_OF_LIST(), +}; + +static const VMStateDescription vmstate_vhost_vdpa_device = { + .name ="vhost-vdpa-device" , + .unmigratable =1 , + .minimum_version_id =1 , + .version_id =1 , + .fields = (VMStateField []) { +VMSTATE_VIRTIO_DEVICE , +VMSTATE_END_OF_LIST () + }, +}; + +static void vhost_vdpa_device_class_init (ObjectClass *klass,void *data) +{ +DeviceClass *dc =DEVICE_CLASS (klass); +VirtioDeviceClass *vdc =VIRTIO_DEVICE_CLASS (klass); + +device_class_set_props (dc, vhost_vdpa_device_properties); + dc->desc ="VDPA-based generic device assignment" ; + dc->vmsd = &vmstate_vhost_vdpa_device; +set_bit (DEVICE_CATEGORY_MISC , dc->categories); + vdc->realize = vhost_vdpa_device_realize; + vdc->unrealize = vhost_vdpa_device_unrealize; + vdc->get_config = vhost_vdpa_device_get_config; + vdc->set_config = vhost_vdpa_device_set_config; + vdc->get_features = vhost_vdpa_device_get_features; + vdc->set_status = vhost_vdpa_device_set_status; +} + +static void vhost_vdpa_device_instance_init (Object *obj) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (obj); + +device_add_bootindex_property (obj, &s->bootindex,"bootindex" , +NULL ,DEVICE (obj)); +} + +static const TypeInfo vhost_vdpa_device_info = { + .name =TYPE_VHOST_VDPA_DEVICE , + .parent =TYPE_VIRTIO_DEVICE , + .instance_size =sizeof (VhostVdpaDevice), + .class_init = vhost_vdpa_device_class_init, + .instance_init = vhost_vdpa_device_instance_init, +}; + +static void register_vhost_vdpa_device_type (void ) +{ +type_register_static (&vhost_vdpa_device_info); +} + +type_init (register_vhost_vdpa_device_type);include/hw/virtio/vdpa-dev.h new file +43
-1@@ -0,0 +1,43 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk.h" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#ifndef _VHOST_VDPA_DEVICE_H +#define _VHOST_VDPA_DEVICE_H + +#include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-vdpa.h" +#include "qom/object.h" + + +#define TYPE_VHOST_VDPA_DEVICE "vhost-vdpa-device" +OBJECT_DECLARE_SIMPLE_TYPE(VhostVdpaDevice, VHOST_VDPA_DEVICE) + +struct VhostVdpaDevice { +VirtIODevice parent_obj; +char *vhostdev; +int vhostfd; +int32_t bootindex; +uint32_t vdev_id; +uint32_t num_queues; +struct vhost_dev dev; +struct vhost_vdpa vdpa; +VirtQueue **virtqs; +uint8_t *config; +int config_size; +uint16_t queue_size; +bool started; +int (*post_init)(VhostVdpaDevice *v,Error **errp); +}; + +#endif --Quoted 2 lines
2.23.0
.
.
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
In reply to Jason WangQuoted 379 lines · Jason Wang
Quoted 20 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+) create mode 100644 hw/virtio/vdpa-dev.c create mode 100644 include/hw/virtio/vdpa-dev.hhw/virtio/Kconfig +0
-0hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 7 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCIDo we have the plan to add
VIRTIO_MMIOsupport?Quoted 80 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +61
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ;hw/virtio/vdpa-dev.c line 71
Quoted 10 lines
hw/virtio/vdpa-dev.c new file +10
-0+ } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ; + } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {Is this better to set error messages for all the possible failures during realization?
Quoted 254 lines
hw/virtio/vdpa-dev.c new file +254
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) { +error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev);
Quoted 8 lines
hw/virtio/vdpa-dev.c line 331
Quoted 6 lines
hw/virtio/vdpa-dev.c new file +6
-0+ } +} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),This is probably not needed since we can "abuse"
/dev/fd/Xfor vhostdev.
IIRC for other vhost devices (e.g. vhost-vsock) the management layer (e.g. libvirt) opens the device (because I think QEMU has less permissions) and then passes the fd to QEMU, could this be useful for this scenario as well?
Thanks,
Stefano
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
Quoted 391 lines · Stefano Garzarella
Quoted 380 lines · Jason Wang
Quoted 20 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+) create mode 100644 hw/virtio/vdpa-dev.c create mode 100644 include/hw/virtio/vdpa-dev.hhw/virtio/Kconfig +0
-0hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 7 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCIDo we have the plan to add
VIRTIO_MMIOsupport?Quoted 80 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +61
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ;hw/virtio/vdpa-dev.c line 71
Quoted 10 lines
hw/virtio/vdpa-dev.c new file +10
-0+ } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ; + } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {Is this better to set error messages for all the possible failures during realization?
Quoted 255 lines
hw/virtio/vdpa-dev.c new file +255
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) { +error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev); + }Quoted 8 lines
hw/virtio/vdpa-dev.c line 331
Quoted 5 lines
hw/virtio/vdpa-dev.c new file +5
-0+} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),This is probably not needed since we can "abuse"
/dev/fd/Xfor vhostdev.
Quoted 4 lines
IIRC for other vhost devices (e.g. vhost-vsock) the management layer (e.g. libvirt) opens the device (because I think QEMU has less permissions) and then passes the fd to QEMU, could this be useful for this scenario as well?
Yes, it's the way current vhost-vDPA did (the fd is passing via SCM_RIGHTS).
Thanks
Quoted 4 lines
Thanks,
Stefano
Re: [PATCH v4 3/4] vdpa: add vdpa-dev support
Quoted 392 lines · Jason Wang
Quoted 391 lines · Stefano Garzarella
Quoted 388 lines · Jason Wang
Quoted 20 lines · Longpeng(Mike)
From: Longpeng <[email protected]>
Supports vdpa-dev.
Signed-off-by Longpeng <[email protected]>--- hw/virtio/Kconfig | 5 + hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev.c | 385 +++++++++++++++++++++++++++++++++++ include/hw/virtio/vdpa-dev.h | 43 ++++ 4 files changed, 434 insertions(+) create mode 100644 hw/virtio/vdpa-dev.c create mode 100644 include/hw/virtio/vdpa-dev.hhw/virtio/Kconfig +0
-0hw/virtio/Kconfig config VHOST_USER_RNG · line 74
Quoted 7 lines
hw/virtio/Kconfig +4
-0@@ -68,3 +68,8 @@ config VHOST_USER_RNG bool default y depends on VIRTIO && VHOST_USER + +config VHOST_VDPA_DEV + bool + default y if VIRTIO_PCIDo we have the plan to add
VIRTIO_MMIOsupport?Quoted 80 lines
hw/virtio/Kconfig +1
-0+ depends on VIRTIO && VHOST_VDPA && LINUXhw/virtio/meson.build +1
-0@@ -29,6 +29,7 @@ virtio_ss.add(when: 'CONFIG_VHOST_USER_I2C', if_true: files('vhost-user-i2c.c')) virtio_ss.add(when: ['CONFIG_VIRTIO_PCI', 'CONFIG_VHOST_USER_I2C'], if_true: files('vhost-user-i2c-pci.c')) virtio_ss.add(when: 'CONFIG_VHOST_USER_RNG', if_true: files('vhost-user-rng.c')) virtio_ss.add(when: ['CONFIG_VHOST_USER_RNG', 'CONFIG_VIRTIO_PCI'], if_true: files('vhost-user-rng-pci.c')) +virtio_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev.c')) virtio_pci_ss = ss.source_set() virtio_pci_ss.add(when: 'CONFIG_VHOST_VSOCK', if_true: files('vhost-vsock-pci.c'))hw/virtio/vdpa-dev.c new file +61
-0@@ -0,0 +1,385 @@ +/* +* Vhost Vdpa Device +* +* Copyright (c) Huawei Technologies Co., Ltd. 2022. All Rights Reserved. +* +* Authors: +* Longpeng <[email protected]> +* +* Largely based on the "vhost-user-blk-pci.c" and "vhost-user-blk.c" implemented by: +* Changpeng Liu <[email protected]> +* +* This work is licensed under the terms of the GNU LGPL, version 2 or later. +* See the COPYING.LIB file in the top-level directory. +*/ +#include "qemu/osdep.h" +#include <sys/ioctl.h> +#include <linux/vhost.h> +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/cutils.h" +#include "hw/qdev-core.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "hw/virtio/vhost.h" +#include "hw/virtio/virtio.h" +#include "hw/virtio/virtio-bus.h" +#include "hw/virtio/virtio-access.h" +#include "hw/virtio/vdpa-dev.h" +#include "sysemu/sysemu.h" +#include "sysemu/runstate.h" + +static void +vhost_vdpa_device_dummy_handle_output (VirtIODevice *vdev,VirtQueue *vq) +{ +/* Nothing to do */ +} + +static uint32_t +vhost_vdpa_device_get_u32 (int fd,unsigned long int cmd,Error **errp) +{ +uint32_t val = (uint32_t )-1 ; + +if (ioctl (fd, cmd, &val) <0 ) { +error_setg (errp,"vhost-vdpa-device: cmd 0x%lx failed: %s" , + cmd,strerror (errno)); + } + +return val; +} + +static void vhost_vdpa_device_realize (DeviceState *dev,Error **errp) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *v =VHOST_VDPA_DEVICE (vdev); +uint16_t max_queue_size; +struct vhost_virtqueue *vqs; +int i, ret; + +if (!v->vhostdev && v->vhostfd ==-1 ) { +error_setg (errp,"both vhostdev and vhostfd are missing" ); +return ;hw/virtio/vdpa-dev.c line 71
Quoted 10 lines
hw/virtio/vdpa-dev.c new file +10
-0+ } + +if (v->vhostdev && v->vhostfd !=-1 ) { +error_setg (errp,"both vhostdev and vhostfd are set" ); +return ; + } + +if (v->vhostfd ==-1 ) { + v->vhostfd =qemu_open (v->vhostdev,O_RDWR , errp); +if (*errp) {Is this better to set error messages for all the possible failures during realization?
Quoted 250 lines
hw/virtio/vdpa-dev.c new file +250
-0+return ; + } + } + v->vdpa.device_fd = v->vhostfd; + + v->vdev_id =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_DEVICE_ID , errp); +if (*errp) { + goto out; + } + + max_queue_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VRING_NUM , errp); +if (*errp) { + goto out; + } + +if (v->queue_size > max_queue_size) { +error_setg (errp,"vhost-vdpa-device: invalid queue_size: %u (max:%u)" , + v->queue_size, max_queue_size); + goto out; + }else if (!v->queue_size) { + v->queue_size = max_queue_size; + } + + v->num_queues =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_VQS_COUNT , errp); +if (*errp) { + goto out; + } + +if (!v->num_queues || v->num_queues >VIRTIO_QUEUE_MAX ) { +error_setg (errp,"invalid number of virtqueues: %u (max:%u)" , + v->num_queues,VIRTIO_QUEUE_MAX ); + goto out; + } + + v->dev.nvqs = v->num_queues; + vqs = g_new0(struct vhost_virtqueue, v->dev.nvqs); + v->dev.vqs = vqs; + v->dev.vq_index =0 ; + v->dev.vq_index_end = v->dev.nvqs; + v->dev.backend_features =0 ; + v->started = false; + + ret =vhost_dev_init (&v->dev, &v->vdpa,VHOST_BACKEND_TYPE_VDPA ,0 ,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: vhost initialization failed: %s" , +strerror (-ret)); + goto free_vqs; + } + + v->config_size =vhost_vdpa_device_get_u32 (v->vhostfd, +VHOST_VDPA_GET_CONFIG_SIZE , errp); +if (*errp) { + goto vhost_cleanup; + } + +if (v->post_init && v->post_init (v, errp) <0 ) { + goto free_virtio; + } + + v->config =g_malloc0 (v->config_size); + + ret =vhost_dev_get_config (&v->dev, v->config, v->config_size,NULL ); +if (ret <0 ) { +error_setg (errp,"vhost-vdpa-device: get config failed" ); + goto free_config; + } + +virtio_init (vdev,"vhost-vdpa" , v->vdev_id, v->config_size); + + v->virtqs = g_new0(VirtQueue *, v->dev.nvqs); +for (i =0 ; i < v->dev.nvqs; i++) { + v->virtqs[i] =virtio_add_queue (vdev, v->queue_size, + vhost_vdpa_device_dummy_handle_output); + } + +return ; + +free_virtio: +for (i =0 ; i < v->num_queues; i++) { +virtio_delete_queue (v->virtqs[i]); + } +g_free (v->virtqs); +virtio_cleanup (vdev); +free_config: +g_free (v->config); +vhost_cleanup: +vhost_dev_cleanup (&v->dev); +free_vqs: +g_free (vqs); +out: +qemu_close (v->vhostfd); + v->vhostfd =-1 ; +} + +static void vhost_vdpa_device_unrealize (DeviceState *dev) +{ +VirtIODevice *vdev =VIRTIO_DEVICE (dev); +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int i; + +virtio_set_status (vdev,0 ); + +for (i =0 ; i < s->num_queues; i++) { +virtio_delete_queue (s->virtqs[i]); + } +g_free (s->virtqs); +virtio_cleanup (vdev); + +g_free (s->config); +g_free (s->dev.vqs); +vhost_dev_cleanup (&s->dev); +qemu_close (s->vhostfd); + s->vhostfd =-1 ; +} + +static void +vhost_vdpa_device_get_config (VirtIODevice *vdev,uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); + +memcpy (config, s->config, s->config_size); +} + +static void +vhost_vdpa_device_set_config (VirtIODevice *vdev,const uint8_t *config) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +int ret; + + ret =vhost_dev_set_config (&s->dev, s->config,0 , s->config_size, +VHOST_SET_CONFIG_TYPE_MASTER ); +if (ret) { +error_report ("set device config space failed" ); +return ; + } +} + +static uint64_t vhost_vdpa_device_get_features (VirtIODevice *vdev, +uint64_t features, +Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +uint64_t backend_features = s->dev.features; + +if (!virtio_has_feature (features,VIRTIO_F_IOMMU_PLATFORM )) { +virtio_clear_feature (&backend_features,VIRTIO_F_IOMMU_PLATFORM ); + } + +return backend_features; +} + +static int vhost_vdpa_device_start (VirtIODevice *vdev,Error **errp) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int i, ret; + +if (!k->set_guest_notifiers) { +error_setg (errp,"binding does not support guest notifiers" ); +return -ENOSYS ; + } + + ret =vhost_dev_enable_notifiers (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error enabling host notifiers" ); +return ret; + } + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, true); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error binding guest notifier" ); + goto err_host_notifiers; + } + + s->dev.acked_features = vdev->guest_features; + + ret =vhost_dev_start (&s->dev, vdev); +if (ret <0 ) { +error_setg_errno (errp, -ret,"Error starting vhost" ); + goto err_guest_notifiers; + } + s->started = true; + +/* +* guest_notifier_mask/pending not used yet, so just unmask +* everything here. virtio-pci will do the right thing by +* enabling/disabling irqfd. +*/ +for (i =0 ; i < s->dev.nvqs; i++) { +vhost_virtqueue_mask (&s->dev, vdev, i, false); + } + +return ret; + +err_guest_notifiers: + k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +err_host_notifiers: +vhost_dev_disable_notifiers (&s->dev, vdev); +return ret; +} + +static void vhost_vdpa_device_stop (VirtIODevice *vdev) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +BusState *qbus =BUS (qdev_get_parent_bus (DEVICE (vdev))); +VirtioBusClass *k =VIRTIO_BUS_GET_CLASS (qbus); +int ret; + +if (!s->started) { +return ; + } + s->started = false; + +if (!k->set_guest_notifiers) { +return ; + } + +vhost_dev_stop (&s->dev, vdev); + + ret = k->set_guest_notifiers (qbus->parent, s->dev.nvqs, false); +if (ret <0 ) { +error_report ("vhost guest notifier cleanup failed: %d" , ret); +return ; + } + +vhost_dev_disable_notifiers (&s->dev, vdev); +} + +static void vhost_vdpa_device_set_status (VirtIODevice *vdev,uint8_t status) +{ +VhostVdpaDevice *s =VHOST_VDPA_DEVICE (vdev); +bool should_start =virtio_device_started (vdev, status); +Error *local_err =NULL ; +int ret; + +if (!vdev->vm_running) { + should_start = false; + } + +if (s->started == should_start) { +return ; + } + +if (should_start) { + ret =vhost_vdpa_device_start (vdev, &local_err); +if (ret <0 ) {hw/virtio/vdpa-dev.c line 331
Quoted 10 lines
hw/virtio/vdpa-dev.c new file +10
-0+error_reportf_err (local_err,"vhost-vdpa-device: start failed: " ); + } + }else { +vhost_vdpa_device_stop (vdev); + } +} + +static Property vhost_vdpa_device_properties[] = { +DEFINE_PROP_STRING ("vhostdev" , VhostVdpaDevice, vhostdev), +DEFINE_PROP_INT32 ("vhostfd" , VhostVdpaDevice, vhostfd,-1 ),This is probably not needed since we can "abuse"
/dev/fd/Xfor vhostdev.
Quoted 6 lines
Quoted 4 lines
IIRC for other vhost devices (e.g. vhost-vsock) the management layer (e.g. libvirt) opens the device (because I think QEMU has less permissions) and then passes the fd to QEMU, could this be useful for this scenario as well?
Yes, it's the way current vhost-vDPA did (the fd is passing via
SCM_RIGHTS).
Okay, I looked at qemu_open() and I see what you mean. Basically libvirt can pass the fd directly with vhostdev=/dev/fdset/X instead of having a dedicated property.
I did not know that, I agree to remove "vhostfd" then!
Thanks for the clarification,
Stefano
PATCH v4 4/4 vdpa: add vdpa-dev-pci support
In reply to Longpeng(Mike) viaFrom: Longpeng <[email protected]>
Supports vdpa-dev-pci, we can use the device as follow:
-device vhost-vdpa-device-pci,vhostdev=/dev/vhost-vdpa-X
--- hw/virtio/meson.build | 1 + hw/virtio/vdpa-dev-pci.c | 101 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 hw/virtio/vdpa-dev-pci.c
hw/virtio/meson.build +1 -0
@@ -50,6 +50,7 @@ virtio_pci_ss.add(when: 'CONFIG_VIRTIO_SERIAL', if_true: files('virtio-serial-pc virtio_pci_ss.add(when: 'CONFIG_VIRTIO_PMEM', if_true: files('virtio-pmem-pci.c')) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_IOMMU', if_true: files('virtio-iommu-pci.c')) virtio_pci_ss.add(when: 'CONFIG_VIRTIO_MEM', if_true: files('virtio-mem-pci.c')) +virtio_pci_ss.add(when: 'CONFIG_VHOST_VDPA_DEV', if_true: files('vdpa-dev-pci.c')) virtio_ss.add_all(when: 'CONFIG_VIRTIO_PCI', if_true: virtio_pci_ss)
hw/virtio/vdpa-dev-pci.c new file +101 -0
101 lines not on this page · raw message
Signature · 1 line
2.23.0