Skip to contentSkip to conversation

vDPA support

PATCH Superseded Jason Wang 4 participants 36 messages Feb 10, 2020 – Feb 19, 2020

Subsystems (5)

v2 of 9 Superseded Latest is v9 →

lore
Share & tools
lore

Jason Wang

author
#0 lore raw

v2 cover vDPA support

Hi all:

This is an updated version of kernel support for vDPA device. Various changes were made based on the feedback since last verion. One major change is to drop the sysfs API and leave the management interface for future development, and introudce the incremental DMA bus operations. Please see changelog for more information.

The work on vhost, IFCVF (intel VF driver for vDPA) and qemu is ongoing.

Please provide feedback.

Thanks

Change from V1:

- drop sysfs API, leave the management interface to future development (Michael)
- introduce incremental DMA ops (dma_map/dma_unmap) (Michael)
- introduce dma_device and use it instead of parent device for doing IOMMU or DMA from bus driver (Michael, Jason, Ling Shan, Tiwei)
- accept parent device and dma device when register vdpa device
- coding style and compile fixes (Randy)
- using vdpa_xxx instead of xxx_vdpa (Jason)
- ove vDPA accessors to header and make it static inline (Jason)
- split vdp_register_device() into two helpers vdpa_init_device() and vdpa_register_device() which allows intermediate step to be done (Jason)
- warn on invalidate queue state when fail to creating virtqueue (Jason)
- make to_virtio_vdpa_device() static (Jason)
- use kmalloc/kfree instead of devres for virtio vdpa device (Jason)
- avoid using cast in vdpa bus function (Jason)
- introduce module_vdpa_driver and fix module refcnt (Jason)
- fix returning freed address in vdapsim coherent DMA addr allocation (Dan)
- various other fixes and tweaks

V1: https://lkml.org/lkml/2020/1/16/353

Jason Wang (5):
vhost: factor out IOTLB
vringh: IOTLB support
vDPA: introduce vDPA bus
virtio: introduce a vDPA based transport
vdpasim: vDPA device simulator

 MAINTAINERS                    |   2 +
 drivers/vhost/Kconfig          |   7 +
 drivers/vhost/Kconfig.vringh   |   1 +
 drivers/vhost/Makefile         |   2 +
 drivers/vhost/net.c            |   2 +-
 drivers/vhost/vhost.c          | 221 ++++-------
 drivers/vhost/vhost.h          |  36 +-
 drivers/vhost/vhost_iotlb.c    | 171 +++++++++
 drivers/vhost/vringh.c         | 421 ++++++++++++++++++--
 drivers/virtio/Kconfig         |  15 +
 drivers/virtio/Makefile        |   2 +
 drivers/virtio/vdpa/Kconfig    |  26 ++
 drivers/virtio/vdpa/Makefile   |   3 +
 drivers/virtio/vdpa/vdpa.c     | 160 ++++++++
 drivers/virtio/vdpa/vdpa_sim.c | 678 +++++++++++++++++++++++++++++++++
 drivers/virtio/virtio_vdpa.c   | 392 +++++++++++++++++++
 include/linux/vdpa.h           | 233 +++++++++++
 include/linux/vhost_iotlb.h    |  45 +++
 include/linux/vringh.h         |  36 ++
 19 files changed, 2249 insertions(+), 204 deletions(-)
 create mode 100644 drivers/vhost/vhost_iotlb.c
 create mode 100644 drivers/virtio/vdpa/Kconfig
 create mode 100644 drivers/virtio/vdpa/Makefile
 create mode 100644 drivers/virtio/vdpa/vdpa.c
 create mode 100644 drivers/virtio/vdpa/vdpa_sim.c
 create mode 100644 drivers/virtio/virtio_vdpa.c
 create mode 100644 include/linux/vdpa.h
 create mode 100644 include/linux/vhost_iotlb.h
Signature · 1 line
2.19.1
author
#1 lore raw

PATCH v2 1/5 vhost: factor out IOTLB

This patch factors out IOTLB into a dedicated module in order to be reused by other modules like vringh. User may choose to enable the automatic retiring by specifying VHOST_IOTLB_FLAG_RETIRE flag to fit for the case of vhost device IOTLB implementation.

---
 MAINTAINERS                 |   1 +
 drivers/vhost/Kconfig       |   7 ++
 drivers/vhost/Makefile      |   2 +
 drivers/vhost/net.c         |   2 +-
 drivers/vhost/vhost.c       | 221 +++++++++++-------------------------
 drivers/vhost/vhost.h       |  36 ++----
 drivers/vhost/vhost_iotlb.c | 171 ++++++++++++++++++++++++++++
 include/linux/vhost_iotlb.h |  45 ++++++++
 8 files changed, 304 insertions(+), 181 deletions(-)
 create mode 100644 drivers/vhost/vhost_iotlb.c
 create mode 100644 include/linux/vhost_iotlb.h
MAINTAINERS +1 -0
index 2549f10eb0b1..d4bda9c900fa 100644
@@ -17607,6 +17607,7 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git
 S:	Maintained
 F:	drivers/vhost/
 F:	include/uapi/linux/vhost.h
+F:	include/linux/vhost_iotlb.h
 
 VIRTIO INPUT DRIVER
 M:	Gerd Hoffmann <[email protected]>
drivers/vhost/Kconfig +7 -0
index 3d03ccbd1adc..eef634ff9a6e 100644
@@ -36,6 +36,7 @@ config VHOST_VSOCK
 
 config VHOST
 	tristate
+	select VHOST_IOTLB
 	---help---
 	  This option is selected by any driver which needs to access
 	  the core of vhost.
@@ -54,3 +55,9 @@ config VHOST_CROSS_ENDIAN_LEGACY
 	  adds some overhead, it is disabled by default.
 
 	  If unsure, say "N".
+
+config VHOST_IOTLB
+	tristate
+	default m
+	help
+	  Generic IOTLB implementation for vhost and vringh.
drivers/vhost/Makefile +2 -0
index 6c6df24f770c..df99756fbb26 100644
@@ -11,3 +11,5 @@ vhost_vsock-y := vsock.o
 obj-$(CONFIG_VHOST_RING) += vringh.o
 
 obj-$(CONFIG_VHOST)	+= vhost.o
+
+obj-$(CONFIG_VHOST_IOTLB) += vhost_iotlb.o
drivers/vhost/net.c +1 -1
index e158159671fa..e4a20d7a2921 100644
@@ -1594,7 +1594,7 @@ static long vhost_net_reset_owner(struct vhost_net *n)
 	struct socket *tx_sock = NULL;
 	struct socket *rx_sock = NULL;
 	long err;
-	struct vhost_umem *umem;
+	struct vhost_iotlb *umem;
 
 	mutex_lock(&n->dev.mutex);
 	err = vhost_dev_check_owner(&n->dev);
drivers/vhost/vhost.c +69 -152

432 lines not on this page · raw message

drivers/vhost/vhost.h +8 -28
index a123fd70847e..b99c6ffb6be1 100644
@@ -12,6 +12,7 @@
 #include <linux/virtio_config.h>
 #include <linux/virtio_ring.h>
 #include <linux/atomic.h>
+#include <linux/vhost_iotlb.h>
 
 struct vhost_work;
 typedef void (*vhost_work_fn_t)(struct vhost_work *work);
@@ -52,27 +53,6 @@ struct vhost_log {
 	u64 len;
 };
 
-#define START(node) ((node)->start)
-#define LAST(node) ((node)->last)
-
-struct vhost_umem_node {
-	struct rb_node rb;
-	struct list_head link;
-	__u64 start;
-	__u64 last;
-	__u64 size;
-	__u64 userspace_addr;
-	__u32 perm;
-	__u32 flags_padding;
-	__u64 __subtree_last;
-};
-
-struct vhost_umem {
-	struct rb_root_cached umem_tree;
-	struct list_head umem_list;
-	int numem;
-};
-
 enum vhost_uaddr_type {
 	VHOST_ADDR_DESC = 0,
 	VHOST_ADDR_AVAIL = 1,
@@ -90,7 +70,7 @@ struct vhost_virtqueue {
 	struct vring_desc __user *desc;
 	struct vring_avail __user *avail;
 	struct vring_used __user *used;
-	const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS];
+	const struct vhost_iotlb_map *meta_iotlb[VHOST_NUM_ADDRS];
 	struct file *kick;
 	struct eventfd_ctx *call_ctx;
 	struct eventfd_ctx *error_ctx;
@@ -128,8 +108,8 @@ struct vhost_virtqueue {
 	struct iovec *indirect;
 	struct vring_used_elem *heads;
 	/* Protected by virtqueue mutex. */
-	struct vhost_umem *umem;
-	struct vhost_umem *iotlb;
+	struct vhost_iotlb *umem;
+	struct vhost_iotlb *iotlb;
 	void *private_data;
 	u64 acked_features;
 	u64 acked_backend_features;
@@ -164,8 +144,8 @@ struct vhost_dev {
 	struct eventfd_ctx *log_ctx;
 	struct llist_head work_list;
 	struct task_struct *worker;
-	struct vhost_umem *umem;
-	struct vhost_umem *iotlb;
+	struct vhost_iotlb *umem;
+	struct vhost_iotlb *iotlb;
 	spinlock_t iotlb_lock;
 	struct list_head read_list;
 	struct list_head pending_list;
@@ -182,8 +162,8 @@ void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs,
 long vhost_dev_set_owner(struct vhost_dev *dev);
 bool vhost_dev_has_owner(struct vhost_dev *dev);
 long vhost_dev_check_owner(struct vhost_dev *);
-struct vhost_umem *vhost_dev_reset_owner_prepare(void);
-void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_umem *);
+struct vhost_iotlb *vhost_dev_reset_owner_prepare(void);
+void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_iotlb *iotlb);
 void vhost_dev_cleanup(struct vhost_dev *);
 void vhost_dev_stop(struct vhost_dev *);
 long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp);
drivers/vhost/vhost_iotlb.c new file +171 -0

171 lines not on this page · raw message

include/linux/vhost_iotlb.h new file +45 -0

45 lines not on this page · raw message

Signature · 1 line
2.19.1
author
#2 lore raw

PATCH v2 2/5 vringh: IOTLB support

In reply to Jason Wang

This patch implements the third memory accessor for vringh besides current kernel and userspace accessors. This idea is to allow vringh to do the address translation through an IOTLB which is implemented via vhost_map interval tree. Users should setup and IOVA to PA mapping in this IOTLB.

This allows us to:

- Using vringh to access virtqueues with vIOMMU
- Using vringh to implement software vDPA devices

---
 drivers/vhost/Kconfig.vringh |   1 +
 drivers/vhost/vringh.c       | 421 +++++++++++++++++++++++++++++++++--
 include/linux/vringh.h       |  36 +++
 3 files changed, 435 insertions(+), 23 deletions(-)
drivers/vhost/Kconfig.vringh +1 -0
index c1fe36a9b8d4..a8d4dd0cb06e 100644
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config VHOST_RING
 	tristate
+	select VHOST_IOTLB
 	---help---
 	  This option is selected by any driver which needs to access
 	  the host side of a virtio ring.
drivers/vhost/vringh.c +398 -23

557 lines not on this page · raw message

include/linux/vringh.h +36 -0
index d237087eb257..bd0503ca6f8f 100644
@@ -14,6 +14,8 @@
 #include <linux/virtio_byteorder.h>
 #include <linux/uio.h>
 #include <linux/slab.h>
+#include <linux/dma-direction.h>
+#include <linux/vhost_iotlb.h>
 #include <asm/barrier.h>
 
 /* virtio_ring with information needed for host access. */
@@ -39,6 +41,9 @@ struct vringh {
 	/* The vring (note: it may contain user pointers!) */
 	struct vring vring;
 
+	/* IOTLB for this vring */
+	struct vhost_iotlb *iotlb;
+
 	/* The function to call to notify the guest about added buffers */
 	void (*notify)(struct vringh *);
 };
@@ -248,4 +253,35 @@ static inline __virtio64 cpu_to_vringh64(const struct vringh *vrh, u64 val)
 {
 	return __cpu_to_virtio64(vringh_is_little_endian(vrh), val);
 }
+
+void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb);
+
+int vringh_init_iotlb(struct vringh *vrh, u64 features,
+		      unsigned int num, bool weak_barriers,
+		      struct vring_desc *desc,
+		      struct vring_avail *avail,
+		      struct vring_used *used);
+
+int vringh_getdesc_iotlb(struct vringh *vrh,
+			 struct vringh_kiov *riov,
+			 struct vringh_kiov *wiov,
+			 u16 *head,
+			 gfp_t gfp);
+
+ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,
+			      struct vringh_kiov *riov,
+			      void *dst, size_t len);
+ssize_t vringh_iov_push_iotlb(struct vringh *vrh,
+			      struct vringh_kiov *wiov,
+			      const void *src, size_t len);
+
+void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num);
+
+int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len);
+
+bool vringh_notify_enable_iotlb(struct vringh *vrh);
+void vringh_notify_disable_iotlb(struct vringh *vrh);
+
+int vringh_need_notify_iotlb(struct vringh *vrh);
+
 #endif /* _LINUX_VRINGH_H */
Signature · 1 line
2.19.1
author
#3 lore raw

PATCH v2 3/5 vDPA: introduce vDPA bus

In reply to Jason Wang

vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. vDPA hardware devices are usually implemented through PCIE with the following types:

- PF (Physical Function) - A single Physical Function
- VF (Virtual Function) - Device that supports single root I/O virtualization (SR-IOV). Its Virtual Function (VF) represents a virtualized instance of the device that can be assigned to different partitions
- ADI (Assignable Device Interface) and its equivalents - With technologies such as Intel Scalable IOV, a virtual device (VDEV) composed by host OS utilizing one or more ADIs. Or its equivalent like SF (Sub function) from Mellanox.

From a driver's perspective, depends on how and where the DMA translation is done, vDPA devices are split into two types:

- Platform specific DMA translation - From the driver's perspective, the device can be used on a platform where device access to data in memory is limited and/or translated. An example is a PCIE vDPA whose DMA request was tagged via a bus (e.g PCIE) specific way. DMA translation and protection are done at PCIE bus IOMMU level.
- Device specific DMA translation - The device implements DMA isolation and protection through its own logic. An example is a vDPA device which uses on-chip IOMMU.

To hide the differences and complexity of the above types for a vDPA device/IOMMU options and in order to present a generic virtio device to the upper layer, a device agnostic framework is required.

This patch introduces a software vDPA bus which abstracts the common attributes of vDPA device, vDPA bus driver and the communication method (vdpa_config_ops) between the vDPA device abstraction and the vDPA bus driver:

With the abstraction of vDPA bus and vDPA bus operations, the difference and complexity of the under layer hardware is hidden from upper layer. The vDPA bus drivers on top can use a unified vdpa_config_ops to control different types of vDPA device.

---
 MAINTAINERS                  |   1 +
 drivers/virtio/Kconfig       |   2 +
 drivers/virtio/Makefile      |   1 +
 drivers/virtio/vdpa/Kconfig  |   9 ++
 drivers/virtio/vdpa/Makefile |   2 +
 drivers/virtio/vdpa/vdpa.c   | 160 ++++++++++++++++++++++++
 include/linux/vdpa.h         | 233 +++++++++++++++++++++++++++++++++++
 7 files changed, 408 insertions(+)
 create mode 100644 drivers/virtio/vdpa/Kconfig
 create mode 100644 drivers/virtio/vdpa/Makefile
 create mode 100644 drivers/virtio/vdpa/vdpa.c
 create mode 100644 include/linux/vdpa.h
MAINTAINERS +1 -0
index d4bda9c900fa..578d2a581e3b 100644
@@ -17540,6 +17540,7 @@ F:	tools/virtio/
 F:	drivers/net/virtio_net.c
 F:	drivers/block/virtio_blk.c
 F:	include/linux/virtio*.h
+F:	include/linux/vdpa.h
 F:	include/uapi/linux/virtio_*.h
 F:	drivers/crypto/virtio/
 F:	mm/balloon_compaction.c
drivers/virtio/Kconfig +2 -0
index 078615cf2afc..9c4fdb64d9ac 100644
@@ -96,3 +96,5 @@ config VIRTIO_MMIO_CMDLINE_DEVICES
 	 If unsure, say 'N'.
 
 endif # VIRTIO_MENU
+
+source "drivers/virtio/vdpa/Kconfig"
drivers/virtio/Makefile +1 -0
index 3a2b5c5dcf46..fdf5eacd0d0a 100644
@@ -6,3 +6,4 @@ virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o
 virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o
 obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
 obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o
+obj-$(CONFIG_VDPA) += vdpa/
drivers/virtio/vdpa/Kconfig new file +9 -0
new file mode 100644
index 000000000000..7a99170e6c30
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VDPA
+	tristate
+        default m
+        help
+          Enable this module to support vDPA device that uses a
+          datapath which complies with virtio specifications with
+          vendor specific control path.
+
drivers/virtio/vdpa/Makefile new file +2 -0
new file mode 100644
index 000000000000..ee6a35e8a4fb
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VDPA) += vdpa.o
drivers/virtio/vdpa/vdpa.c new file +160 -0

160 lines not on this page · raw message

include/linux/vdpa.h new file +233 -0

233 lines not on this page · raw message

Signature · 1 line
2.19.1
#4 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 5 lines · Jason Wang
drivers/virtio/vdpa/vdpa.c new file +5 -0
+/**
+ * vdpa_register_device - register a vDPA device
+ * Callers must have a succeed call of vdpa_init_device() before.
+ * @vdev: the vdpa device to be registered to vDPA bus
+ *

drivers/virtio/vdpa/vdpa.c line 99

Quoted 10 lines
drivers/virtio/vdpa/vdpa.c new file +10 -0
+ * Returns an error when fail to add to vDPA bus
+ */
+int vdpa_register_device(struct vdpa_device *vdev)
+{
+	int err = device_add(&vdev->dev);
+
+	if (err) {
+		put_device(&vdev->dev);
+		ida_simple_remove(&vdpa_index_ida, vdev->index);
+	}

This is a very dangerous construction, I've seen it lead to driver bugs. Better to require the driver to always do the put_device on error unwind

The ida_simple_remove should probably be part of the class release function to make everything work right

Quoted 6 lines
drivers/virtio/vdpa/vdpa.c new file +6 -0
+/**
+ * vdpa_unregister_driver - unregister a vDPA device driver
+ * @drv: the vdpa device driver to be unregistered
+ */
+void vdpa_unregister_driver(struct vdpa_driver *drv)
+{

drivers/virtio/vdpa/vdpa.c line 149

Quoted 10 lines
drivers/virtio/vdpa/vdpa.c new file +10 -0
+	driver_unregister(&drv->driver);
+}
+EXPORT_SYMBOL_GPL(vdpa_unregister_driver);
+
+static int vdpa_init(void)
+{
+	if (bus_register(&vdpa_bus) != 0)
+		panic("virtio bus registration failed");
+	return 0;
+}

Linus will tell you not to kill the kernel - return the error code and propagate it up to the module init function.

include/linux/vdpa.h line 30

Quoted 12 lines
include/linux/vdpa.h new file +12 -0
+/**
+ * vDPA device - representation of a vDPA device
+ * @dev: underlying device
+ * @dma_dev: the actual device that is performing DMA
+ * @config: the configuration ops for this device.
+ * @index: device index
+ */
+struct vdpa_device {
+	struct device dev;
+	struct device *dma_dev;
+	const struct vdpa_config_ops *config;
+	int index;

unsigned values shuld be unsigned int

Jason

author
#5 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 11 lines · Jason Gunthorpe
Quoted 10 lines · Jason Wang
drivers/virtio/vdpa/vdpa.c new file +10 -0
+/**
+ * vdpa_register_device - register a vDPA device
+ * Callers must have a succeed call of vdpa_init_device() before.
+ * @vdev: the vdpa device to be registered to vDPA bus
+ *
+ * Returns an error when fail to add to vDPA bus
+ */
+int vdpa_register_device(struct vdpa_device *vdev)
+{
+	int err = device_add(&vdev->dev);
Quoted 8 lines

drivers/virtio/vdpa/vdpa.c line 99

Quoted 5 lines
drivers/virtio/vdpa/vdpa.c new file +5 -0
+
+	if (err) {
+		put_device(&vdev->dev);
+		ida_simple_remove(&vdpa_index_ida, vdev->index);
+	}

This is a very dangerous construction, I've seen it lead to driver bugs. Better to require the driver to always do the put_device on error unwind

Ok.

Quoted 3 lines

The ida_simple_remove should probably be part of the class release function to make everything work right

It looks to me bus instead of class is the correct abstraction here since the devices share a set of programming interface but not the semantics.

Or do you actually mean type here?

Quoted 11 lines
Quoted 11 lines
drivers/virtio/vdpa/vdpa.c new file +10 -0
+/**
+ * vdpa_unregister_driver - unregister a vDPA device driver
+ * @drv: the vdpa device driver to be unregistered
+ */
+void vdpa_unregister_driver(struct vdpa_driver *drv)
+{
+	driver_unregister(&drv->driver);
+}
+EXPORT_SYMBOL_GPL(vdpa_unregister_driver);
+
Quoted 8 lines

drivers/virtio/vdpa/vdpa.c line 149

Quoted 6 lines
drivers/virtio/vdpa/vdpa.c new file +6 -0
+static int vdpa_init(void)
+{
+	if (bus_register(&vdpa_bus) != 0)
+		panic("virtio bus registration failed");
+	return 0;
+}

Linus will tell you not to kill the kernel - return the error code and propagate it up to the module init function.

Yes, will fix.

Quoted 15 lines

include/linux/vdpa.h line 30

Quoted 13 lines
include/linux/vdpa.h new file +12 -0
+/**
+ * vDPA device - representation of a vDPA device
+ * @dev: underlying device
+ * @dma_dev: the actual device that is performing DMA
+ * @config: the configuration ops for this device.
+ * @index: device index
+ */
+struct vdpa_device {
+	struct device dev;
+	struct device *dma_dev;
+	const struct vdpa_config_ops *config;
+	int index;

unsigned values shuld be unsigned int

Quoted 1 line

Jason

Will fix.

Thanks

Quoted 1 line
#6 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 5 lines · Jason Wang
Quoted 2 lines

The ida_simple_remove should probably be part of the class release function to make everything work right

It looks to me bus instead of class is the correct abstraction here since the devices share a set of programming interface but not the semantics.

device_release() doesn't call the bus release? You have dev, type or class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Jason

author
#7 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang
Quoted 2 lines

The ida_simple_remove should probably be part of the class release function to make everything work right

It looks to me bus instead of class is the correct abstraction here since the devices share a set of programming interface but not the semantics.

device_release() doesn't call the bus release?

What it did is:

        if (dev->release)
                dev->release(dev);
        else if (dev->type && dev->type->release)
                dev->type->release(dev);
        else if (dev->class && dev->class->dev_release)
                dev->class->dev_release(dev);
        else
                WARN(1, KERN_ERR "Device '%s' does not have a release() function, it is broken and must be fixed. See Documentation/kobject.txt.\n", dev_name(dev));

So it looks not.

Quoted 5 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

Thanks

#8 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 8 lines · Jason Wang
Quoted 5 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

While I suppose there are variations, typically 'class' devices are user facing things and 'bus' devices are internal facing (ie like a PCI device)

So why is this using a bus? VDPA is a user facing object, so the driver should create a class vhost_vdpa device directly, and that driver should live in the drivers/vhost/ directory.

For the PCI VF case this driver would bind to a PCI device like everything else

For our future SF/ADI cases the driver would bind to some SF/ADI/whatever device on a bus.

I don't see a reason for VDPA to be creating busses..

Jason

author
#9 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang
Quoted 4 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

While I suppose there are variations, typically 'class' devices are user facing things and 'bus' devices are internal facing (ie like a PCI device)

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

"

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

"

Quoted 4 lines

So why is this using a bus? VDPA is a user facing object, so the driver should create a class vhost_vdpa device directly, and that driver should live in the drivers/vhost/ directory.

This is because we want vDPA to be generic for being used by different drivers which is not limited to vhost-vdpa. E.g in this series, it allows vDPA to be used by kernel virtio drivers. And in the future, we will probably introduce more drivers in the future.

Quoted 6 lines

For the PCI VF case this driver would bind to a PCI device like everything else

For our future SF/ADI cases the driver would bind to some SF/ADI/whatever device on a bus.

All these driver will still be bound to their own bus (PCI or other). And what the driver needs is to present a vDPA device to virtual vDPA bus on top.

Thanks

Quoted 5 lines

I don't see a reason for VDPA to be creating busses..

Jason

#10 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 20 lines · Jason Wang
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang
Quoted 4 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

While I suppose there are variations, typically 'class' devices are user facing things and 'bus' devices are internal facing (ie like a PCI device)

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

Quoted 8 lines

"

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

"

Here we are talking about the /dev/XX node that provides the programming interface. All the vdpa devices have the same basic chardev interface and discover any semantic variations 'in band'

Quoted 8 lines
Quoted 3 lines

So why is this using a bus? VDPA is a user facing object, so the driver should create a class vhost_vdpa device directly, and that driver should live in the drivers/vhost/ directory.

This is because we want vDPA to be generic for being used by different drivers which is not limited to vhost-vdpa. E.g in this series, it allows vDPA to be used by kernel virtio drivers. And in the future, we will probably introduce more drivers in the future.

I don't see how that connects with using a bus.

Every class of virtio traffic is going to need a special HW driver to enable VDPA, that special driver can create the correct vhost side class device.

Quoted 9 lines
Quoted 5 lines

For the PCI VF case this driver would bind to a PCI device like everything else

For our future SF/ADI cases the driver would bind to some SF/ADI/whatever device on a bus.

All these driver will still be bound to their own bus (PCI or other). And what the driver needs is to present a vDPA device to virtual vDPA bus on top.

Again, I can't see any reason to inject a 'vdpa virtual bus' on top. That seems like mis-using the driver core.

Jason
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#11 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 43 lines · Jason Gunthorpe
Quoted 20 lines · Jason Wang
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang
Quoted 4 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

While I suppose there are variations, typically 'class' devices are user facing things and 'bus' devices are internal facing (ie like a PCI device)

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

Quoted 8 lines

"

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

"

Here we are talking about the /dev/XX node that provides the programming interface. All the vdpa devices have the same basic chardev interface and discover any semantic variations 'in band'

Quoted 8 lines
Quoted 3 lines

So why is this using a bus? VDPA is a user facing object, so the driver should create a class vhost_vdpa device directly, and that driver should live in the drivers/vhost/ directory.

This is because we want vDPA to be generic for being used by different drivers which is not limited to vhost-vdpa. E.g in this series, it allows vDPA to be used by kernel virtio drivers. And in the future, we will probably introduce more drivers in the future.

Quoted 5 lines

I don't see how that connects with using a bus.

Every class of virtio traffic is going to need a special HW driver to enable VDPA, that special driver can create the correct vhost side class device.

That's just a ton of useless code duplication, and a good chance to have minor variations in implementations confusing userspace.

Instead, each device implement the same interface, and then vhost sits on top.

Quoted 6 lines
Quoted 6 lines
Quoted 5 lines

For the PCI VF case this driver would bind to a PCI device like everything else

For our future SF/ADI cases the driver would bind to some SF/ADI/whatever device on a bus.

Quoted 8 lines
Quoted 3 lines

All these driver will still be bound to their own bus (PCI or other). And what the driver needs is to present a vDPA device to virtual vDPA bus on top.

Again, I can't see any reason to inject a 'vdpa virtual bus' on top. That seems like mis-using the driver core.

Jason

That bus is exactly what Greg KH proposed. There are other ways to solve this I guess but this bikeshedding is getting tiring. Come on it's an internal kernel interface, if we feel it was a wrong direction to take we can change our minds later. Main thing is getting UAPI right.

Signature · 1 line
MST
#12 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 46 lines · Michael S. Tsirkin
Quoted 45 lines · Jason Gunthorpe
Quoted 20 lines · Jason Wang
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang
Quoted 4 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

While I suppose there are variations, typically 'class' devices are user facing things and 'bus' devices are internal facing (ie like a PCI device)

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

Quoted 8 lines

"

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

"

Here we are talking about the /dev/XX node that provides the programming interface. All the vdpa devices have the same basic chardev interface and discover any semantic variations 'in band'

Quoted 8 lines
Quoted 3 lines

So why is this using a bus? VDPA is a user facing object, so the driver should create a class vhost_vdpa device directly, and that driver should live in the drivers/vhost/ directory.

This is because we want vDPA to be generic for being used by different drivers which is not limited to vhost-vdpa. E.g in this series, it allows vDPA to be used by kernel virtio drivers. And in the future, we will probably introduce more drivers in the future.

I don't see how that connects with using a bus.

Quoted 7 lines
Quoted 3 lines

Every class of virtio traffic is going to need a special HW driver to enable VDPA, that special driver can create the correct vhost side class device.

That's just a ton of useless code duplication, and a good chance to have minor variations in implementations confusing userspace.

What? Why? This is how almost every user of the driver core works.

I don't see how you get any duplication unless the subsystem core is badly done wrong.

The 'class' is supposed to provide all the library functions to remove this duplication. Instead of plugging the HW driver in via some bus scheme every subsystem has its own 'ops' that the HW driver provides to the subsystem's class via subsystem_register()

This is the *standard* pattern to use the driver core.

This is almost there, it just has this extra bus part to convey the HW ops instead of directly.

Quoted 2 lines

Instead, each device implement the same interface, and then vhost sits on top.

Sure, but plugging in via ops/etc not via a bus and another struct device.

Quoted 2 lines

That bus is exactly what Greg KH proposed. There are other ways to solve this I guess but this bikeshedding is getting tiring.

This discussion was for a different goal, IMHO.

Quoted 3 lines

Come on it's an internal kernel interface, if we feel it was a wrong direction to take we can change our minds later. Main thing is getting UAPI right.

This discussion started because the refcounting has been busted up in every version posted so far. It is not bikeshedding when these bugs are actually being caused by trying to abuse the driver core and shoehorn in a bus that isn't needed when a class is the correct thing to use.

Jason

#13 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 4 lines · Jason Gunthorpe
Quoted 2 lines

That bus is exactly what Greg KH proposed. There are other ways to solve this I guess but this bikeshedding is getting tiring.

This discussion was for a different goal, IMHO.

Hmm couldn't find it anymore. What was the goal there in your opinion?

Signature · 1 line
MST
#14 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 7 lines · Michael S. Tsirkin
Quoted 4 lines · Jason Gunthorpe
Quoted 2 lines

That bus is exactly what Greg KH proposed. There are other ways to solve this I guess but this bikeshedding is getting tiring.

This discussion was for a different goal, IMHO.

Hmm couldn't find it anymore. What was the goal there in your opinion?

I think it was largely talking about how to model things like ADI/SF/etc, plus stuff got very confused when the discussion tried to explain what mdev's role was vs the driver core.

The standard driver model is a 'bus' driver provides the HW access (think PCI level things), and a 'hw driver' attaches to the bus device, and instantiates a 'subsystem device' (think netdev, rdma, etc) using some per-subsystem XXX_register(). The 'hw driver' pulls in functions from the 'subsystem' using a combination of callbacks and library-style calls so there is no code duplication.

As a subsystem, vhost&vdpa should expect its 'HW driver' to bind to devices on busses, for instance I would expect:

- A future SF/ADI/'virtual bus' as a child of multi-functional PCI device Exactly how this works is still under active discussion and is one place where Greg said 'use a bus'.
- An existing PCI, platform, or other bus and device. No need for an extra bus here, PCI is the bus.
- No bus, ie for a simulator or binding to a netdev. (existing vhost?)

They point is that the HW driver's job is to adapt from the bus level interfaces (eg readl/writel) to the subsystem level (eg something like the vdpa_ops).

For instance that Intel driver should be a pci_driver to bind to a struct pci_device for its VF and then call some 'vhost&vdpa' _register() function to pass its ops to the subsystem which in turn creates the struct device of the subsystem calls, common char devices, sysfs, etc and calls the driver's ops in response to uAPI calls.

This is already almost how things were setup in v2 of the patches, near as I can see, just that a bus was inserted somehow instead of having only the vhost class. So it iwas confusing and the lifetime model becomes too complicated to implement correctly...

Jason

author
#15 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 5 lines · Jason Gunthorpe
Quoted 4 lines · Michael S. Tsirkin
Quoted 3 lines · Jason Gunthorpe
Quoted 2 lines

That bus is exactly what Greg KH proposed. There are other ways to solve this I guess but this bikeshedding is getting tiring.

This discussion was for a different goal, IMHO.

Quoted 8 lines
Quoted 1 line

Hmm couldn't find it anymore. What was the goal there in your opinion?

I think it was largely talking about how to model things like ADI/SF/etc, plus stuff got very confused when the discussion tried to explain what mdev's role was vs the driver core.

The standard driver model is a 'bus' driver provides the HW access (think PCI level things), and a 'hw driver' attaches to the bus device,

This is not true, kernel had already had plenty virtual bus where virtual devices and drivers could be attached, besides mdev and virtio, you can see vop, rpmsg, visorbus etc.

Quoted 2 lines

and instantiates a 'subsystem device' (think netdev, rdma, etc) using some per-subsystem XXX_register().

Well, if you go through virtio spec, we support ~20 types of different devices. Classes like netdev and rdma are correct since they have a clear set of semantics their own. But grouping network and scsi into a single class looks wrong, that's the work of a virtual bus.

The class should be done on top of vDPA device instead of vDPA device itself:

- For kernel driver, netdev, blk dev could be done on top
- For userspace driver, the class could be done by the drivers inside VM or userspace (dpdk)

Quoted 3 lines

The 'hw driver' pulls in
functions from the 'subsystem' using a combination of callbacks and library-style calls so there is no code duplication.

The point is we want vDPA devices to be used by different subsystems, not only vhost, but also netdev, blk, crypto (every subsystem that can use virtio devices). That's why we introduce vDPA bus and introduce different drivers on top.

Quoted 7 lines

As a subsystem, vhost&vdpa should expect its 'HW driver' to bind to devices on busses, for instance I would expect:

- A future SF/ADI/'virtual bus' as a child of multi-functional PCI device Exactly how this works is still under active discussion and is one place where Greg said 'use a bus'.

That's ok but it's something that is not directly related to vDPA which can be implemented by any kinds of devices/buses:

struct XXX_device {
struct vdpa_device vdpa;
struct adi_device/pci_device *lowerdev;
}
...

Quoted 2 lines

- An existing PCI, platform, or other bus and device. No need for an extra bus here, PCI is the bus.

There're several examples that a bus is needed on top.

A good example is Mellanox TmFIFO driver which is a platform device driver but register itself as a virtio device in order to be used by virito-console driver on the virtio bus.

But it's a pity that the device can not be used by userspace driver due to the limitation of virito bus which is designed for kernel driver. That's why vDPA bus is introduced which abstract the common requirements of both kernel and userspace drivers which allow the a single HW driver to be used by kernel drivers (and the subsystems on top) and userspace drivers.

Quoted 1 line

- No bus, ie for a simulator or binding to a netdev. (existing vhost?)

Note, simulator can have its own class (sysfs etc.).

Quoted 11 lines

They point is that the HW driver's job is to adapt from the bus level interfaces (eg readl/writel) to the subsystem level (eg something like the vdpa_ops).

For instance that Intel driver should be a pci_driver to bind to a struct pci_device for its VF and then call some 'vhost&vdpa' _register() function to pass its ops to the subsystem which in turn creates the struct device of the subsystem calls, common char devices, sysfs, etc and calls the driver's ops in response to uAPI calls.

Quoted 3 lines

This is already almost how things were setup in v2 of the patches, near as I can see, just that a bus was inserted somehow instead of having only the vhost class.

Well the series (plus mdev part) uses a bus since day 0. It's not something new.

Thanks

Quoted 5 lines

So it iwas confusing and the lifetime
model becomes too complicated to implement correctly...

Jason

#16 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 7 lines · Jason Wang
Quoted 3 lines

The standard driver model is a 'bus' driver provides the HW access (think PCI level things), and a 'hw driver' attaches to the bus device,

This is not true, kernel had already had plenty virtual bus where virtual devices and drivers could be attached, besides mdev and virtio, you can see vop, rpmsg, visorbus etc.

Sure, but those are not connecting HW into the kernel..

Quoted 8 lines
Quoted 2 lines

and instantiates a 'subsystem device' (think netdev, rdma, etc) using some per-subsystem XXX_register().

Well, if you go through virtio spec, we support ~20 types of different devices. Classes like netdev and rdma are correct since they have a clear set of semantics their own. But grouping network and scsi into a single class looks wrong, that's the work of a virtual bus.

rdma also has about 20 different types of things it supports on top of the generic ib_device.

The central point in RDMA is the 'struct ib_device' which is a device class. You can discover all RDMA devices by looking in /sys/class/infiniband/

It has an internal bus like thing (which probably should have been an actual bus, but this was done 15 years ago) which allows other subsystems to have drivers to match and bind their own drivers to the struct ib_device.

So you'd have a chain like:

struct pci_device -> struct ib_device -> [ib client bus thing] -> struct net_device

And the various char devs are created by clients connecting to the ib_device and creating char devs on their own classes.

Since ib_devices are multi-queue we can have all 20 devices running concurrently and there are various schemes to manage when the various things are created.

Quoted 8 lines
Quoted 3 lines

The 'hw driver' pulls in
functions from the 'subsystem' using a combination of callbacks and library-style calls so there is no code duplication.

The point is we want vDPA devices to be used by different subsystems, not only vhost, but also netdev, blk, crypto (every subsystem that can use virtio devices). That's why we introduce vDPA bus and introduce different drivers on top.

See the other mail, it seems struct virtio_device serves this purpose already, confused why a struct vdpa_device and another bus is being introduced

Quoted 5 lines

There're several examples that a bus is needed on top.

A good example is Mellanox TmFIFO driver which is a platform device driver but register itself as a virtio device in order to be used by virito-console driver on the virtio bus.

How is that another bus? The platform bus is the HW bus, the TmFIFO is the HW driver, and virtio_device is the subsystem.

This seems reasonable/normal so far..

Quoted 5 lines

But it's a pity that the device can not be used by userspace driver due to the limitation of virito bus which is designed for kernel driver. That's why vDPA bus is introduced which abstract the common requirements of both kernel and userspace drivers which allow the a single HW driver to be used by kernel drivers (and the subsystems on top) and userspace drivers.

Ah! Maybe this is the source of all this strangeness - the userspace driver is something parallel to the struct virtio_device instead of being a consumer of it?? That certianly would mess up the driver model quite a lot.

Then you want to add another bus to switch between vhost and struct virtio_device? But only for vdpa?

But as you point out something like TmFIFO is left hanging. Seems like the wrong abstraction point..

Jason

author
#17 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 9 lines · Jason Gunthorpe
Quoted 6 lines · Jason Wang
Quoted 3 lines

The standard driver model is a 'bus' driver provides the HW access (think PCI level things), and a 'hw driver' attaches to the bus device,

This is not true, kernel had already had plenty virtual bus where virtual devices and drivers could be attached, besides mdev and virtio, you can see vop, rpmsg, visorbus etc.

Sure, but those are not connecting HW into the kernel..

Well the virtual devices are normally implemented via a real HW driver. E.g for virio bus, its transport driver could be driver of real hardware (e.g PCI).

Quoted 11 lines
Quoted 8 lines
Quoted 3 lines

and instantiates a 'subsystem device' (think netdev, rdma, etc) using some per-subsystem XXX_register().

Well, if you go through virtio spec, we support ~20 types of different devices. Classes like netdev and rdma are correct since they have a clear set of semantics their own. But grouping network and scsi into a single class looks wrong, that's the work of a virtual bus.

rdma also has about 20 different types of things it supports on top of the generic ib_device.

Quoted 7 lines

The central point in RDMA is the 'struct ib_device' which is a device class. You can discover all RDMA devices by looking in /sys/class/infiniband/

It has an internal bus like thing (which probably should have been an actual bus, but this was done 15 years ago) which allows other subsystems to have drivers to match and bind their own drivers to the struct ib_device.

Right.

Quoted 4 lines

So you'd have a chain like:

struct pci_device -> struct ib_device -> [ib client bus thing] -> struct net_device

So for vDPA we want to have:

kernel datapath:

struct pci_device -> struct vDPA device -> [ vDPA bus] -> struct virtio_device -> [virtio bus] -> struct net_device

userspace datapath:

struct pci_device -> struct vDPA device -> [ vDPA bus] -> struct vhost_device -> UAPI -> userspace driver

Quoted 21 lines

And the various char devs are created by clients connecting to the ib_device and creating char devs on their own classes.

Since ib_devices are multi-queue we can have all 20 devices running concurrently and there are various schemes to manage when the various things are created.

Quoted 7 lines
Quoted 3 lines

The 'hw driver' pulls in
functions from the 'subsystem' using a combination of callbacks and library-style calls so there is no code duplication.

The point is we want vDPA devices to be used by different subsystems, not only vhost, but also netdev, blk, crypto (every subsystem that can use virtio devices). That's why we introduce vDPA bus and introduce different drivers on top.

See the other mail, it seems struct virtio_device serves this purpose already, confused why a struct vdpa_device and another bus is being introduced

Quoted 2 lines

There're several examples that a bus is needed on top.

Quoted 7 lines
Quoted 3 lines

A good example is Mellanox TmFIFO driver which is a platform device driver but register itself as a virtio device in order to be used by virito-console driver on the virtio bus.

How is that another bus? The platform bus is the HW bus, the TmFIFO is the HW driver, and virtio_device is the subsystem.

This seems reasonable/normal so far..

Yes, that's reasonable. This example is to answer the question why bus is used instead of class here.

Quoted 9 lines
Quoted 6 lines

But it's a pity that the device can not be used by userspace driver due to the limitation of virito bus which is designed for kernel driver. That's why vDPA bus is introduced which abstract the common requirements of both kernel and userspace drivers which allow the a single HW driver to be used by kernel drivers (and the subsystems on top) and userspace drivers.

Ah! Maybe this is the source of all this strangeness - the userspace driver is something parallel to the struct virtio_device instead of being a consumer of it??

userspace driver is not parallel to virtio_device. The vhost_device is parallel to virtio_device actually.

Quoted 5 lines

That certianly would mess up the driver model
quite a lot.

Then you want to add another bus to switch between vhost and struct virtio_device? But only for vdpa?

Still, vhost works on top of vDPA bus directly (see the reply above).

Quoted 3 lines

But as you point out something like TmFIFO is left hanging. Seems like the wrong abstraction point..

You know, even refactoring virtio-bus is not for free, TmFIFO driver needs changes anyhow.

Thanks

Quoted 3 lines

Jason

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#18 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

In reply to Jason Gunthorpe
Quoted 4 lines · Jason Gunthorpe

The 'class' is supposed to provide all the library functions to remove this duplication. Instead of plugging the HW driver in via some bus scheme every subsystem has its own 'ops' that the HW driver provides to the subsystem's class via subsystem_register()

Hmm I'm not familiar with subsystem_register. A grep didn't find it in the kernel either ...

Signature · 1 line
MST
#19 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 8 lines · Michael S. Tsirkin
Quoted 4 lines · Jason Gunthorpe

The 'class' is supposed to provide all the library functions to remove this duplication. Instead of plugging the HW driver in via some bus scheme every subsystem has its own 'ops' that the HW driver provides to the subsystem's class via subsystem_register()

Hmm I'm not familiar with subsystem_register. A grep didn't find it in the kernel either ...

I mean it is the registration function provided by the subsystem that owns the class, for instance tpm_chip_register(), ib_register_device(), register_netdev(), rtc_register_device() etc

So if you have some vhost (vhost net?) class then you'd have some vhost_vdpa_init/alloc(); vhost_vdpa_register(), sequence presumably. (vs trying to do it with a bus matcher)

I recommend to look at rtc and tpm for fairly simple easy to follow patterns for creating a subsystem in the kernel. A subsystem owns a class, allows HW drivers to plug in to it, and provides a consistent user API via a cdev/sysfs/etc.

The driver model class should revolve around the char dev and sysfs uABI - if you enumerate the devices on the class then they should all follow the char dev and sysfs interfaces contract of that class.

Those examples show how to do all the refcounting semi-sanely, introduce sysfs, cdevs, etc.

I thought the latest proposal was to use the existing vhost class and largely the existing vhost API, so it probably just needs to make sure the common class-wide stuff is split from the 'driver' stuff of the existing vhost to netdev.

Jason

author
#20 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 25 lines · Jason Gunthorpe
Quoted 7 lines · Michael S. Tsirkin
Quoted 4 lines · Jason Gunthorpe

The 'class' is supposed to provide all the library functions to remove this duplication. Instead of plugging the HW driver in via some bus scheme every subsystem has its own 'ops' that the HW driver provides to the subsystem's class via subsystem_register()

Hmm I'm not familiar with subsystem_register. A grep didn't find it in the kernel either ...

I mean it is the registration function provided by the subsystem that owns the class, for instance tpm_chip_register(), ib_register_device(), register_netdev(), rtc_register_device() etc

So if you have some vhost (vhost net?) class then you'd have some vhost_vdpa_init/alloc(); vhost_vdpa_register(), sequence presumably. (vs trying to do it with a bus matcher)

I recommend to look at rtc and tpm for fairly simple easy to follow patterns for creating a subsystem in the kernel. A subsystem owns a class, allows HW drivers to plug in to it, and provides a consistent user API via a cdev/sysfs/etc.

The driver model class should revolve around the char dev and sysfs uABI - if you enumerate the devices on the class then they should all follow the char dev and sysfs interfaces contract of that class.

Quoted 7 lines

Those examples show how to do all the refcounting semi-sanely, introduce sysfs, cdevs, etc.

I thought the latest proposal was to use the existing vhost class and largely the existing vhost API, so it probably just needs to make sure the common class-wide stuff is split from the 'driver' stuff of the existing vhost to netdev.

Still, netdev is only one of the type we want to support. And we can not guarantee or forecast that vhost is the only API that is used.

Let's take virtio as an example, it is implemented through a bus which allows different subsystems on top. And it can provide a variety of different uAPIs. For best performance, VFIO could be used for userspace drivers, but it requires the bus has support from VFIO.

For vDPA devices, it's just the same logic. A bus allows different drivers and subsystems on top. One of the subsystem could be vhost that provides a unified API for userspace driver.

Thanks

Quoted 3 lines

Jason

author
#21 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

In reply to Jason Gunthorpe
Quoted 21 lines · Jason Gunthorpe
Quoted 20 lines · Jason Wang
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang
Quoted 4 lines

You have dev, type or
class to choose from. Type is rarely used and doesn't seem to be used by vdpa, so class seems the right choice

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Jason

Yes, but my understanding is class and bus are mutually exclusive. So we can't add a class to a device which is already attached on a bus.

While I suppose there are variations, typically 'class' devices are user facing things and 'bus' devices are internal facing (ie like a PCI device)

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

"

Quoted 8 lines
Quoted 6 lines

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

"

Here we are talking about the /dev/XX node that provides the programming interface.

I'm confused here, are you suggesting to use class to create char device in vhost-vdpa? That's fine but the comment should go for vhost-vdpa patch.

Quoted 2 lines

All the vdpa devices have the same basic
chardev interface and discover any semantic variations 'in band'

That's not true, char interface is only used for vhost. Kernel virtio driver does not need char dev but a device on the virtio bus.

Quoted 10 lines
Quoted 9 lines
Quoted 4 lines

So why is this using a bus? VDPA is a user facing object, so the driver should create a class vhost_vdpa device directly, and that driver should live in the drivers/vhost/ directory.

This is because we want vDPA to be generic for being used by different drivers which is not limited to vhost-vdpa. E.g in this series, it allows vDPA to be used by kernel virtio drivers. And in the future, we will probably introduce more drivers in the future.

I don't see how that connects with using a bus.

This is demonstrated in the virito-vdpa driver. So if you want to use kernel virito driver for vDPA device, a bus is most straight forward.

Quoted 4 lines

Every class of virtio traffic is going to need a special HW driver to enable VDPA, that special driver can create the correct vhost side class device.

Are you saying, e.g it's the charge of IFCVF driver to create vhost char dev and other stuffs?

Quoted 11 lines
Quoted 9 lines
Quoted 6 lines

For the PCI VF case this driver would bind to a PCI device like everything else

For our future SF/ADI cases the driver would bind to some SF/ADI/whatever device on a bus.

All these driver will still be bound to their own bus (PCI or other). And what the driver needs is to present a vDPA device to virtual vDPA bus on top.

Again, I can't see any reason to inject a 'vdpa virtual bus' on top. That seems like mis-using the driver core.

I don't think so. Vhost is not the only programming interface for vDPA. We don't want a device that can only work for userspace drivers and only have a single set of userspace APIs.

Thanks

Quoted 3 lines

Jason

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#22 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 11 lines · Jason Wang
Quoted 11 lines
Quoted 11 lines

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

"

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

"

Here we are talking about the /dev/XX node that provides the programming interface.

I'm confused here, are you suggesting to use class to create char device in vhost-vdpa? That's fine but the comment should go for vhost-vdpa patch.

Certainly yes, something creating many char devs should have a class. That makes the sysfs work as expected

I suppose this is vhost user? I admit I don't really see how this vhost stuff works, all I see are global misc devices? Very unusual for a new subsystem to be using global misc devices..

I would have expected that a single VDPA device comes out as a single char dev linked to only that VDPA device.

Quoted 5 lines
Quoted 2 lines

All the vdpa devices have the same basic
chardev interface and discover any semantic variations 'in band'

That's not true, char interface is only used for vhost. Kernel virtio driver does not need char dev but a device on the virtio bus.

Okay, this is fine, but why do you need two busses to accomplish this?

Shouldn't the 'struct virito_device' be the plug in point for HW drivers I was talking about - and from there a vhost-user can connect to the struct virtio_device to give it a char dev or a kernel driver can connect to link it to another subsystem?

It is easy to see something is going wrong with this design because the drivers/virtio/virtio_vdpa.c mainly contains a bunch of trampoline functions reflecting identical calls from one ops struct to a different ops struct. This suggests the 'vdpa' is some subclass of 'virtio' and it is possibly better to model it by extending 'struct virito_device' to include the vdpa specific stuff.

Where does the vhost-user char dev get invovled in with the v2 series? Is that included?

Quoted 6 lines
Quoted 3 lines

Every class of virtio traffic is going to need a special HW driver to enable VDPA, that special driver can create the correct vhost side class device.

Are you saying, e.g it's the charge of IFCVF driver to create vhost char dev and other stuffs?

No.

Jason

author
#23 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 17 lines · Jason Gunthorpe
Quoted 15 lines · Jason Wang
Quoted 14 lines
Quoted 4 lines

Though all vDPA devices have the same programming interface, but the semantic is different. So it looks to me that use bus complies what class.rst said:

Quoted 8 lines

"

Each device class defines a set of semantics and a programming interface that devices of that class adhere to. Device drivers are the implementation of that programming interface for a particular device on a particular bus.

"

Here we are talking about the /dev/XX node that provides the programming interface.

Quoted 6 lines
Quoted 2 lines

I'm confused here, are you suggesting to use class to create char device in vhost-vdpa? That's fine but the comment should go for vhost-vdpa patch.

Certainly yes, something creating many char devs should have a class. That makes the sysfs work as expected

I suppose this is vhost user?

Actually not.

Vhost-user is the vhost protocol that is used for userspace vhost backend (usually though a UNIX domain socket).

What's being done in the vhost-vpda is a new type of the vhost in kernel.

Quoted 3 lines

I admit I don't really see how this
vhost stuff works, all I see are global misc devices? Very unusual for a new subsystem to be using global misc devices..

Vhost is not a subsystem right now, e.g for it's net implementation, it was loosely coupled with a socket.

I thought you were copied in the patch [1], maybe we can move vhost related discussion there to avoid confusion.

[1] https://lwn.net/Articles/811210/

Quoted 9 lines

I would have expected that a single VDPA device comes out as a single char dev linked to only that VDPA device.

Quoted 4 lines
Quoted 2 lines

All the vdpa devices have the same basic
chardev interface and discover any semantic variations 'in band'

That's not true, char interface is only used for vhost. Kernel virtio driver does not need char dev but a device on the virtio bus.

Okay, this is fine, but why do you need two busses to accomplish this?

The reasons are:

- vDPA ops is designed to be functional as a software assisted transport (control path) for virtio, so it's fit for a new transport driver but not directly into virtio bus. VOP use similar design.
- virtio bus is designed for kernel drivers but not userspace, and it can not be easily extended to support userspace driver but requires some major refactoring. E.g the virtio bus operations requires the virtqueue to be allocated by the transport driver.

So it's cheaper and simpler to introduce a new bus instead of refactoring a well known bus and API where brunches of drivers and devices had been implemented for years.

Quoted 5 lines

Shouldn't the 'struct virito_device' be the plug in point for HW drivers I was talking about - and from there a vhost-user can connect to the struct virtio_device to give it a char dev or a kernel driver can connect to link it to another subsystem?

From vhost point of view, it would only need to connect vDPA bus, no need to go for virtio bus. Vhost device talks to vDPA device through vDPA bus. Virito device talks to vDPA device through a new vDPA transport driver.

Quoted 5 lines

It is easy to see something is going wrong with this design because the drivers/virtio/virtio_vdpa.c mainly contains a bunch of trampoline functions reflecting identical calls from one ops struct to a different ops struct.

That's pretty normal, since part of the virtio ops could be 1:1 mapped to some device function. If you see MMIO and PCI transport, you can see something similar. The only difference is that in the case of VDPA the function is assisted or emulated by hardware vDPA driver.

Quoted 3 lines

This suggests the 'vdpa' is some subclass of
'virtio' and it is possibly better to model it by extending 'struct virito_device' to include the vdpa specific stuff.

Going for such kind of modeling, virtio-pci and virtio-mmio could be also treated as a subclass of virtio as well, they were all implemented via a dedicated transport driver.

Quoted 3 lines

Where does the vhost-user char dev get invovled in with the v2 series? Is that included?

We're working on the a new version, but for the bus/driver part it should be the same as version 1.

Thanks

Quoted 10 lines
Quoted 6 lines
Quoted 4 lines

Every class of virtio traffic is going to need a special HW driver to enable VDPA, that special driver can create the correct vhost side class device.

Are you saying, e.g it's the charge of IFCVF driver to create vhost char dev and other stuffs?

No.

Jason

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#24 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 4 lines · Jason Wang

I thought you were copied in the patch [1], maybe we can move vhost related discussion there to avoid confusion.

[1] https://lwn.net/Articles/811210/

Wow, that is .. confusing.

So this is supposed to duplicate the uAPI of vhost-user? But it is open coded and duplicated because .. vdpa?

Quoted 3 lines

So it's cheaper and simpler to introduce a new bus instead of refactoring a well known bus and API where brunches of drivers and devices had been implemented for years.

If you reason for this approach is to ease the implementation then you should talk about it in the cover letters/etc

Maybe it is reasonable to do this because the rework is too great, I don't know, but to me this whole thing looks rather messy.

Remember this stuff is all uAPI as it shows up in sysfs, so you can easilly get stuck with it forever.

Jason

#25 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 11 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang

I thought you were copied in the patch [1], maybe we can move vhost related discussion there to avoid confusion.

[1] https://lwn.net/Articles/811210/

Wow, that is .. confusing.

So this is supposed to duplicate the uAPI of vhost-user? But it is open coded and duplicated because .. vdpa?

Do you mean the vhost-user in DPDK? There is no vhost-user in Linux kernel.

Thanks,
Tiwei

Quoted 15 lines
Quoted 4 lines

So it's cheaper and simpler to introduce a new bus instead of refactoring a well known bus and API where brunches of drivers and devices had been implemented for years.

If you reason for this approach is to ease the implementation then you should talk about it in the cover letters/etc

Maybe it is reasonable to do this because the rework is too great, I don't know, but to me this whole thing looks rather messy.

Remember this stuff is all uAPI as it shows up in sysfs, so you can easilly get stuck with it forever.

Jason

author
#26 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

In reply to Jason Gunthorpe
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jason Wang

I thought you were copied in the patch [1], maybe we can move vhost related discussion there to avoid confusion.

[1] https://lwn.net/Articles/811210/

Wow, that is .. confusing.

So this is supposed to duplicate the uAPI of vhost-user?

It tries to reuse the uAPI of vhost with some extension.

Quoted 2 lines

But it is
open coded and duplicated because .. vdpa?

I'm not sure I get here, vhost module is reused for vhost-vdpa and all current vhost device (e.g net) uses their own char device.

Quoted 6 lines
Quoted 4 lines

So it's cheaper and simpler to introduce a new bus instead of refactoring a well known bus and API where brunches of drivers and devices had been implemented for years.

If you reason for this approach is to ease the implementation then you should talk about it in the cover letters/etc

I will add more rationale in both cover letter and this patch.

Thanks

Quoted 9 lines

Maybe it is reasonable to do this because the rework is too great, I don't know, but to me this whole thing looks rather messy.

Remember this stuff is all uAPI as it shows up in sysfs, so you can easilly get stuck with it forever.

Jason

#27 lore raw

Re: [PATCH V2 3/5] vDPA: introduce vDPA bus

Quoted 6 lines · Jason Wang
Quoted 2 lines

But it is
open coded and duplicated because .. vdpa?

I'm not sure I get here, vhost module is reused for vhost-vdpa and all current vhost device (e.g net) uses their own char device.

I mean there shouldn't be two fops implementing the same uAPI

Jason

author
#28 lore raw

PATCH v2 4/5 virtio: introduce a vDPA based transport

In reply to Jason Wang

This patch introduces a vDPA transport for virtio. This is used to use kernel virtio driver to drive the mediated device that is capable of populating virtqueue directly.

A new virtio-vdpa driver will be registered to the vDPA bus, when a new virtio-vdpa device is probed, it will register the device with vdpa based config ops. This means it is a software transport between vDPA driver and vDPA device. The transport was implemented through bus_ops of vDPA parent.

---
 drivers/virtio/Kconfig       |  13 ++
 drivers/virtio/Makefile      |   1 +
 drivers/virtio/virtio_vdpa.c | 392 +++++++++++++++++++++++++++++++++++
 3 files changed, 406 insertions(+)
 create mode 100644 drivers/virtio/virtio_vdpa.c
drivers/virtio/Kconfig +13 -0
index 9c4fdb64d9ac..0df3676b0f4f 100644
@@ -43,6 +43,19 @@ config VIRTIO_PCI_LEGACY
 
 	  If unsure, say Y.
 
+config VIRTIO_VDPA
+	tristate "vDPA driver for virtio devices"
+        select VDPA
+        select VIRTIO
+	help
+	  This driver provides support for virtio based paravirtual
+	  device driver over vDPA bus. For this to be useful, you need
+	  an appropriate vDPA device implementation that operates on a
+          physical device to allow the datapath of virtio to be
+	  offloaded to hardware.
+
+	  If unsure, say M.
+
 config VIRTIO_PMEM
 	tristate "Support for virtio pmem driver"
 	depends on VIRTIO
drivers/virtio/Makefile +1 -0
index fdf5eacd0d0a..3407ac03fe60 100644
@@ -6,4 +6,5 @@ virtio_pci-y := virtio_pci_modern.o virtio_pci_common.o
 virtio_pci-$(CONFIG_VIRTIO_PCI_LEGACY) += virtio_pci_legacy.o
 obj-$(CONFIG_VIRTIO_BALLOON) += virtio_balloon.o
 obj-$(CONFIG_VIRTIO_INPUT) += virtio_input.o
+obj-$(CONFIG_VIRTIO_VDPA) += virtio_vdpa.o
 obj-$(CONFIG_VDPA) += vdpa/
drivers/virtio/virtio_vdpa.c new file +392 -0

392 lines not on this page · raw message

Signature · 1 line
2.19.1
#29 lore raw

Re: [PATCH V2 4/5] virtio: introduce a vDPA based transport

Quoted 3 lines · Jason Wang

This patch introduces a vDPA transport for virtio. This is used to use kernel virtio driver to drive the mediated device that is capable of populating virtqueue directly.

Is this comment still right? Is there a mediated device still?

Jason

author
#30 lore raw

Re: [PATCH V2 4/5] virtio: introduce a vDPA based transport

Quoted 7 lines · Jason Gunthorpe
Quoted 3 lines · Jason Wang

This patch introduces a vDPA transport for virtio. This is used to use kernel virtio driver to drive the mediated device that is capable of populating virtqueue directly.

Is this comment still right? Is there a mediated device still?

Jason

No, will fix.

Thanks

Quoted 1 line

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

author
#31 lore raw

PATCH v2 5/5 vdpasim: vDPA device simulator

In reply to Jason Wang

This patch implements a software vDPA networking device. The datapath is implemented through vringh and workqueue. The device has an on-chip IOMMU which translates IOVA to PA. For kernel virtio drivers, vDPA simulator driver provides dma_ops. For vhost driers, set_map() methods of vdpa_config_ops is implemented to accept mappings from vhost.

Currently, vDPA device simulator will loopback TX traffic to RX. So the main use case for the device is vDPA feature testing, prototyping and development.

Note, there's no management API implemented, a vDPA device will be registered once the module is probed. We need to handle this in the future development.

---
 drivers/virtio/vdpa/Kconfig    |  17 +
 drivers/virtio/vdpa/Makefile   |   1 +
 drivers/virtio/vdpa/vdpa_sim.c | 678 +++++++++++++++++++++++++++++++++
 3 files changed, 696 insertions(+)
 create mode 100644 drivers/virtio/vdpa/vdpa_sim.c
drivers/virtio/vdpa/Kconfig +17 -0
index 7a99170e6c30..a7888974dda8 100644
@@ -7,3 +7,20 @@ config VDPA
           datapath which complies with virtio specifications with
           vendor specific control path.
 
+menuconfig VDPA_MENU
+	bool "VDPA drivers"
+	default n
+
+if VDPA_MENU
+
+config VDPA_SIM
+	tristate "vDPA device simulator"
+        select VDPA
+        default n
+        help
+          vDPA networking device simulator which loop TX traffic back
+          to RX. This device is used for testing, prototyping and
+          development of vDPA.
+
+endif # VDPA_MENU
+
drivers/virtio/vdpa/Makefile +1 -0
index ee6a35e8a4fb..5ec0e6ae3c57 100644
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_VDPA) += vdpa.o
+obj-$(CONFIG_VDPA_SIM) += vdpa_sim.o
drivers/virtio/vdpa/vdpa_sim.c new file +678 -0

678 lines not on this page · raw message

Signature · 1 line
2.19.1
#32 lore raw

Re: [PATCH V2 5/5] vdpasim: vDPA device simulator

Quoted 34 lines · Jason Wang

This patch implements a software vDPA networking device. The datapath is implemented through vringh and workqueue. The device has an on-chip IOMMU which translates IOVA to PA. For kernel virtio drivers, vDPA simulator driver provides dma_ops. For vhost driers, set_map() methods of vdpa_config_ops is implemented to accept mappings from vhost.

Currently, vDPA device simulator will loopback TX traffic to RX. So the main use case for the device is vDPA feature testing, prototyping and development.

Note, there's no management API implemented, a vDPA device will be registered once the module is probed. We need to handle this in the future development.

---
 drivers/virtio/vdpa/Kconfig    |  17 +
 drivers/virtio/vdpa/Makefile   |   1 +
 drivers/virtio/vdpa/vdpa_sim.c | 678 +++++++++++++++++++++++++++++++++
 3 files changed, 696 insertions(+)
 create mode 100644 drivers/virtio/vdpa/vdpa_sim.c
drivers/virtio/vdpa/Kconfig +4 -0
index 7a99170e6c30..a7888974dda8 100644
@@ -7,3 +7,20 @@ config VDPA
           datapath which complies with virtio specifications with
           vendor specific control path.
 
+menuconfig VDPA_MENU
+	bool "VDPA drivers"
+	default n
+

drivers/virtio/vdpa/Kconfig config VDPA · line 23

Quoted 10 lines
drivers/virtio/vdpa/Kconfig +10 -0
+if VDPA_MENU
+
+config VDPA_SIM
+	tristate "vDPA device simulator"
+        select VDPA
+        default n
+        help
+          vDPA networking device simulator which loop TX traffic back
+          to RX. This device is used for testing, prototyping and
+          development of vDPA.

So how about we make this depend on RUNTIME_TESTING_MENU?

Quoted 697 lines
drivers/virtio/vdpa/Kconfig +3 -0
+
+endif # VDPA_MENU
+
drivers/virtio/vdpa/Makefile +1 -0
index ee6a35e8a4fb..5ec0e6ae3c57 100644
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_VDPA) += vdpa.o
+obj-$(CONFIG_VDPA_SIM) += vdpa_sim.o
drivers/virtio/vdpa/vdpa_sim.c new file +678 -0
new file mode 100644
index 000000000000..89783a2b9773
@@ -0,0 +1,678 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * VDPA networking device simulator.
+ *
+ * Copyright (c) 2020, Red Hat Inc. All rights reserved.
+ *     Author: Jason Wang <[email protected]>
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/uuid.h>
+#include <linux/iommu.h>
+#include <linux/sysfs.h>
+#include <linux/file.h>
+#include <linux/etherdevice.h>
+#include <linux/vringh.h>
+#include <linux/vdpa.h>
+#include <linux/vhost_iotlb.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_net.h>
+
+#define DRV_VERSION  "0.1"
+#define DRV_AUTHOR   "Jason Wang <[email protected]>"
+#define DRV_DESC     "vDPA Device Simulator"
+#define DRV_LICENSE  "GPL v2"
+
+struct vdpasim_dev {
+	struct device	dev;
+};
+
+struct vdpasim_dev *vdpasim_dev;
+
+struct vdpasim_virtqueue {
+	struct vringh vring;
+	struct vringh_kiov iov;
+	unsigned short head;
+	bool ready;
+	u64 desc_addr;
+	u64 device_addr;
+	u64 driver_addr;
+	u32 num;
+	void *private;
+	irqreturn_t (*cb)(void *data);
+};
+
+#define VDPASIM_QUEUE_ALIGN PAGE_SIZE
+#define VDPASIM_QUEUE_MAX 256
+#define VDPASIM_DEVICE_ID 0x1
+#define VDPASIM_VENDOR_ID 0
+#define VDPASIM_VQ_NUM 0x2
+#define VDPASIM_NAME "vdpasim-netdev"
+
+static u64 vdpasim_features = (1ULL << VIRTIO_F_ANY_LAYOUT) |
+			      (1ULL << VIRTIO_F_VERSION_1)  |
+			      (1ULL << VIRTIO_F_IOMMU_PLATFORM);
+
+/* State of each vdpasim device */
+struct vdpasim {
+	struct vdpasim_virtqueue vqs[2];
+	struct work_struct work;
+	/* spinlock to synchronize virtqueue state */
+	spinlock_t lock;
+	struct vdpa_device vdpa;
+	struct virtio_net_config config;
+	struct vhost_iotlb *iommu;
+	void *buffer;
+	u32 status;
+	u32 generation;
+	u64 features;
+};
+
+struct vdpasim *vdpa_sim;
+
+static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)
+{
+	return container_of(vdpa, struct vdpasim, vdpa);
+}
+
+static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
+{
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+	int ret;
+
+	ret = vringh_init_iotlb(&vq->vring, vdpasim_features,
+				VDPASIM_QUEUE_MAX, false,
+				(struct vring_desc *)(uintptr_t)vq->desc_addr,
+				(struct vring_avail *)
+				(uintptr_t)vq->driver_addr,
+				(struct vring_used *)
+				(uintptr_t)vq->device_addr);
+}
+
+static void vdpasim_vq_reset(struct vdpasim_virtqueue *vq)
+{
+	vq->ready = 0;
+	vq->desc_addr = 0;
+	vq->driver_addr = 0;
+	vq->device_addr = 0;
+	vq->cb = NULL;
+	vq->private = NULL;
+	vringh_init_iotlb(&vq->vring, vdpasim_features, VDPASIM_QUEUE_MAX,
+			  false, 0, 0, 0);
+}
+
+static void vdpasim_reset(struct vdpasim *vdpasim)
+{
+	int i;
+
+	for (i = 0; i < VDPASIM_VQ_NUM; i++)
+		vdpasim_vq_reset(&vdpasim->vqs[i]);
+
+	vhost_iotlb_reset(vdpasim->iommu);
+
+	vdpasim->features = 0;
+	vdpasim->status = 0;
+	++vdpasim->generation;
+}
+
+static void vdpasim_work(struct work_struct *work)
+{
+	struct vdpasim *vdpasim = container_of(work, struct
+						 vdpasim, work);
+	struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
+	struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
+	size_t read, write, total_write;
+	int err;
+	int pkts = 0;
+
+	spin_lock(&vdpasim->lock);
+
+	if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
+		goto out;
+
+	if (!txq->ready || !rxq->ready)
+		goto out;
+
+	while (true) {
+		total_write = 0;
+		err = vringh_getdesc_iotlb(&txq->vring, &txq->iov, NULL,
+					   &txq->head, GFP_ATOMIC);
+		if (err <= 0)
+			break;
+
+		err = vringh_getdesc_iotlb(&rxq->vring, NULL, &rxq->iov,
+					   &rxq->head, GFP_ATOMIC);
+		if (err <= 0) {
+			vringh_complete_iotlb(&txq->vring, txq->head, 0);
+			break;
+		}
+
+		while (true) {
+			read = vringh_iov_pull_iotlb(&txq->vring, &txq->iov,
+						     vdpasim->buffer,
+						     PAGE_SIZE);
+			if (read <= 0)
+				break;
+
+			write = vringh_iov_push_iotlb(&rxq->vring, &rxq->iov,
+						      vdpasim->buffer, read);
+			if (write <= 0)
+				break;
+
+			total_write += write;
+		}
+
+		/* Make sure data is wrote before advancing index */
+		smp_wmb();
+
+		vringh_complete_iotlb(&txq->vring, txq->head, 0);
+		vringh_complete_iotlb(&rxq->vring, rxq->head, total_write);
+
+		/* Make sure used is visible before rasing the interrupt. */
+		smp_wmb();
+
+		local_bh_disable();
+		if (txq->cb)
+			txq->cb(txq->private);
+		if (rxq->cb)
+			rxq->cb(rxq->private);
+		local_bh_enable();
+
+		if (++pkts > 4) {
+			schedule_work(&vdpasim->work);
+			goto out;
+		}
+	}
+
+out:
+	spin_unlock(&vdpasim->lock);
+}
+
+static int dir_to_perm(enum dma_data_direction dir)
+{
+	int perm = -EFAULT;
+
+	switch (dir) {
+	case DMA_FROM_DEVICE:
+		perm = VHOST_MAP_WO;
+		break;
+	case DMA_TO_DEVICE:
+		perm = VHOST_MAP_RO;
+		break;
+	case DMA_BIDIRECTIONAL:
+		perm = VHOST_MAP_RW;
+		break;
+	default:
+		break;
+	}
+
+	return perm;
+}
+
+static dma_addr_t vdpasim_map_page(struct device *dev, struct page *page,
+				   unsigned long offset, size_t size,
+				   enum dma_data_direction dir,
+				   unsigned long attrs)
+{
+	struct vdpa_device *vdpa = dev_to_vdpa(dev);
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vhost_iotlb *iommu = vdpasim->iommu;
+	u64 pa = (page_to_pfn(page) << PAGE_SHIFT) + offset;
+	int ret, perm = dir_to_perm(dir);
+
+	if (perm < 0)
+		return DMA_MAPPING_ERROR;
+
+	/* For simplicity, use identical mapping to avoid e.g iova
+	 * allocator.
+	 */
+	ret = vhost_iotlb_add_range(iommu, pa, pa + size - 1,
+				    pa, dir_to_perm(dir));
+	if (ret)
+		return DMA_MAPPING_ERROR;
+
+	return (dma_addr_t)(pa);
+}
+
+static void vdpasim_unmap_page(struct device *dev, dma_addr_t dma_addr,
+			       size_t size, enum dma_data_direction dir,
+			       unsigned long attrs)
+{
+	struct vdpa_device *vdpa = dev_to_vdpa(dev);
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vhost_iotlb *iommu = vdpasim->iommu;
+
+	vhost_iotlb_del_range(iommu, (u64)dma_addr,
+			      (u64)dma_addr + size - 1);
+}
+
+static void *vdpasim_alloc_coherent(struct device *dev, size_t size,
+				    dma_addr_t *dma_addr, gfp_t flag,
+				    unsigned long attrs)
+{
+	struct vdpa_device *vdpa = dev_to_vdpa(dev);
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vhost_iotlb *iommu = vdpasim->iommu;
+	void *addr = kmalloc(size, flag);
+	int ret;
+
+	if (!addr)
+		*dma_addr = DMA_MAPPING_ERROR;
+	else {
+		u64 pa = virt_to_phys(addr);
+
+		ret = vhost_iotlb_add_range(iommu, (u64)pa,
+					    (u64)pa + size - 1,
+					    pa, VHOST_MAP_RW);
+		if (ret) {
+			*dma_addr = DMA_MAPPING_ERROR;
+			kfree(addr);
+			addr = NULL;
+		} else
+			*dma_addr = (dma_addr_t)pa;
+	}
+
+	return addr;
+}
+
+static void vdpasim_free_coherent(struct device *dev, size_t size,
+				void *vaddr, dma_addr_t dma_addr,
+				unsigned long attrs)
+{
+	struct vdpa_device *vdpa = dev_to_vdpa(dev);
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vhost_iotlb *iommu = vdpasim->iommu;
+
+	vhost_iotlb_del_range(iommu, (u64)dma_addr,
+			      (u64)dma_addr + size - 1);
+	kfree(phys_to_virt((uintptr_t)dma_addr));
+}
+
+static const struct dma_map_ops vdpasim_dma_ops = {
+	.map_page = vdpasim_map_page,
+	.unmap_page = vdpasim_unmap_page,
+	.alloc = vdpasim_alloc_coherent,
+	.free = vdpasim_free_coherent,
+};
+
+static void vdpasim_release_dev(struct device *_d)
+{
+	struct vdpa_device *vdpa = dev_to_vdpa(_d);
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	kfree(vdpasim->buffer);
+	kfree(vdpasim);
+}
+
+static const struct vdpa_config_ops vdpasim_net_config_ops;
+
+static struct vdpasim *vdpasim_create(void)
+{
+	struct vdpasim *vdpasim;
+	struct virtio_net_config *config;
+	struct vdpa_device *vdpa;
+	struct device *dev;
+	int ret = -ENOMEM;
+
+	vdpasim = kzalloc(sizeof(*vdpasim), GFP_KERNEL);
+	if (!vdpasim)
+		goto err_vdpa_alloc;
+
+	vdpasim->buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!vdpasim->buffer)
+		goto err_buffer_alloc;
+
+	vdpasim->iommu = vhost_iotlb_alloc(2048, 0);
+	if (!vdpasim->iommu)
+		goto err_iotlb;
+
+	config = &vdpasim->config;
+	config->mtu = 1500;
+	config->status = VIRTIO_NET_S_LINK_UP;
+	eth_random_addr(config->mac);
+
+	INIT_WORK(&vdpasim->work, vdpasim_work);
+	spin_lock_init(&vdpasim->lock);
+
+	vdpa = &vdpasim->vdpa;
+	vdpa->dev.release = vdpasim_release_dev;
+
+	vringh_set_iotlb(&vdpasim->vqs[0].vring, vdpasim->iommu);
+	vringh_set_iotlb(&vdpasim->vqs[1].vring, vdpasim->iommu);
+
+	dev = &vdpa->dev;
+	dev->coherent_dma_mask = DMA_BIT_MASK(64);
+	set_dma_ops(dev, &vdpasim_dma_ops);
+
+	ret = vdpa_init_device(vdpa, &vdpasim_dev->dev, dev,
+			       &vdpasim_net_config_ops);
+	if (ret)
+		goto err_init;
+
+	ret = vdpa_register_device(vdpa);
+	if (ret)
+		goto err_register;
+
+	return vdpasim;
+
+err_register:
+	put_device(&vdpa->dev);
+err_init:
+	vhost_iotlb_free(vdpasim->iommu);
+err_iotlb:
+	kfree(vdpasim->buffer);
+err_buffer_alloc:
+	kfree(vdpasim);
+err_vdpa_alloc:
+	return ERR_PTR(ret);
+}
+
+static int vdpasim_set_vq_address(struct vdpa_device *vdpa, u16 idx,
+				  u64 desc_area, u64 driver_area,
+				  u64 device_area)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	vq->desc_addr = desc_area;
+	vq->driver_addr = driver_area;
+	vq->device_addr = device_area;
+
+	return 0;
+}
+
+static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	vq->num = num;
+}
+
+static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	if (vq->ready)
+		schedule_work(&vdpasim->work);
+}
+
+static void vdpasim_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
+			      struct vdpa_callback *cb)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	vq->cb = cb->callback;
+	vq->private = cb->private;
+}
+
+static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	spin_lock(&vdpasim->lock);
+	vq->ready = ready;
+	if (vq->ready)
+		vdpasim_queue_ready(vdpasim, idx);
+	spin_unlock(&vdpasim->lock);
+}
+
+static bool vdpasim_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	return vq->ready;
+}
+
+static int vdpasim_set_vq_state(struct vdpa_device *vdpa, u16 idx, u64 state)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+	struct vringh *vrh = &vq->vring;
+
+	spin_lock(&vdpasim->lock);
+	vrh->last_avail_idx = state;
+	spin_unlock(&vdpasim->lock);
+
+	return 0;
+}
+
+static u64 vdpasim_get_vq_state(struct vdpa_device *vdpa, u16 idx)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+	struct vringh *vrh = &vq->vring;
+
+	return vrh->last_avail_idx;
+}
+
+static u16 vdpasim_get_vq_align(struct vdpa_device *vdpa)
+{
+	return VDPASIM_QUEUE_ALIGN;
+}
+
+static u64 vdpasim_get_features(struct vdpa_device *vdpa)
+{
+	return vdpasim_features;
+}
+
+static int vdpasim_set_features(struct vdpa_device *vdpa, u64 features)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	/* DMA mapping must be done by driver */
+	if (!(features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)))
+		return -EINVAL;
+
+	vdpasim->features = features & vdpasim_features;
+
+	return 0;
+}
+
+static void vdpasim_set_config_cb(struct vdpa_device *vdpa,
+				  struct vdpa_callback *cb)
+{
+	/* We don't support config interrupt */
+}
+
+static u16 vdpasim_get_vq_num_max(struct vdpa_device *vdpa)
+{
+	return VDPASIM_QUEUE_MAX;
+}
+
+static u32 vdpasim_get_device_id(struct vdpa_device *vdpa)
+{
+	return VDPASIM_DEVICE_ID;
+}
+
+static u32 vdpasim_get_vendor_id(struct vdpa_device *vdpa)
+{
+	return VDPASIM_VENDOR_ID;
+}
+
+static u8 vdpasim_get_status(struct vdpa_device *vdpa)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	u8 status;
+
+	spin_lock(&vdpasim->lock);
+	status = vdpasim->status;
+	spin_unlock(&vdpasim->lock);
+
+	return vdpasim->status;
+}
+
+static void vdpasim_set_status(struct vdpa_device *vdpa, u8 status)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	spin_lock(&vdpasim->lock);
+	vdpasim->status = status;
+	if (status == 0)
+		vdpasim_reset(vdpasim);
+	spin_unlock(&vdpasim->lock);
+}
+
+static void vdpasim_get_config(struct vdpa_device *vdpa, unsigned int offset,
+			     void *buf, unsigned int len)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	if (offset + len < sizeof(struct virtio_net_config))
+		memcpy(buf, &vdpasim->config + offset, len);
+}
+
+static void vdpasim_set_config(struct vdpa_device *vdpa, unsigned int offset,
+			     const void *buf, unsigned int len)
+{
+	/* No writable config supportted by vdpasim */
+}
+
+static u32 vdpasim_get_generation(struct vdpa_device *vdpa)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	return vdpasim->generation;
+}
+
+static int vdpasim_set_map(struct vdpa_device *vdpa,
+			   struct vhost_iotlb *iotlb)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vhost_iotlb_map *map;
+	u64 start = 0ULL, last = 0ULL - 1;
+	int ret;
+
+	vhost_iotlb_reset(vdpasim->iommu);
+
+	for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
+	     map = vhost_iotlb_itree_next(map, start, last)) {
+		ret = vhost_iotlb_add_range(vdpasim->iommu, map->start,
+					    map->last, map->addr, map->perm);
+		if (ret)
+			goto err;
+	}
+	return 0;
+
+err:
+	vhost_iotlb_reset(vdpasim->iommu);
+	return ret;
+}
+
+static int vdpasim_dma_map(struct vdpa_device *vdpa, u64 iova, u64 size,
+			   u64 pa, u32 perm)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	return vhost_iotlb_add_range(vdpasim->iommu, iova,
+				     iova + size - 1, pa, perm);
+}
+
+static int vdpasim_dma_unmap(struct vdpa_device *vdpa, u64 iova, u64 size)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+
+	vhost_iotlb_del_range(vdpasim->iommu, iova, iova + size - 1);
+
+	return 0;
+}
+
+static const struct vdpa_config_ops vdpasim_net_config_ops = {
+	.set_vq_address         = vdpasim_set_vq_address,
+	.set_vq_num             = vdpasim_set_vq_num,
+	.kick_vq                = vdpasim_kick_vq,
+	.set_vq_cb              = vdpasim_set_vq_cb,
+	.set_vq_ready           = vdpasim_set_vq_ready,
+	.get_vq_ready           = vdpasim_get_vq_ready,
+	.set_vq_state           = vdpasim_set_vq_state,
+	.get_vq_state           = vdpasim_get_vq_state,
+	.get_vq_align           = vdpasim_get_vq_align,
+	.get_features           = vdpasim_get_features,
+	.set_features           = vdpasim_set_features,
+	.set_config_cb          = vdpasim_set_config_cb,
+	.get_vq_num_max         = vdpasim_get_vq_num_max,
+	.get_device_id          = vdpasim_get_device_id,
+	.get_vendor_id          = vdpasim_get_vendor_id,
+	.get_status             = vdpasim_get_status,
+	.set_status             = vdpasim_set_status,
+	.get_config             = vdpasim_get_config,
+	.set_config             = vdpasim_set_config,
+	.get_generation         = vdpasim_get_generation,
+	.set_map                = vdpasim_set_map,
+	.dma_map                = vdpasim_dma_map,
+	.dma_unmap              = vdpasim_dma_unmap,
+};
+
+static void vdpasim_device_release(struct device *dev)
+{
+	struct vdpasim_dev *vdpasim_dev =
+	       container_of(dev, struct vdpasim_dev, dev);
+
+	vdpasim_dev->dev.bus = NULL;
+	kfree(vdpasim_dev);
+}
+
+static int __init vdpasim_dev_init(void)
+{
+	struct device *dev;
+	int ret = 0;
+
+	vdpasim_dev = kzalloc(sizeof(*vdpasim_dev), GFP_KERNEL);
+	if (!vdpasim_dev)
+		return -ENOMEM;
+
+	dev = &vdpasim_dev->dev;
+	dev->release = vdpasim_device_release;
+	dev_set_name(dev, "%s", VDPASIM_NAME);
+
+	ret = device_register(&vdpasim_dev->dev);
+	if (ret)
+		goto err_register;
+
+	if (!vdpasim_create())
+		goto err_register;
+
+	return 0;
+
+err_register:
+	kfree(vdpasim_dev);
+	vdpasim_dev = NULL;
+	return ret;
+}
+
+static int vdpasim_device_remove_cb(struct device *dev, void *data)
+{
+	struct vdpa_device *vdpa = dev_to_vdpa(dev);
+
+	vdpa_unregister_device(vdpa);
+
+	return 0;
+}
+
+static void __exit vdpasim_dev_exit(void)
+{
+	device_for_each_child(&vdpasim_dev->dev, NULL,
+			      vdpasim_device_remove_cb);
+	device_unregister(&vdpasim_dev->dev);
+}
+
+module_init(vdpasim_dev_init)
+module_exit(vdpasim_dev_exit)
+
+MODULE_VERSION(DRV_VERSION);
+MODULE_LICENSE(DRV_LICENSE);
+MODULE_AUTHOR(DRV_AUTHOR);
+MODULE_DESCRIPTION(DRV_DESC);
Signature · 1 line
2.19.1
author
#33 lore raw

Re: [PATCH V2 5/5] vdpasim: vDPA device simulator

Quoted 31 lines · Michael S. Tsirkin
Quoted 30 lines · Jason Wang

This patch implements a software vDPA networking device. The datapath is implemented through vringh and workqueue. The device has an on-chip IOMMU which translates IOVA to PA. For kernel virtio drivers, vDPA simulator driver provides dma_ops. For vhost driers, set_map() methods of vdpa_config_ops is implemented to accept mappings from vhost.

Currently, vDPA device simulator will loopback TX traffic to RX. So the main use case for the device is vDPA feature testing, prototyping and development.

Note, there's no management API implemented, a vDPA device will be registered once the module is probed. We need to handle this in the future development.

---
  drivers/virtio/vdpa/Kconfig    |  17 +
  drivers/virtio/vdpa/Makefile   |   1 +
  drivers/virtio/vdpa/vdpa_sim.c | 678 +++++++++++++++++++++++++++++++++
  3 files changed, 696 insertions(+)

create mode 100644 drivers/virtio/vdpa/vdpa_sim.c

drivers/virtio/vdpa/Kconfig +0 -0
index 7a99170e6c30..a7888974dda8 100644
@@ -7,3 +7,20 @@ config VDPA
            datapath which complies with virtio specifications with
            vendor specific control path.
 
Quoted 17 lines
Quoted 4 lines
drivers/virtio/vdpa/Kconfig +4 -0
+menuconfig VDPA_MENU
+	bool "VDPA drivers"
+	default n
+

drivers/virtio/vdpa/Kconfig config VDPA · line 23

Quoted 10 lines
drivers/virtio/vdpa/Kconfig +10 -0
+if VDPA_MENU
+
+config VDPA_SIM
+	tristate "vDPA device simulator"
+        select VDPA
+        default n
+        help
+          vDPA networking device simulator which loop TX traffic back
+          to RX. This device is used for testing, prototyping and
+          development of vDPA.

So how about we make this depend on RUNTIME_TESTING_MENU?

I'm not sure how much it can help but I can do that in next version.

Thanks

RUNTIME_TESTING_MENU

#34 lore raw

Re: [PATCH V2 5/5] vdpasim: vDPA device simulator

In reply to Jason Wang
Quoted 21 lines · Jason Wang
drivers/virtio/vdpa/vdpa_sim.c new file +21 -0
+
+static struct vdpasim *vdpasim_create(void)
+{
+	struct vdpasim *vdpasim;
+	struct virtio_net_config *config;
+	struct vdpa_device *vdpa;
+	struct device *dev;
+	int ret = -ENOMEM;
+
+	vdpasim = kzalloc(sizeof(*vdpasim), GFP_KERNEL);
+	if (!vdpasim)
+		goto err_vdpa_alloc;
+
+	vdpasim->buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!vdpasim->buffer)
+		goto err_buffer_alloc;
+
+	vdpasim->iommu = vhost_iotlb_alloc(2048, 0);
+	if (!vdpasim->iommu)
+		goto err_iotlb;
+

drivers/virtio/vdpa/vdpa_sim.c line 347

Quoted 10 lines
drivers/virtio/vdpa/vdpa_sim.c new file +10 -0
+	config = &vdpasim->config;
+	config->mtu = 1500;
+	config->status = VIRTIO_NET_S_LINK_UP;
+	eth_random_addr(config->mac);
+
+	INIT_WORK(&vdpasim->work, vdpasim_work);
+	spin_lock_init(&vdpasim->lock);
+
+	vdpa = &vdpasim->vdpa;
+	vdpa->dev.release = vdpasim_release_dev;

The driver should not provide the release function.

Again the safest model is 'vdpa_alloc_device' which combines the kzalloc and the vdpa_init_device() and returns something that is error unwound with put_device()

The subsystem owns the release and does the kfree and other cleanup like releasing the IDA.

Quoted 5 lines
drivers/virtio/vdpa/vdpa_sim.c new file +5 -0
+	vringh_set_iotlb(&vdpasim->vqs[0].vring, vdpasim->iommu);
+	vringh_set_iotlb(&vdpasim->vqs[1].vring, vdpasim->iommu);
+
+	dev = &vdpa->dev;
+	dev->coherent_dma_mask = DMA_BIT_MASK(64);

drivers/virtio/vdpa/vdpa_sim.c line 363

Quoted 10 lines
drivers/virtio/vdpa/vdpa_sim.c new file +10 -0
+	set_dma_ops(dev, &vdpasim_dma_ops);
+
+	ret = vdpa_init_device(vdpa, &vdpasim_dev->dev, dev,
+			       &vdpasim_net_config_ops);
+	if (ret)
+		goto err_init;
+
+	ret = vdpa_register_device(vdpa);
+	if (ret)
+		goto err_register;

See? This error unwind is now all wrong:

drivers/virtio/vdpa/vdpa_sim.c line 368

Quoted 5 lines
drivers/virtio/vdpa/vdpa_sim.c new file +5 -0
+
+	return vdpasim;
+
+err_register:
+	put_device(&vdpa->dev);

Double put_device

drivers/virtio/vdpa/vdpa_sim.c line 374

Quoted 6 lines
drivers/virtio/vdpa/vdpa_sim.c new file +6 -0
+err_init:
+	vhost_iotlb_free(vdpasim->iommu);
+err_iotlb:
+	kfree(vdpasim->buffer);
+err_buffer_alloc:
+	kfree(vdpasim);

kfree after vdpa_init_device() is incorrect, as the put_device now does kfree via release

Quoted 9 lines
drivers/virtio/vdpa/vdpa_sim.c new file +9 -0
+static int __init vdpasim_dev_init(void)
+{
+	struct device *dev;
+	int ret = 0;
+
+	vdpasim_dev = kzalloc(sizeof(*vdpasim_dev), GFP_KERNEL);
+	if (!vdpasim_dev)
+		return -ENOMEM;
+

drivers/virtio/vdpa/vdpa_sim.c line 646

Quoted 10 lines
drivers/virtio/vdpa/vdpa_sim.c new file +10 -0
+	dev = &vdpasim_dev->dev;
+	dev->release = vdpasim_device_release;
+	dev_set_name(dev, "%s", VDPASIM_NAME);
+
+	ret = device_register(&vdpasim_dev->dev);
+	if (ret)
+		goto err_register;
+
+	if (!vdpasim_create())
+		goto err_register;

Wrong error unwind here too

drivers/virtio/vdpa/vdpa_sim.c line 654

Quoted 7 lines
drivers/virtio/vdpa/vdpa_sim.c new file +7 -0
+	return 0;
+
+err_register:
+	kfree(vdpasim_dev);
+	vdpasim_dev = NULL;
+	return ret;
+}

Jason

author
#35 lore raw

Re: [PATCH V2 5/5] vdpasim: vDPA device simulator

Quoted 34 lines · Jason Gunthorpe
Quoted 21 lines · Jason Wang
drivers/virtio/vdpa/vdpa_sim.c new file +21 -0
+
+static struct vdpasim *vdpasim_create(void)
+{
+	struct vdpasim *vdpasim;
+	struct virtio_net_config *config;
+	struct vdpa_device *vdpa;
+	struct device *dev;
+	int ret = -ENOMEM;
+
+	vdpasim = kzalloc(sizeof(*vdpasim), GFP_KERNEL);
+	if (!vdpasim)
+		goto err_vdpa_alloc;
+
+	vdpasim->buffer = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!vdpasim->buffer)
+		goto err_buffer_alloc;
+
+	vdpasim->iommu = vhost_iotlb_alloc(2048, 0);
+	if (!vdpasim->iommu)
+		goto err_iotlb;
+

drivers/virtio/vdpa/vdpa_sim.c line 347

Quoted 10 lines
drivers/virtio/vdpa/vdpa_sim.c new file +10 -0
+	config = &vdpasim->config;
+	config->mtu = 1500;
+	config->status = VIRTIO_NET_S_LINK_UP;
+	eth_random_addr(config->mac);
+
+	INIT_WORK(&vdpasim->work, vdpasim_work);
+	spin_lock_init(&vdpasim->lock);
+
+	vdpa = &vdpasim->vdpa;
+	vdpa->dev.release = vdpasim_release_dev;

The driver should not provide the release function.

Quoted 6 lines

Again the safest model is 'vdpa_alloc_device' which combines the kzalloc and the vdpa_init_device() and returns something that is error unwound with put_device()

The subsystem owns the release and does the kfree and other cleanup like releasing the IDA.

So I think if we agree bus instead of class is used. vDPA bus can provide a release function in vdpa_alloc_device()?

Quoted 18 lines
Quoted 6 lines
drivers/virtio/vdpa/vdpa_sim.c new file +5 -0
+	vringh_set_iotlb(&vdpasim->vqs[0].vring, vdpasim->iommu);
+	vringh_set_iotlb(&vdpasim->vqs[1].vring, vdpasim->iommu);
+
+	dev = &vdpa->dev;
+	dev->coherent_dma_mask = DMA_BIT_MASK(64);

drivers/virtio/vdpa/vdpa_sim.c line 363

Quoted 10 lines
drivers/virtio/vdpa/vdpa_sim.c new file +10 -0
+	set_dma_ops(dev, &vdpasim_dma_ops);
+
+	ret = vdpa_init_device(vdpa, &vdpasim_dev->dev, dev,
+			       &vdpasim_net_config_ops);
+	if (ret)
+		goto err_init;
+
+	ret = vdpa_register_device(vdpa);
+	if (ret)
+		goto err_register;

See? This error unwind is now all wrong:

Quoted 6 lines

drivers/virtio/vdpa/vdpa_sim.c line 368

Quoted 5 lines
drivers/virtio/vdpa/vdpa_sim.c new file +5 -0
+
+	return vdpasim;
+
+err_register:
+	put_device(&vdpa->dev);

Double put_device

Yes.

Quoted 9 lines

drivers/virtio/vdpa/vdpa_sim.c line 374

Quoted 7 lines
drivers/virtio/vdpa/vdpa_sim.c new file +6 -0
+err_init:
+	vhost_iotlb_free(vdpasim->iommu);
+err_iotlb:
+	kfree(vdpasim->buffer);
+err_buffer_alloc:
+	kfree(vdpasim);

kfree after vdpa_init_device() is incorrect, as the put_device now does kfree via release

Ok, will fix.

Quoted 13 lines
Quoted 13 lines
drivers/virtio/vdpa/vdpa_sim.c new file +12 -0
+static int __init vdpasim_dev_init(void)
+{
+	struct device *dev;
+	int ret = 0;
+
+	vdpasim_dev = kzalloc(sizeof(*vdpasim_dev), GFP_KERNEL);
+	if (!vdpasim_dev)
+		return -ENOMEM;
+
+	dev = &vdpasim_dev->dev;
+	dev->release = vdpasim_device_release;
+	dev_set_name(dev, "%s", VDPASIM_NAME);
Quoted 8 lines

drivers/virtio/vdpa/vdpa_sim.c line 646

Quoted 7 lines
drivers/virtio/vdpa/vdpa_sim.c new file +7 -0
+
+	ret = device_register(&vdpasim_dev->dev);
+	if (ret)
+		goto err_register;
+
+	if (!vdpasim_create())
+		goto err_register;

Wrong error unwind here too

Will fix.

Thanks

Quoted 10 lines

drivers/virtio/vdpa/vdpa_sim.c line 654

Quoted 8 lines
drivers/virtio/vdpa/vdpa_sim.c new file +7 -0
+	return 0;
+
+err_register:
+	kfree(vdpasim_dev);
+	vdpasim_dev = NULL;
+	return ret;
+}

Jason