vDPA: doorbell mapping
Subsystems (7)
cover vDPA: doorbell mapping
Hi all:
This series introduce basic functionality of doorbell mapping support for vhost-vDPA. Userspace program may use mmap() to map a the doorbell of a specific virtqueue into its address space. This is help to reudce the syscall or vmexit overhead.
A new vdpa_config_ops was introduced to report the location of the doorbell, vhost_vdpa may then choose to map the doorbell when:
- The doorbell register is localted at page boundary
- The doorbell register does not share page with non doorbell registers.
With these two requriements, doorbells layout could be modelled more easily from guest (e.g Qemu's page-per-vq model) and it would be more safe to avoid exposing other registers to userspace directly.
IFCVF was reported to support this feature but unfortuantely the one I used does not meet those requirements. So a new virtio-pci driver for vDPA bus is introduced, and I verify this with page-per-vq=on with a userspace vhost-vdpa driver in guest.
Please review.
Thanks
Jason Wang (6):
vhost: allow device that does not depend on vhost worker
vhost: use mmgrab() instead of mmget() for non worker device
vdpa: introduce get_vq_notification method
vhost_vdpa: support doorbell mapping via mmap
vdpa: introduce virtio pci driver
vdpa: vp_vdpa: report doorbell location
drivers/vdpa/Kconfig | 6 + drivers/vdpa/Makefile | 1 + drivers/vdpa/vp_vdpa/Makefile | 2 + drivers/vdpa/vp_vdpa/vp_vdpa.c | 604 +++++++++++++++++++++++++++++++++ drivers/vhost/net.c | 2 +- drivers/vhost/scsi.c | 2 +- drivers/vhost/vdpa.c | 61 +++- drivers/vhost/vhost.c | 80 +++-- drivers/vhost/vhost.h | 2 + drivers/vhost/vsock.c | 2 +- include/linux/vdpa.h | 16 + 11 files changed, 753 insertions(+), 25 deletions(-) create mode 100644 drivers/vdpa/vp_vdpa/Makefile create mode 100644 drivers/vdpa/vp_vdpa/vp_vdpa.c
Signature · 1 line
2.20.1
PATCH 1/6 vhost: allow device that does not depend on vhost worker
vDPA device currently relays the eventfd via vhost worker. This is inefficient due the latency of wakeup and scheduling, so this patch tries to introduce a use_worker attribute for the vhost device. When use_worker is not set with vhost_dev_init(), vhost won't try to allocate a worker thread and the vhost_poll will be processed directly in the wakeup function.
This help for vDPA since it reduces the latency caused by vhost worker.
In my testing, it saves 0.2 ms in pings between VMs on a mutual host.
--- drivers/vhost/net.c | 2 +- drivers/vhost/scsi.c | 2 +- drivers/vhost/vdpa.c | 2 +- drivers/vhost/vhost.c | 38 +++++++++++++++++++++++++------------- drivers/vhost/vhost.h | 2 ++ drivers/vhost/vsock.c | 2 +- 6 files changed, 31 insertions(+), 17 deletions(-)
drivers/vhost/net.c +1 -1
@@ -1326,7 +1326,7 @@ static int vhost_net_open(struct inode *inode, struct file *f) }vhost_dev_init (dev, vqs,VHOST_NET_VQ_MAX ,UIO_MAXIOV +VHOST_NET_BATCH , -VHOST_NET_PKT_WEIGHT ,VHOST_NET_WEIGHT , +VHOST_NET_PKT_WEIGHT ,VHOST_NET_WEIGHT , true,NULL );vhost_poll_init (n->poll +VHOST_NET_VQ_TX , handle_tx_net,EPOLLOUT , dev);
drivers/vhost/scsi.c +1 -1
@@ -1628,7 +1628,7 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; }vhost_dev_init (&vs->dev, vqs,VHOST_SCSI_MAX_VQ ,UIO_MAXIOV , -VHOST_SCSI_WEIGHT ,0 ,NULL ); +VHOST_SCSI_WEIGHT ,0 , true,NULL );vhost_scsi_init_inflight (vs,NULL );
drivers/vhost/vdpa.c +1 -1
@@ -696,7 +696,7 @@ static int vhost_vdpa_open(struct inode *inode, struct file *filep) vqs[i] = &v->vqs[i]; vqs[i]->handle_kick = handle_vq_kick; } -vhost_dev_init (dev, vqs, nvqs,0 ,0 ,0 , +vhost_dev_init (dev, vqs, nvqs,0 ,0 ,0 , false, vhost_vdpa_process_iotlb_msg); dev->iotlb =vhost_iotlb_alloc (0 ,0 );
drivers/vhost/vhost.c +25 -13
@@ -166,11 +166,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) { struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); +struct vhost_work *work = &poll->work;if (!(key_to_poll (key) & poll->mask))return 0 ; -vhost_poll_queue (poll); +if (!poll->dev->use_worker) + work->fn (work); +else +vhost_poll_queue (poll); +return 0 ; }
@@ -454,6 +459,7 @@ static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,void vhost_dev_init (struct vhost_dev *dev,struct vhost_virtqueue **vqs,int nvqs,int iov_limit,int weight,int byte_weight, +bool use_worker,int (*msg_handler)(struct vhost_dev *dev,struct vhost_iotlb_msg *msg)) {
@@ -471,6 +477,7 @@ void vhost_dev_init(struct vhost_dev *dev, dev->iov_limit = iov_limit; dev->weight = weight; dev->byte_weight = byte_weight; + dev->use_worker = use_worker; dev->msg_handler = msg_handler;init_llist_head (&dev->work_list);init_waitqueue_head (&dev->wait);
@@ -549,18 +556,21 @@ long vhost_dev_set_owner(struct vhost_dev *dev)/* No owner, become one */ dev->mm =get_task_mm (current); dev->kcov_handle =kcov_common_handle (); - worker =kthread_create (vhost_worker, dev,"vhost-%d" , current->pid); -if (IS_ERR (worker)) { - err =PTR_ERR (worker); - goto err_worker; - } +if (dev->use_worker) { + worker =kthread_create (vhost_worker, dev, +"vhost-%d" , current->pid); +if (IS_ERR (worker)) { + err =PTR_ERR (worker); + goto err_worker; + } - dev->worker = worker; -wake_up_process (worker);/* avoid contributing to loadavg */ + dev->worker = worker; +wake_up_process (worker);/* avoid contributing to loadavg */ - err =vhost_attach_cgroups (dev); -if (err) - goto err_cgroup; + err =vhost_attach_cgroups (dev); +if (err) + goto err_cgroup; + } err =vhost_dev_alloc_iovecs (dev);if (err)
@@ -568,8 +578,10 @@ long vhost_dev_set_owner(struct vhost_dev *dev)return 0 ; err_cgroup: -kthread_stop (worker); - dev->worker =NULL ; +if (dev->worker) { +kthread_stop (dev->worker); + dev->worker =NULL ; + } err_worker:if (dev->mm)mmput (dev->mm);
drivers/vhost/vhost.h +2 -0
@@ -154,6 +154,7 @@ struct vhost_dev {int weight;int byte_weight;u64 kcov_handle; +bool use_worker;int (*msg_handler)(struct vhost_dev *dev,struct vhost_iotlb_msg *msg); };
@@ -161,6 +162,7 @@ struct vhost_dev {bool vhost_exceeds_weight (struct vhost_virtqueue *vq,int pkts,int total_len);void vhost_dev_init (struct vhost_dev *,struct vhost_virtqueue **vqs,int nvqs,int iov_limit,int weight,int byte_weight, +bool use_worker,int (*msg_handler)(struct vhost_dev *dev,struct vhost_iotlb_msg *msg));long vhost_dev_set_owner (struct vhost_dev *dev);
drivers/vhost/vsock.c +1 -1
@@ -621,7 +621,7 @@ static int vhost_vsock_dev_open(struct inode *inode, struct file *file)vhost_dev_init (&vsock->dev, vqs,ARRAY_SIZE (vsock->vqs),UIO_MAXIOV ,VHOST_VSOCK_PKT_WEIGHT , -VHOST_VSOCK_WEIGHT ,NULL ); +VHOST_VSOCK_WEIGHT , true,NULL ); file->private_data = vsock;spin_lock_init (&vsock->send_pkt_list_lock);
Signature · 1 line
2.20.1
Michael S. Tsirkin
maintainerRe: [PATCH 1/6] vhost: allow device that does not depend on vhost worker
Quoted 12 lines · Jason Wang
drivers/vhost/vhost.c +1
-0@@ -166,11 +166,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) { struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); +struct vhost_work *work = &poll->work;if (!(key_to_poll (key) & poll->mask))return 0 ;
drivers/vhost/vhost.c static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, · line 181
Quoted 10 lines
drivers/vhost/vhost.c +5
-1-vhost_poll_queue (poll); +if (!poll->dev->use_worker) + work->fn (work); +else +vhost_poll_queue (poll); +return 0 ; }
So a wakeup function wakes up eventfd directly.
What if user supplies e.g. the same eventfd as ioeventfd?
Won't this cause infinite loops?
Signature · 1 line
MST
Re: [PATCH 1/6] vhost: allow device that does not depend on vhost worker
Quoted 25 lines · Michael S. Tsirkin
Quoted 12 lines · Jason Wang
drivers/vhost/vhost.c +1
-0@@ -166,11 +166,16 @@ static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) { struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); +struct vhost_work *work = &poll->work;if (!(key_to_poll (key) & poll->mask))return 0 ;drivers/vhost/vhost.c static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, · line 181
Quoted 10 lines
drivers/vhost/vhost.c +5
-1-vhost_poll_queue (poll); +if (!poll->dev->use_worker) + work->fn (work); +else +vhost_poll_queue (poll); +return 0 ; }So a wakeup function wakes up eventfd directly.
Quoted 3 lines
What if user supplies e.g. the same eventfd as ioeventfd?
Won't this cause infinite loops?
I'm not sure I get this.
This basically calls handle_vq directly when eventfd is woken up. The infinite loops can only happen when handle_vq() tries to write to ioeventfd itslef which should be a bug of the device.
Thanks
Quoted 1 line
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
PATCH 2/6 vhost: use mmgrab() instead of mmget() for non worker device
In reply to Jason WangFor the device that doesn't use vhost worker and use_mm(), mmget() is too heavy weight and it may brings troubles for implementing mmap() support for vDPA device.
This is because, an reference to the address space was held via mm_get() in vhost_dev_set_owner() and an reference to the file was held in mmap(). This means when process exits, the mm can not be released thus we can not release the file.
This patch tries to use mmgrab() instead of mmget(), which allows the address space to be destroy in process exit without releasing the mm structure itself. This is sufficient for vDPA device which pin user pages and does not depend on the address space to work.
--- drivers/vhost/vhost.c | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-)
drivers/vhost/vhost.c +34 -8
@@ -541,6 +541,36 @@ bool vhost_dev_has_owner(struct vhost_dev *dev) }EXPORT_SYMBOL_GPL (vhost_dev_has_owner); +static void vhost_attach_mm (struct vhost_dev *dev) +{ +/* No owner, become one */ +if (dev->use_worker) { + dev->mm =get_task_mm (current); + }else { +/* vDPA device does not use worker thead, so there's +* no need to hold the address space for mm. This help +* to avoid deadlock in the case of mmap() which may +* held the refcnt of the file and depends on release +* method to remove vma. +*/ + dev->mm = current->mm; +mmgrab (dev->mm); + } +} + +static void vhost_detach_mm (struct vhost_dev *dev) +{ +if (!dev->mm) +return ; + +if (dev->use_worker) +mmput (dev->mm); +else +mmdrop (dev->mm); + + dev->mm =NULL ; +} +/* Caller should have device mutex */ long vhost_dev_set_owner (struct vhost_dev *dev) {
@@ -553,8 +583,8 @@ long vhost_dev_set_owner(struct vhost_dev *dev) goto err_mm; } -/* No owner, become one */ - dev->mm =get_task_mm (current); +vhost_attach_mm (dev); + dev->kcov_handle =kcov_common_handle ();if (dev->use_worker) { worker = kthread_create(vhost_worker, dev,
@@ -583,9 +613,7 @@ long vhost_dev_set_owner(struct vhost_dev *dev) dev->worker =NULL ; } err_worker: -if (dev->mm) -mmput (dev->mm); - dev->mm =NULL ; +vhost_detach_mm (dev); dev->kcov_handle =0 ; err_mm:return err;
@@ -682,9 +710,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev) dev->worker =NULL ; dev->kcov_handle =0 ; } -if (dev->mm) -mmput (dev->mm); - dev->mm =NULL ; +vhost_detach_mm (dev); }EXPORT_SYMBOL_GPL (vhost_dev_cleanup);
Signature · 1 line
2.20.1
PATCH 3/6 vdpa: introduce get_vq_notification method
In reply to Jason WangThis patch introduces a new method in the vdpa_config_ops which reports the physical address and the size of the doorbell for a specific virtqueue.
This will be used by the future patches that maps doorbell to userspace.
--- include/linux/vdpa.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
include/linux/vdpa.h +16 -0
@@ -17,6 +17,16 @@ struct vdpa_callback {void *private; }; +/** +* vDPA notification area +* @addr: base address of the notification area +* @size: size of the notification area +*/ +struct vdpa_notification_area { +resource_size_t addr; +resource_size_t size; +}; +/** * vDPA device - representation of a vDPA device * @dev: underlying device
@@ -73,6 +83,10 @@ struct vdpa_device {* @vdev: vdpa device * @idx: virtqueue index * Returns virtqueue state (last_avail_idx) +* @get_vq_notification: Get the notification area for a virtqueue +* @vdev: vdpa device +* @idx: virtqueue index +* Returns the notifcation area * @get_vq_align: Get the virtqueue align requirement * for the device * @vdev: vdpa device
@@ -162,6 +176,8 @@ struct vdpa_config_ops {bool (*get_vq_ready)(struct vdpa_device *vdev,u16 idx);int (*set_vq_state)(struct vdpa_device *vdev,u16 idx,u64 state);u64 (*get_vq_state)(struct vdpa_device *vdev,u16 idx); +struct vdpa_notification_area + (*get_vq_notification)(struct vdpa_device *vdev,u16 idx);/* Device ops */ u32 (*get_vq_align)(struct vdpa_device *vdev);
Signature · 1 line
2.20.1
PATCH 4/6 vhost_vdpa: support doorbell mapping via mmap
In reply to Jason WangCurrently the doorbell is relayed via eventfd which may have significant overhead because of the cost of vmexits or syscall. This patch introduces mmap() based doorbell mapping which can eliminate the overhead caused by vmexit or syscall.
To ease the userspace modeling of the doorbell layout (usually virtio-pci), this patch starts from a doorbell per page model. Vhost-vdpa only support the hardware doorbell that sit at the boundary of a page and does not share the page with other registers.
Doorbell of each virtqueue must be mapped separately, pgoff is the index of the virtqueue. This allows userspace to map a subset of the doorbell which may be useful for the implementation of software assisted virtqueue (control vq) in the future.
--- drivers/vhost/vdpa.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)
drivers/vhost/vdpa.c +59 -0
@@ -15,6 +15,7 @@#include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/mm.h> #include <linux/iommu.h> #include <linux/uuid.h> #include <linux/vdpa.h>
@@ -741,12 +742,70 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)return 0 ; } +static vm_fault_t vhost_vdpa_fault (struct vm_fault *vmf) +{ +struct vhost_vdpa *v = vmf->vma->vm_file->private_data; +struct vdpa_device *vdpa = v->vdpa; +const struct vdpa_config_ops *ops = vdpa->config; +struct vdpa_notification_area notify; +struct vm_area_struct *vma = vmf->vma; +u16 index = vma->vm_pgoff; + + notify = ops->get_vq_notification (vdpa, index); + + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); +if (remap_pfn_range (vma, vmf->address &PAGE_MASK , + notify.addr >>PAGE_SHIFT ,PAGE_SIZE , + vma->vm_page_prot)) +return VM_FAULT_SIGBUS ; + +return VM_FAULT_NOPAGE ; +} + +static const struct vm_operations_struct vhost_vdpa_vm_ops = { + .fault = vhost_vdpa_fault, +}; + +static int vhost_vdpa_mmap (struct file *file,struct vm_area_struct *vma) +{ +struct vhost_vdpa *v = vma->vm_file->private_data; +struct vdpa_device *vdpa = v->vdpa; +const struct vdpa_config_ops *ops = vdpa->config; +struct vdpa_notification_area notify; +int index = vma->vm_pgoff; + +if (vma->vm_end - vma->vm_start !=PAGE_SIZE ) +return -EINVAL ; +if ((vma->vm_flags &VM_SHARED ) ==0 ) +return -EINVAL ; +if (vma->vm_flags &VM_READ ) +return -EINVAL ; +if (index >65535 ) +return -EINVAL ; +if (!ops->get_vq_notification) +return -ENOTSUPP ; + +/* To be safe and easily modelled by userspace, We only +* support the doorbell which sits on the page boundary and +* does not share the page with other registers. +*/ + notify = ops->get_vq_notification (vdpa, index); +if (notify.addr & (PAGE_SIZE -1 )) +return -EINVAL ; +if (vma->vm_end - vma->vm_start != notify.size) +return -ENOTSUPP ; + + vma->vm_ops = &vhost_vdpa_vm_ops; +return 0 ; +} +static const struct file_operations vhost_vdpa_fops = { .owner =THIS_MODULE , .open = vhost_vdpa_open, .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, + .mmap = vhost_vdpa_mmap, .compat_ioctl = compat_ptr_ioctl, };
Signature · 1 line
2.20.1
Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Hi,
Quoted 4 lines · Jason Wang
Currently the doorbell is relayed via eventfd which may have significant overhead because of the cost of vmexits or syscall. This patch introduces
mmap()based doorbell mapping which can eliminate the overhead caused by vmexit or syscall.
Just wondering. I know very little about vdpa. But how is such a "sw doorbell" monitored or observed, if no fault or wmexit etc. Is there some kind of polling used?
Quoted 98 lines
To ease the userspace modeling of the doorbell layout (usually virtio-pci), this patch starts from a doorbell per page model. Vhost-vdpa only support the hardware doorbell that sit at the boundary of a page and does not share the page with other registers.
Doorbell of each virtqueue must be mapped separately, pgoff is the index of the virtqueue. This allows userspace to map a subset of the doorbell which may be useful for the implementation of software assisted virtqueue (control vq) in the future.
Signed-off-by Jason Wang <[email protected]>--- drivers/vhost/vdpa.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)drivers/vhost/vdpa.c +59
-0@@ -15,6 +15,7 @@#include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/mm.h> #include <linux/iommu.h> #include <linux/uuid.h> #include <linux/vdpa.h> @@ -741,12 +742,70 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)return 0 ; } +static vm_fault_t vhost_vdpa_fault (struct vm_fault *vmf) +{ +struct vhost_vdpa *v = vmf->vma->vm_file->private_data; +struct vdpa_device *vdpa = v->vdpa; +const struct vdpa_config_ops *ops = vdpa->config; +struct vdpa_notification_area notify; +struct vm_area_struct *vma = vmf->vma; +u16 index = vma->vm_pgoff; + + notify = ops->get_vq_notification (vdpa, index); + + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); +if (remap_pfn_range (vma, vmf->address &PAGE_MASK , + notify.addr >>PAGE_SHIFT ,PAGE_SIZE , + vma->vm_page_prot)) +return VM_FAULT_SIGBUS ; + +return VM_FAULT_NOPAGE ; +} + +static const struct vm_operations_struct vhost_vdpa_vm_ops = { + .fault = vhost_vdpa_fault, +}; + +static int vhost_vdpa_mmap (struct file *file,struct vm_area_struct *vma) +{ +struct vhost_vdpa *v = vma->vm_file->private_data; +struct vdpa_device *vdpa = v->vdpa; +const struct vdpa_config_ops *ops = vdpa->config; +struct vdpa_notification_area notify; +int index = vma->vm_pgoff; + +if (vma->vm_end - vma->vm_start !=PAGE_SIZE ) +return -EINVAL ; +if ((vma->vm_flags &VM_SHARED ) ==0 ) +return -EINVAL ; +if (vma->vm_flags &VM_READ ) +return -EINVAL ; +if (index >65535 ) +return -EINVAL ; +if (!ops->get_vq_notification) +return -ENOTSUPP ; + +/* To be safe and easily modelled by userspace, We only +* support the doorbell which sits on the page boundary and +* does not share the page with other registers. +*/ + notify = ops->get_vq_notification (vdpa, index); +if (notify.addr & (PAGE_SIZE -1 )) +return -EINVAL ; +if (vma->vm_end - vma->vm_start != notify.size) +return -ENOTSUPP ; + + vma->vm_ops = &vhost_vdpa_vm_ops; +return 0 ; +} +static const struct file_operations vhost_vdpa_fops = { .owner =THIS_MODULE , .open = vhost_vdpa_open, .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, + .mmap = vhost_vdpa_mmap, .compat_ioctl = compat_ptr_ioctl, };
Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 10 lines · Mika Penttilä
Hi,
Quoted 4 lines · Jason Wang
Currently the doorbell is relayed via eventfd which may have significant overhead because of the cost of vmexits or syscall. This patch introduces
mmap()based doorbell mapping which can eliminate the overhead caused by vmexit or syscall.Just wondering. I know very little about vdpa. But how is such a "sw doorbell" monitored or observed, if no fault or wmexit etc. Is there some kind of polling used?
Hi Mika:
It's not a software doorbell. It just allow userspace to map page of hardware doorbell directly into userspace.
Without this, for KVM, it needs to trap the MMIO access of the guest and write to eventfd, for other userspace driver, it needs to write to eventfd. vhost-vDPA's eventfd wakeup function may let the driver to do touch the doorbell.
With this, since the doorbell page is mapped into userspace address space, guest or other userspace driver may write directly to the hardware doorbell register.
Thanks
Quoted 100 lines
Quoted 99 lines
To ease the userspace modeling of the doorbell layout (usually virtio-pci), this patch starts from a doorbell per page model. Vhost-vdpa only support the hardware doorbell that sit at the boundary of a page and does not share the page with other registers.
Doorbell of each virtqueue must be mapped separately, pgoff is the index of the virtqueue. This allows userspace to map a subset of the doorbell which may be useful for the implementation of software assisted virtqueue (control vq) in the future.
Signed-off-by Jason Wang <[email protected]>---
drivers/vhost/vdpa.c| 59 ++++++++++++++++++++++++++++++++++++++++++++1 file changed, 59 insertions(+)drivers/vhost/vdpa.c +58
-0@@ -15,6 +15,7 @@ #include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/mm.h> #include <linux/iommu.h> #include <linux/uuid.h> #include <linux/vdpa.h>@@ -741,12 +742,70 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) return 0; } +static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) +{ + struct vhost_vdpa *v = vmf->vma->vm_file->private_data; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_notification_area notify; + struct vm_area_struct *vma = vmf->vma; + u16 index = vma->vm_pgoff; + + notify = ops->get_vq_notification(vdpa, index); + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (remap_pfn_range(vma, vmf->address & PAGE_MASK, + notify.addr >> PAGE_SHIFT, PAGE_SIZE, + vma->vm_page_prot)) + return VM_FAULT_SIGBUS; + + return VM_FAULT_NOPAGE; +} + +static const struct vm_operations_struct vhost_vdpa_vm_ops = { + .fault = vhost_vdpa_fault, +}; + +static int vhost_vdpa_mmap (struct file *file,struct vm_area_struct *vma) +{ + struct vhost_vdpa *v = vma->vm_file->private_data; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_notification_area notify; + int index = vma->vm_pgoff; + + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + if ((vma->vm_flags & VM_SHARED) == 0) + return -EINVAL; + if (vma->vm_flags & VM_READ) + return -EINVAL; + if (index > 65535) + return -EINVAL; + if (!ops->get_vq_notification) + return -ENOTSUPP; + + /* To be safe and easily modelled by userspace, We only + * support the doorbell which sits on the page boundary and + * does not share the page with other registers. + */ + notify = ops->get_vq_notification(vdpa, index); + if (notify.addr & (PAGE_SIZE - 1)) + return -EINVAL; + if (vma->vm_end - vma->vm_start != notify.size) + return -ENOTSUPP; + + vma->vm_ops = &vhost_vdpa_vm_ops; + return 0; +} + static const struct file_operations vhost_vdpa_fops = { .owner = THIS_MODULE, .open = vhost_vdpa_open, .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, + .mmap = vhost_vdpa_mmap, .compat_ioctl = compat_ptr_ioctl,};
[virtio-dev] Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
In reply to Jason WangGiven the need for 4K doorbell such that QEMU can easily map, ect, and assuming that I have a HW device which exposes 2 VQ's, with a notification area off of BAR3, offset=whatever, notifier_multiplier=4, we don't need to have 2 x 4K pages mapped into the VM for both doorbells do we? The guest driver would ring DB0 at BAR4+offset, and DB1 at BAR4+offset+(4*1).
The 4K per DB is useful how? This allows for QEMU trapping of individual DBs, that can then be used to do what, just forward the DBs via some other scheme - this makes sense for non-HW related Virtio devices I guess. Is this why there is a qemu option?
Rob Miller
[email protected]
(919)721-3339
Quoted 108 lines · Jason Wang
Currently the doorbell is relayed via eventfd which may have significant overhead because of the cost of vmexits or syscall. This patch introduces
mmap()based doorbell mapping which can eliminate the overhead caused by vmexit or syscall.To ease the userspace modeling of the doorbell layout (usually virtio-pci), this patch starts from a doorbell per page model. Vhost-vdpa only support the hardware doorbell that sit at the boundary of a page and does not share the page with other registers.
Doorbell of each virtqueue must be mapped separately, pgoff is the index of the virtqueue. This allows userspace to map a subset of the doorbell which may be useful for the implementation of software assisted virtqueue (control vq) in the future.
Signed-off-by Jason Wang <[email protected]>--- drivers/vhost/vdpa.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)drivers/vhost/vdpa.c +59
-1@@ -15,6 +15,7 @@#include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/mm.h> #include <linux/iommu.h> #include <linux/uuid.h> #include <linux/vdpa.h> @@ -741,12 +742,70 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep)return 0 ; } +static vm_fault_t vhost_vdpa_fault (struct vm_fault *vmf) +{ +struct vhost_vdpa *v = vmf->vma->vm_file->private_data; +struct vdpa_device *vdpa = v->vdpa; +const struct vdpa_config_ops *ops = vdpa->config; +struct vdpa_notification_area notify; +struct vm_area_struct *vma = vmf->vma; +u16 index = vma->vm_pgoff; + + notify = ops->get_vq_notification (vdpa, index); + + vma->vm_page_prot =pgprot_noncached (vma->vm_page_prot); +if (remap_pfn_range (vma, vmf->address &PAGE_MASK , + notify.addr >>PAGE_SHIFT ,PAGE_SIZE , + vma->vm_page_prot)) +return VM_FAULT_SIGBUS ; + +return VM_FAULT_NOPAGE ; +} + +static const struct vm_operations_struct vhost_vdpa_vm_ops = { + .fault = vhost_vdpa_fault, +}; + +static int vhost_vdpa_mmap (struct file *file,struct vm_area_struct *vma) +{ +struct vhost_vdpa *v = vma->vm_file->private_data; +struct vdpa_device *vdpa = v->vdpa; +const struct vdpa_config_ops *ops = vdpa->config; +struct vdpa_notification_area notify; +int index = vma->vm_pgoff; + +if (vma->vm_end - vma->vm_start !=PAGE_SIZE ) +return -EINVAL ; +if ((vma->vm_flags &VM_SHARED ) ==0 ) +return -EINVAL ; +if (vma->vm_flags &VM_READ ) +return -EINVAL ; +if (index >65535 ) +return -EINVAL ; +if (!ops->get_vq_notification) +return -ENOTSUPP ; + +/* To be safe and easily modelled by userspace, We only +* support the doorbell which sits on the page boundary and +* does not share the page with other registers. +*/ + notify = ops->get_vq_notification (vdpa, index); +if (notify.addr & (PAGE_SIZE -1 )) +return -EINVAL ; +if (vma->vm_end - vma->vm_start != notify.size) +return -ENOTSUPP ; + + vma->vm_ops = &vhost_vdpa_vm_ops; +return 0 ; +} +static const struct file_operations vhost_vdpa_fops = { .owner =THIS_MODULE , .open = vhost_vdpa_open, .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, + .mmap = vhost_vdpa_mmap, .compat_ioctl = compat_ptr_ioctl, }; --2.20.1
Re: [virtio-dev] Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 1 line · Rob Miller
Given the need for 4K doorbell such that QEMU can easily map, ect, and assuming that I have a HW device which exposes 2 VQ's, with a notification area off of BAR3, offset=whatever, notifier_multiplier=4, we don't need to have 2 x 4K pages mapped into the VM for both doorbells do we? The guest driver would ring DB0 at BAR4+offset, and DB1 at BAR4+offset+(4*1).
This requires qemu to advertise notifier_multiplier = 4 to guest, it has several implications:
- guest may think control virtqueue sit in the same page then we can't trap access to control virtqueue, qemu may lose the state of the device
- it requires the destination to have exact the same queue_notify_off and notifier_multiplier in order to let migration to work
So for doorbell mapping itself, we can support the mapping of doorbells at once which may occupy one or more pages, but it may bring troubles for other features.
Quoted 2 lines
The 4K per DB is useful how? This allows for QEMU trapping of individual DBs, that can then be used to do what, just forward the DBs via some other scheme - this makes sense for non-HW related Virtio devices I guess. Is this why there is a qemu option?
The page per vq option provides extra flexibility. Note that it's the layout seen by guest, so actually for hardware vendor, the most easy way is to:
- use a single doorbell register for all virtqueues except for the control virtqueue (driver can differ queue index by the value wrote by the driver)
- do not share the page with other registers
In this way, it doesn't require much BAR space and we can still:
- map doorbell separately to guest: guest may see a page per vq doorbell layout but in fact all those pages are mapped into the same page
- trap the control virtqueue, (don't do mmap for control vq)
Thanks
Quoted 109 lines
Rob Miller
[email protected] <mailto:[email protected]> (919)721-3339On Fri, May 29, 2020 at 4:03 AM Jason Wang <[email protected] <mailto:[email protected]>> wrote:
Currently the doorbell is relayed via eventfd which may have significant overhead because of the cost of vmexits or syscall. This patch introduces mmap() based doorbell mapping which can eliminate the overhead caused by vmexit or syscall. To ease the userspace modeling of the doorbell layout (usually virtio-pci), this patch starts from a doorbell per page model. Vhost-vdpa only support the hardware doorbell that sit at the boundary of a page and does not share the page with other registers. Doorbell of each virtqueue must be mapped separately, pgoff is the index of the virtqueue. This allows userspace to map a subset of the doorbell which may be useful for the implementation of software assisted virtqueue (control vq) in the future. Signed-off-by: Jason Wang <[email protected] <mailto:[email protected]>> --- drivers/vhost/vdpa.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 6ff72289f488..bbe23cea139a 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -15,6 +15,7 @@ #include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/mm.h> #include <linux/iommu.h> #include <linux/uuid.h> #include <linux/vdpa.h> @@ -741,12 +742,70 @@ static int vhost_vdpa_release(struct inode *inode, struct file *filep) return 0; } +static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) +{ + struct vhost_vdpa *v = vmf->vma->vm_file->private_data; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_notification_area notify; + struct vm_area_struct *vma = vmf->vma; + u16 index = vma->vm_pgoff; + + notify = ops->get_vq_notification(vdpa, index); + + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + if (remap_pfn_range(vma, vmf->address & PAGE_MASK, + notify.addr >> PAGE_SHIFT, PAGE_SIZE, + vma->vm_page_prot)) + return VM_FAULT_SIGBUS; + + return VM_FAULT_NOPAGE; +} + +static const struct vm_operations_struct vhost_vdpa_vm_ops = { + .fault = vhost_vdpa_fault, +}; + +static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct vhost_vdpa *v = vma->vm_file->private_data; + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + struct vdpa_notification_area notify; + int index = vma->vm_pgoff; + + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + if ((vma->vm_flags & VM_SHARED) == 0) + return -EINVAL; + if (vma->vm_flags & VM_READ) + return -EINVAL; + if (index > 65535) + return -EINVAL; + if (!ops->get_vq_notification) + return -ENOTSUPP; + + /* To be safe and easily modelled by userspace, We only + * support the doorbell which sits on the page boundary and + * does not share the page with other registers. + */ + notify = ops->get_vq_notification(vdpa, index); + if (notify.addr & (PAGE_SIZE - 1)) + return -EINVAL; + if (vma->vm_end - vma->vm_start != notify.size) + return -ENOTSUPP; + + vma->vm_ops = &vhost_vdpa_vm_ops; + return 0; +} + static const struct file_operations vhost_vdpa_fops = { .owner = THIS_MODULE, .open = vhost_vdpa_open, .release = vhost_vdpa_release,
Quoted 9 lines
.write_iter = vhost_vdpa_chr_write_iter,
.unlocked_ioctl = vhost_vdpa_unlocked_ioctl,
+ .mmap = vhost_vdpa_mmap,
.compat_ioctl = compat_ptr_ioctl,
};
--
2.20.1_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
kbuild test robot
build botRe: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
In reply to Jason WangI love your patch! Yet something to improve:
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834 base: https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next config: m68k-randconfig-r011-20200601 (attached as .config) compiler: m68k-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68k
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':
>> drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]
754 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~
>> drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754 drivers/vhost/vdpa.c
742
743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf)
744 {
745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data;
746 struct vdpa_device *vdpa = v->vdpa;
747 const struct vdpa_config_ops *ops = vdpa->config;
748 struct vdpa_notification_area notify;
749 struct vm_area_struct *vma = vmf->vma;
750 u16 index = vma->vm_pgoff;
751
752 notify = ops->get_vq_notification(vdpa, index);
753
> 754 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
755 if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
756 notify.addr >> PAGE_SHIFT, PAGE_SIZE,
757 vma->vm_page_prot))
758 return VM_FAULT_SIGBUS;
759
760 return VM_FAULT_NOPAGE;
761 }
762
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Michael S. Tsirkin
maintainerRe: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 47 lines · kbuild test robot
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please see https://stackoverflow.com/a/37406982]url: https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834 base: https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next config: m68k-randconfig-r011-20200601 (attached as .config) compiler: m68k-linux-gcc (GCC) 9.3.0reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68kIf you fix the issue, kindly add following tag as appropriate
Reported-by kbuild test robot <[email protected]>All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]754 |
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754
drivers/vhost/vdpa.c742 743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) 744 { 745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; 746 struct vdpa_device *vdpa = v->vdpa; 747 const struct vdpa_config_ops *ops = vdpa->config; 748 struct vdpa_notification_area notify; 749 struct vm_area_struct *vma = vmf->vma; 750 u16 index = vma->vm_pgoff; 751 752 notify = ops->get_vq_notification(vdpa, index); 753Quoted 1 line
754
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
Quoted 8 lines
755 if (remap_pfn_range(vma,
vmf->address&PAGE_MASK,
756 notify.addr >>PAGE_SHIFT,PAGE_SIZE,
757vma->vm_page_prot))758 return VM_FAULT_SIGBUS; 759 760 return VM_FAULT_NOPAGE; 761 } 762
Yes well, all this remapping clearly has no chance to work on systems without CONFIG_MMU.
Quoted 3 lines
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 56 lines · Michael S. Tsirkin
Quoted 47 lines · kbuild test robot
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please seehttps://stackoverflow.com/a/37406982]url:https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834 base:https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
config: m68k-randconfig-r011-20200601 (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wgethttps://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68kIf you fix the issue, kindly add following tag as appropriate
Reported-by kbuild test robot<[email protected]>All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]754 |
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754
drivers/vhost/vdpa.c742 743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) 744 { 745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; 746 struct vdpa_device *vdpa = v->vdpa; 747 const struct vdpa_config_ops *ops = vdpa->config; 748 struct vdpa_notification_area notify; 749 struct vm_area_struct *vma = vmf->vma; 750 u16 index = vma->vm_pgoff; 751 752 notify = ops->get_vq_notification(vdpa, index); 753Quoted 1 line
754
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);Quoted 8 lines
755 if (remap_pfn_range(vma, vmf->address & PAGE_MASK, 756 notify.addr >> PAGE_SHIFT, PAGE_SIZE, 757 vma->vm_page_prot)) 758 return VM_FAULT_SIGBUS; 759 760 return VM_FAULT_NOPAGE; 761 } 762
Quoted 2 lines
Yes well, all this remapping clearly has no chance to work on systems without
CONFIG_MMU.
It looks to me mmap can work according to Documentation/nommu-mmap.txt. But I'm not sure it's worth to bother.
Thanks
Quoted 3 lines
Michael S. Tsirkin
maintainerRe: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 58 lines · Jason Wang
Quoted 56 lines · Michael S. Tsirkin
Quoted 55 lines · kbuild test robot
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please seehttps://stackoverflow.com/a/37406982]url:https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834 base:https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
config: m68k-randconfig-r011-20200601 (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wgethttps://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68kIf you fix the issue, kindly add following tag as appropriate
Reported-by kbuild test robot<[email protected]>All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]754 |
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754
drivers/vhost/vdpa.c742 743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) 744 { 745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; 746 struct vdpa_device *vdpa = v->vdpa; 747 const struct vdpa_config_ops *ops = vdpa->config; 748 struct vdpa_notification_area notify; 749 struct vm_area_struct *vma = vmf->vma; 750 u16 index = vma->vm_pgoff; 751 752 notify = ops->get_vq_notification(vdpa, index); 753Quoted 1 line
754
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);755 if (remap_pfn_range(vma, vmf->address & PAGE_MASK, 756 notify.addr >> PAGE_SHIFT, PAGE_SIZE, 757 vma->vm_page_prot)) 758 return VM_FAULT_SIGBUS; 759 760 return VM_FAULT_NOPAGE; 761 } 762
Quoted 8 lines
Quoted 2 lines
Yes well, all this remapping clearly has no chance to work on systems without
CONFIG_MMU.It looks to me mmap can work according to
Documentation/nommu-mmap.txt. But I'm not sure it's worth to bother.Thanks
Well
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t prot)
{
if (addr != (pfn << PAGE_SHIFT))
return -EINVAL;
vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
return 0;
}EXPORT_SYMBOL(remap_pfn_range);
So things aren't going to work if you have a fixed PFN which is the case of the hardware device.
Quoted 4 lines
Quoted 4 lines
Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 77 lines · Michael S. Tsirkin
Quoted 57 lines · Jason Wang
Quoted 56 lines · Michael S. Tsirkin
Quoted 55 lines · kbuild test robot
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please seehttps://stackoverflow.com/a/37406982]url:https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834 base:https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
config: m68k-randconfig-r011-20200601 (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wgethttps://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68kIf you fix the issue, kindly add following tag as appropriate
Reported-by kbuild test robot<[email protected]>All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]754 |
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754
drivers/vhost/vdpa.c742 743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) 744 { 745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; 746 struct vdpa_device *vdpa = v->vdpa; 747 const struct vdpa_config_ops *ops = vdpa->config; 748 struct vdpa_notification_area notify; 749 struct vm_area_struct *vma = vmf->vma; 750 u16 index = vma->vm_pgoff; 751 752 notify = ops->get_vq_notification(vdpa, index); 753Quoted 1 line
754
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);755 if (remap_pfn_range(vma, vmf->address & PAGE_MASK, 756 notify.addr >> PAGE_SHIFT, PAGE_SIZE, 757 vma->vm_page_prot)) 758 return VM_FAULT_SIGBUS; 759 760 return VM_FAULT_NOPAGE; 761 } 762Quoted 7 lines
Quoted 2 lines
Yes well, all this remapping clearly has no chance to work on systems without
CONFIG_MMU.It looks to me mmap can work according to
Documentation/nommu-mmap.txt. But I'm not sure it's worth to bother.Thanks
Well
int remap_pfn_range(struct
vm_area_struct*vma, unsigned long addr,
unsigned long pfn, unsigned long size,pgprot_tprot){ if (addr != (pfn << PAGE_SHIFT)) return -EINVAL; vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; return 0; }
Quoted 5 lines
EXPORT_SYMBOL(remap_pfn_range);So things aren't going to work if you have a fixed PFN which is the case of the hardware device.
Looking at the implementation of some drivers e.g mtd_char. If I read the code correctly, we can do this by providing get_unmapped_area method and use physical address directly.
But start form CONFIG_MMU should be fine. Do you prefer making vhost_vdpa depends on CONFIG_MMU or just fail mmap when CONFIG_MMU is not configured?
Thanks
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Michael S. Tsirkin
maintainerRe: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 86 lines · Jason Wang
Quoted 77 lines · Michael S. Tsirkin
Quoted 57 lines · Jason Wang
Quoted 56 lines · Michael S. Tsirkin
Quoted 55 lines · kbuild test robot
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please seehttps://stackoverflow.com/a/37406982]url:https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834 base:https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
config: m68k-randconfig-r011-20200601 (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wgethttps://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68kIf you fix the issue, kindly add following tag as appropriate
Reported-by kbuild test robot<[email protected]>All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]754 |
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754
drivers/vhost/vdpa.c742 743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) 744 { 745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; 746 struct vdpa_device *vdpa = v->vdpa; 747 const struct vdpa_config_ops *ops = vdpa->config; 748 struct vdpa_notification_area notify; 749 struct vm_area_struct *vma = vmf->vma; 750 u16 index = vma->vm_pgoff; 751 752 notify = ops->get_vq_notification(vdpa, index); 753Quoted 1 line
754
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);755 if (remap_pfn_range(vma, vmf->address & PAGE_MASK, 756 notify.addr >> PAGE_SHIFT, PAGE_SIZE, 757 vma->vm_page_prot)) 758 return VM_FAULT_SIGBUS; 759 760 return VM_FAULT_NOPAGE; 761 } 762Quoted 7 lines
Quoted 2 lines
Yes well, all this remapping clearly has no chance to work on systems without
CONFIG_MMU.It looks to me mmap can work according to
Documentation/nommu-mmap.txt. But I'm not sure it's worth to bother.Thanks
Well
int remap_pfn_range(struct
vm_area_struct*vma, unsigned long addr,
unsigned long pfn, unsigned long size,pgprot_tprot){ if (addr != (pfn << PAGE_SHIFT)) return -EINVAL; vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; return 0; }Quoted 5 lines
EXPORT_SYMBOL(remap_pfn_range);So things aren't going to work if you have a fixed PFN which is the case of the hardware device.
Quoted 8 lines
Looking at the implementation of some drivers e.g
mtd_char. If I read the code correctly, we can do this by providingget_unmapped_areamethod and use physical address directly.But start form
CONFIG_MMUshould be fine. Do you prefer makingvhost_vdpadepends onCONFIG_MMUor just fail mmap whenCONFIG_MMUis not configured?Thanks
I'd just not specify the mmap callback at all.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Re: [PATCH 4/6] vhost_vdpa: support doorbell mapping via mmap
Quoted 86 lines · Michael S. Tsirkin
Quoted 85 lines · Jason Wang
Quoted 75 lines · Michael S. Tsirkin
Quoted 57 lines · Jason Wang
Quoted 56 lines · Michael S. Tsirkin
Quoted 55 lines · kbuild test robot
Hi Jason,
I love your patch! Yet something to improve:
[auto build test ERROR on vhost/linux-next]
[also build test ERROR on linus/master v5.7 next-20200529]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system. BTW, we also suggest to use '--base' option to specify the base tree in git format-patch, please seehttps://stackoverflow.com/a/37406982]url:https://github.com/0day-ci/linux/commits/Jason-Wang/vDPA-doorbell-mapping/20200531-070834
base:https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
config: m68k-randconfig-r011-20200601 (attached as .config)
compiler: m68k-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wgethttps://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=m68kIf you fix the issue, kindly add following tag as appropriate
Reported-by kbuild test robot<[email protected]>All errors (new ones prefixed by >>, old ones prefixed by <<):
drivers/vhost/vdpa.c: In function 'vhost_vdpa_fault':Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: implicit declaration of function 'pgprot_noncached' [-Werror=implicit-function-declaration]754 |
vma->vm_page_prot=pgprot_noncached(vma->vm_page_prot);
| ^~~~~~~~~~~~~~~~Quoted 1 line
Quoted 1 line
drivers/vhost/vdpa.c:754:22: error: incompatible types when assigning to type 'pgprot_t' {aka 'struct <anonymous>'} from type 'int'cc1: some warnings being treated as errors
vim +/pgprot_noncached +754
drivers/vhost/vdpa.c742 743 static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) 744 { 745 struct vhost_vdpa *v = vmf->vma->vm_file->private_data; 746 struct vdpa_device *vdpa = v->vdpa; 747 const struct vdpa_config_ops *ops = vdpa->config; 748 struct vdpa_notification_area notify; 749 struct vm_area_struct *vma = vmf->vma; 750 u16 index = vma->vm_pgoff; 751 752 notify = ops->get_vq_notification(vdpa, index); 753 > 754 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 755 if (remap_pfn_range(vma, vmf->address & PAGE_MASK, 756 notify.addr >> PAGE_SHIFT, PAGE_SIZE, 757 vma->vm_page_prot)) 758 return VM_FAULT_SIGBUS; 759 760 return VM_FAULT_NOPAGE; 761 } 762Quoted 6 lines
Quoted 2 lines
Yes well, all this remapping clearly has no chance to work on systems without
CONFIG_MMU.It looks to me mmap can work according to
Documentation/nommu-mmap.txt. But I'm not sure it's worth to bother.Thanks
Well
int remap_pfn_range(struct
vm_area_struct*vma, unsigned long addr,
unsigned long pfn, unsigned long size,pgprot_tprot){ if (addr != (pfn << PAGE_SHIFT)) return -EINVAL; vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; return 0; }Quoted 5 lines
EXPORT_SYMBOL(remap_pfn_range);So things aren't going to work if you have a fixed PFN which is the case of the hardware device.
Looking at the implementation of some drivers e.g
mtd_char. If I read the code correctly, we can do this by providingget_unmapped_areamethod and use physical address directly.
Quoted 5 lines
Quoted 4 lines
But start form
CONFIG_MMUshould be fine. Do you prefer makingvhost_vdpadepends onCONFIG_MMUor just fail mmap whenCONFIG_MMUis not configured?Thanks
I'd just not specify the mmap callback at all.
Ok, will do.
Thanks
Quoted 1 line
PATCH 5/6 vdpa: introduce virtio pci driver
In reply to Jason WangThis patch introduce a vDPA driver for virtio-pci device. It bridges the virtio-pci control command to the vDPA bus. This will be used for developing new features for both software vDPA framework and hardware vDPA feature.
Compared to vdpa_sim, it has several advantages:
- it's a real device driver which allow us to play with real hardware features
- type independent instead of networking specific
Note that since virtio specification does not support get/restore virtqueue state. So we can not use this driver for VM. This can be addressed by extending the virtio specification.
--- drivers/vdpa/Kconfig | 6 + drivers/vdpa/Makefile | 1 + drivers/vdpa/vp_vdpa/Makefile | 2 + drivers/vdpa/vp_vdpa/vp_vdpa.c | 583 +++++++++++++++++++++++++++++++++ 4 files changed, 592 insertions(+) create mode 100644 drivers/vdpa/vp_vdpa/Makefile create mode 100644 drivers/vdpa/vp_vdpa/vp_vdpa.c
drivers/vdpa/Kconfig +6 -0
@@ -28,4 +28,10 @@ config IFCVF To compile this driver as a module, choose M here: the module will be called ifcvf. +config VP_VDPA + tristate "Virtio PCI bridge vDPA driver" + depends on PCI_MSI + help + This kernel module that bridges virtio PCI device to vDPA bus. + endif # VDPA
drivers/vdpa/Makefile +1 -0
@@ -2,3 +2,4 @@ obj-$(CONFIG_VDPA) += vdpa.o obj-$(CONFIG_VDPA_SIM) += vdpa_sim/ obj-$(CONFIG_IFCVF) += ifcvf/ +obj-$(CONFIG_VP_VDPA) += vp_vdpa/
drivers/vdpa/vp_vdpa/Makefile new file +2 -0
@@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VP_VDPA) += vp_vdpa.o
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +583 -0
583 lines not on this page · raw message
Signature · 6 lines
2.20.1 _______________________________________________ Virtualization mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
drivers/vdpa/vp_vdpa/vp_vdpa.c line 313
Quoted 9 lines · Jason Wang
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +9
-0+static void vp_vdpa_set_vq_ready (struct vdpa_device *vdpa, +u16 qid,bool ready) +{ +struct vp_vdpa *vp_vdpa =vdpa_to_vp (vdpa); + +vp_iowrite16 (qid, &vp_vdpa->common->queue_select); +vp_iowrite16 (ready, &vp_vdpa->common->queue_enable); +} +
Looks like this needs to check and just skip the write if ready == 0, right? Of course vdpa core then insists on calling vp_vdpa_get_vq_ready which will warn. Maybe just drop the check from core, move it to drivers which need it?
...
drivers/vdpa/vp_vdpa/vp_vdpa.c line 567
Quoted 4 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + {PCI_DEVICE (PCI_VENDOR_ID_REDHAT_QUMRANET ,PCI_ANY_ID ) }, + {0 } +};
This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Quoted 17 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +15
-0+MODULE_DEVICE_TABLE (pci, vp_vdpa_id_table); + +static struct pci_driver vp_vdpa_driver = { + .name ="vp-vdpa" , + .id_table = vp_vdpa_id_table, + .probe = vp_vdpa_probe, + .remove = vp_vdpa_remove, +}; + +module_pci_driver (vp_vdpa_driver); + +MODULE_AUTHOR ("Jason Wang <[email protected]>" ); +MODULE_DESCRIPTION ("vp-vdpa" ); +MODULE_LICENSE ("GPL" ); +MODULE_VERSION ("1" );Signature · 1 line
2.20.1
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 15 lines · Michael S. Tsirkin
drivers/vdpa/vp_vdpa/vp_vdpa.c line 313
Quoted 9 lines · Jason Wang
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +9
-0+static void vp_vdpa_set_vq_ready (struct vdpa_device *vdpa, +u16 qid,bool ready) +{ +struct vp_vdpa *vp_vdpa =vdpa_to_vp (vdpa); + +vp_iowrite16 (qid, &vp_vdpa->common->queue_select); +vp_iowrite16 (ready, &vp_vdpa->common->queue_enable); +} +Looks like this needs to check and just skip the write if ready == 0, right? Of course vdpa core then insists on calling
vp_vdpa_get_vq_readywhich will warn. Maybe just drop the check from core, move it to drivers which need it?
Quoted 1 line
...
That may work, but it may cause inconsistent semantic for set_vq_ready if we leave it to the driver.
Quoted 10 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c line 567
Quoted 6 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + {PCI_DEVICE (PCI_VENDOR_ID_REDHAT_QUMRANET ,PCI_ANY_ID ) }, + {0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Right, will do.
Thanks
Quoted 18 lines
Quoted 18 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +15
-0+MODULE_DEVICE_TABLE (pci, vp_vdpa_id_table); + +static struct pci_driver vp_vdpa_driver = { + .name ="vp-vdpa" , + .id_table = vp_vdpa_id_table, + .probe = vp_vdpa_probe, + .remove = vp_vdpa_remove, +}; + +module_pci_driver (vp_vdpa_driver); + +MODULE_AUTHOR ("Jason Wang <[email protected]>" ); +MODULE_DESCRIPTION ("vp-vdpa" ); +MODULE_LICENSE ("GPL" ); +MODULE_VERSION ("1" );Signature · 1 line
2.20.1
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 11 lines · Jason Wang
Quoted 9 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c line 567
Quoted 5 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Quoted 3 lines
Right, will do.
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
How about using a dedicated subsystem vendor id for this?
Thanks
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 13 lines · Jason Wang
Quoted 11 lines · Jason Wang
Quoted 9 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c line 567
Quoted 5 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Quoted 6 lines
Quoted 3 lines
Right, will do.
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
We can bind manually. It's not really for production anyway, so not a big deal imho.
Quoted 3 lines
How about using a dedicated subsystem vendor id for this?
Thanks
If virtio vendor id is used then standard driver is expected to bind, right? Maybe use a dedicated vendor id?
Signature · 6 lines
MST _______________________________________________ Virtualization mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 11 lines · Michael S. Tsirkin
Quoted 10 lines · Jason Wang
Quoted 9 lines · Jason Wang
Quoted 8 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c line 567
Quoted 4 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Right, will do.
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
We can bind manually. It's not really for production anyway, so not a big deal imho.
I think you mean doing it via "new_id", right.
Quoted 6 lines
Quoted 4 lines
How about using a dedicated subsystem vendor id for this?
Thanks
If virtio vendor id is used then standard driver is expected to bind, right? Maybe use a dedicated vendor id?
I meant something like:
static const struct pci_device_id vp_vdpa_id_table[] = {
{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID, VP_TEST_VENDOR_ID, VP_TEST_DEVICE_ID) }, { 0 }
};
Thanks
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 15 lines · Jason Wang
Quoted 13 lines · Michael S. Tsirkin
Quoted 12 lines · Jason Wang
Quoted 11 lines · Jason Wang
Quoted 8 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c line 567
Quoted 4 lines
drivers/vdpa/vp_vdpa/vp_vdpa.c new file +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Right, will do.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
We can bind manually. It's not really for production anyway, so not a big deal imho.
I think you mean doing it via "
new_id", right.
I really meant driver_override. This is what people have been using with pci-stub for years now.
Quoted 11 lines
Quoted 7 lines
Quoted 5 lines
How about using a dedicated subsystem vendor id for this?
Thanks
If virtio vendor id is used then standard driver is expected to bind, right? Maybe use a dedicated vendor id?
I meant something like:
Quoted 8 lines
static const struct
pci_device_idvp_vdpa_id_table[] = { {PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID, VP_TEST_VENDOR_ID, VP_TEST_DEVICE_ID)},
{ 0 }
};Thanks
Then regular virtio will still bind to it. It has
drivers/virtio/virtio_pci_common.c: { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) },
Signature · 1 line
MST
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 14 lines · Michael S. Tsirkin
Quoted 13 lines · Jason Wang
Quoted 12 lines · Michael S. Tsirkin
Quoted 11 lines · Jason Wang
Quoted 10 lines · Jason Wang
Quoted 8 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Right, will do.
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
Quoted 1 line
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
We can bind manually. It's not really for production anyway, so not a big deal imho.
I think you mean doing it via "
new_id", right.I really meant
driver_override. This is what people have been using with pci-stub for years now.
Do you want me to implement "driver_overrid" in this series, or a NULL id_table is sufficient?
Quoted 15 lines
Quoted 15 lines
Quoted 6 lines
Quoted 4 lines
How about using a dedicated subsystem vendor id for this?
Thanks
If virtio vendor id is used then standard driver is expected to bind, right? Maybe use a dedicated vendor id?
I meant something like:
static const struct
pci_device_idvp_vdpa_id_table[] = {
{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,PCI_ANY_ID,VP_TEST_VENDOR_ID,VP_TEST_DEVICE_ID) },
{ 0 }
};
Quoted 7 lines
Quoted 2 lines
Thanks
Then regular virtio will still bind to it. It has
drivers/virtio/virtio_pci_common.c: {PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID)},
IFCVF use this to avoid the binding to regular virtio device. Looking at pci_match_one_device() it checks both subvendor and subdevice there.
Thanks
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 21 lines · Jason Wang
Quoted 19 lines · Michael S. Tsirkin
Quoted 18 lines · Jason Wang
Quoted 16 lines · Michael S. Tsirkin
Quoted 9 lines · Jason Wang
Quoted 8 lines · Jason Wang
Quoted 8 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Quoted 4 lines
Quoted 3 lines
Right, will do.
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
We can bind manually. It's not really for production anyway, so not a big deal imho.
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
I think you mean doing it via "
new_id", right.I really meant
driver_override. This is what people have been using with pci-stub for years now.Do you want me to implement "
driver_overrid" in this series, or a NULLid_tableis sufficient?
Doesn't the pci subsystem create driver_override for all devices on the pci bus?
Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 7 lines
Quoted 5 lines
How about using a dedicated subsystem vendor id for this?
Thanks
If virtio vendor id is used then standard driver is expected to bind, right? Maybe use a dedicated vendor id?
I meant something like:
static const struct
pci_device_idvp_vdpa_id_table[] = {
{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,PCI_ANY_ID,VP_TEST_VENDOR_ID,VP_TEST_DEVICE_ID) },
{ 0 }
};Thanks
Quoted 7 lines
Quoted 5 lines
Then regular virtio will still bind to it. It has
drivers/virtio/virtio_pci_common.c: {PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID)},IFCVF use this to avoid the binding to regular virtio device.
Ow. Indeed:
#define IFCVF_VENDOR_ID 0x1AF4
Which is of course not an IFCVF vendor id, it's the Red Hat vendor ID.
I missed that.
Does it actually work if you bind a virtio driver to it? I'm guessing no otherwise they wouldn't need IFC driver, right?
Quoted 4 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.
I guess we need to blacklist IFC in virtio pci probe code. Ugh.
Signature · 1 line
MST
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 21 lines · Michael S. Tsirkin
Quoted 20 lines · Jason Wang
Quoted 19 lines · Michael S. Tsirkin
Quoted 18 lines · Jason Wang
Quoted 10 lines · Michael S. Tsirkin
Quoted 9 lines · Jason Wang
Quoted 8 lines · Jason Wang
Quoted 8 lines
Review
Quoted 4 lines
(quoted diff) +4
-0+static const struct pci_device_id vp_vdpa_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) }, + { 0 } +};This looks like it'll create a mess with either virtio pci or vdpa being loaded at random. Maybe just don't specify any IDs for now. Down the road we could get a distinct vendor ID or a range of device IDs for this.
Quoted 6 lines
Quoted 4 lines
Quoted 3 lines
Right, will do.
Thanks
Rethink about this. If we don't specify any ID, the binding won't work.
We can bind manually. It's not really for production anyway, so not a big deal imho.
I think you mean doing it via "
new_id", right.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I really meant
driver_override. This is what people have been using with pci-stub for years now.Do you want me to implement "
driver_overrid" in this series, or a NULLid_tableis sufficient?Doesn't the pci subsystem create
driver_overridefor all devices on the pci bus?
Yes, I miss this.
Quoted 24 lines
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Quoted 5 lines
Quoted 3 lines
How about using a dedicated subsystem vendor id for this?
Thanks
If virtio vendor id is used then standard driver is expected to bind, right? Maybe use a dedicated vendor id?
I meant something like:
static const struct
pci_device_idvp_vdpa_id_table[] = {
{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,PCI_ANY_ID,VP_TEST_VENDOR_ID,VP_TEST_DEVICE_ID) },
{ 0 }
};Quoted 8 lines
Quoted 7 lines
Quoted 2 lines
Thanks
Then regular virtio will still bind to it. It has
drivers/virtio/virtio_pci_common.c: {PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID)},IFCVF use this to avoid the binding to regular virtio device.
Ow. Indeed:
Quoted 7 lines
#define
IFCVF_VENDOR_ID0x1AF4Which is of course not an IFCVF vendor id, it's the Red Hat vendor ID.
I missed that.
Does it actually work if you bind a virtio driver to it?
It works.
Quoted 2 lines
I'm guessing no otherwise they wouldn't need IFC driver, right?
Looking at the driver, they used a dedicated bar for dealing with virtqueue state save/restore. It
Quoted 10 lines
Quoted 6 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.
I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Thanks
Quoted 6 lines
I guess we need to blacklist IFC in virtio pci probe code. Ugh.
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 11 lines · Jason Wang
Quoted 9 lines
Quoted 5 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.
Quoted 4 lines
I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Thanks
Right but the reverse can happen: virtio-net can bind to IFCVF first.
Signature · 1 line
MST
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 9 lines · Michael S. Tsirkin
Quoted 8 lines · Jason Wang
Quoted 7 lines
Quoted 4 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.
Quoted 5 lines
Quoted 4 lines
I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Thanks
Right but the reverse can happen: virtio-net can bind to IFCVF first.
That's kind of expected. The PF is expected to be bound to virtio-pci to create VF via sysfs.
Thanks
Quoted 1 line
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 14 lines · Jason Wang
Quoted 12 lines · Michael S. Tsirkin
Quoted 11 lines · Jason Wang
Quoted 7 lines
Quoted 4 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Quoted 11 lines
Quoted 2 lines
Quoted 1 line
Thanks
Right but the reverse can happen: virtio-net can bind to IFCVF first.
That's kind of expected. The PF is expected to be bound to virtio-pci to create VF via sysfs.
Thanks
Once VFs are created, don't we want IFCVF to bind rather than virtio-pci?
Signature · 1 line
MST
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 19 lines · Michael S. Tsirkin
Quoted 18 lines · Jason Wang
Quoted 5 lines · Michael S. Tsirkin
Quoted 4 lines · Jason Wang
Quoted 4 lines
Quoted 4 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
Quoted 8 lines
Quoted 7 lines
Quoted 3 lines
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Thanks
Right but the reverse can happen: virtio-net can bind to IFCVF first.
That's kind of expected. The PF is expected to be bound to virtio-pci to create VF via sysfs.
Quoted 6 lines
Quoted 4 lines
Thanks
Once VFs are created, don't we want IFCVF to bind rather than virtio-pci?
Yes, but for PF we need virtio-pci.
Thanks
Quoted 1 line
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 25 lines · Jason Wang
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Jason Wang
Quoted 5 lines · Michael S. Tsirkin
Quoted 4 lines · Jason Wang
Quoted 4 lines
Quoted 4 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
Quoted 8 lines
Quoted 7 lines
Quoted 3 lines
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Thanks
Right but the reverse can happen: virtio-net can bind to IFCVF first.
That's kind of expected. The PF is expected to be bound to virtio-pci to create VF via sysfs.
Thanks
Quoted 8 lines
Quoted 2 lines
Once VFs are created, don't we want IFCVF to bind rather than virtio-pci?
Yes, but for PF we need virtio-pci.
Thanks
(Ab)using the driver_data field for this is an option. What do you think?
Signature · 6 lines
MST _______________________________________________ Virtualization mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 27 lines · Michael S. Tsirkin
Quoted 26 lines · Jason Wang
Quoted 24 lines · Michael S. Tsirkin
Quoted 12 lines · Jason Wang
Quoted 11 lines · Michael S. Tsirkin
Quoted 10 lines · Jason Wang
Quoted 7 lines
Quoted 4 lines
Looking at
pci_match_one_device()it checks both subvendor and subdevice there.Thanks
But IIUC there is no guarantee that driver with a specific subvendor matches in presence of a generic one.
So either IFC or virtio pci can win, whichever binds first.I'm not sure I get there. But I try manually bind IFCVF to qemu's virtio-net-pci, and it fails.
Quoted 9 lines
Quoted 2 lines
Quoted 1 line
Thanks
Right but the reverse can happen: virtio-net can bind to IFCVF first.
That's kind of expected. The PF is expected to be bound to virtio-pci to create VF via sysfs.
Thanks
Once VFs are created, don't we want IFCVF to bind rather than virtio-pci?
Quoted 6 lines
Quoted 4 lines
Yes, but for PF we need virtio-pci.
Thanks
(Ab)using the
driver_datafield for this is an option. What do you think?
Maybe you can elaborate more on this idea?
Thanks
Quoted 1 line
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
In reply to Jason WangQuoted 3 lines · Jason Wang
Note that since virtio specification does not support get/restore virtqueue state. So we can not use this driver for VM. This can be addressed by extending the virtio specification.
Looks like exactly the kind of hardware limitation VDPA is supposed to paper over within guest. So I suggest we use this as
a litmus test, and find ways for VDPA to handle this without spec changes.
Signature · 1 line
MST
Re: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 8 lines · Michael S. Tsirkin
Quoted 3 lines · Jason Wang
Note that since virtio specification does not support get/restore virtqueue state. So we can not use this driver for VM. This can be addressed by extending the virtio specification.
Looks like exactly the kind of hardware limitation VDPA is supposed to paper over within guest. So I suggest we use this as
a litmus test, and find ways for VDPA to handle this without spec changes.
Yes, and just to confirm, do you think it's beneficial to extend virtio specification to support state get/set?
Thanks
Quoted 1 line
Michael S. Tsirkin
maintainerRe: [PATCH 5/6] vdpa: introduce virtio pci driver
Quoted 12 lines · Jason Wang
Quoted 8 lines · Michael S. Tsirkin
Quoted 3 lines · Jason Wang
Note that since virtio specification does not support get/restore virtqueue state. So we can not use this driver for VM. This can be addressed by extending the virtio specification.
Looks like exactly the kind of hardware limitation VDPA is supposed to paper over within guest. So I suggest we use this as
a litmus test, and find ways for VDPA to handle this without spec changes.
Quoted 4 lines
Yes, and just to confirm, do you think it's beneficial to extend virtio specification to support state get/set?
Thanks
Let's leave that for another day. For now vdpa should be flexible enough to work on spec compliant VMs.
Quoted 2 lines
Quoted 2 lines
PATCH 6/6 vdpa: vp_vdpa: report doorbell location
In reply to Jason WangThis patch adds support for reporting doorbell location in vp_vdpa driver. The doorbell mapping could be then enabled by e.g launching qemu's virtio-net-pci device with page-per-vq enabled.
--- drivers/vdpa/vp_vdpa/vp_vdpa.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-)
drivers/vdpa/vp_vdpa/vp_vdpa.c +25 -4
@@ -30,6 +30,7 @@struct vp_vring {void __iomem *notify;char msix_name[256 ]; +resource_size_t notify_pa;struct vdpa_callback cb;int irq; };
@@ -136,7 +137,8 @@ static int find_capability(struct pci_dev *dev, u8 cfg_type,return 0 ; } -static void __iomem *map_capability (struct vp_vdpa *vp_vdpa,int off) +static void __iomem *map_capability (struct vp_vdpa *vp_vdpa,int off, +resource_size_t *pa) {struct pci_dev *pdev = vp_vdpa->pdev;u32 offset;
@@ -149,6 +151,9 @@ static void __iomem *map_capability(struct vp_vdpa *vp_vdpa, int off) off + offsetof(struct virtio_pci_cap , offset), &offset); +if (pa) + *pa =pci_resource_start (pdev, bar) + offset; +return vp_vdpa->base[bar] + offset; }
@@ -283,6 +288,18 @@ static u64 vp_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid)return 0 ; } +static struct vdpa_notification_area +vp_vdpa_get_vq_notification (struct vdpa_device *vdpa,u16 qid) +{ +struct vp_vdpa *vp_vdpa =vdpa_to_vp (vdpa); +struct vdpa_notification_area notify; + + notify.addr = vp_vdpa->vring[qid].notify_pa; + notify.size = vp_vdpa->notify_off_multiplier; + +return notify; +} +static int vp_vdpa_set_vq_state (struct vdpa_device *vdpa,u16 qid,u64 num) {
@@ -423,6 +440,7 @@ static const struct vdpa_config_ops vp_vdpa_ops = { .set_status = vp_vdpa_set_status, .get_vq_num_max = vp_vdpa_get_vq_num_max, .get_vq_state = vp_vdpa_get_vq_state, + .get_vq_notification = vp_vdpa_get_vq_notification, .set_vq_state = vp_vdpa_set_vq_state, .set_vq_cb = vp_vdpa_set_vq_cb, .set_vq_ready = vp_vdpa_set_vq_ready,
@@ -445,6 +463,7 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)struct vp_vdpa *vp_vdpa;int common, notify, device, ret, i;struct virtio_device_id virtio_id; +resource_size_t notify_pa;u16 notify_off;/* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */
@@ -525,9 +544,9 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)if (ret)dev_warn (&pdev->dev,"Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n" ); - vp_vdpa->device =map_capability (vp_vdpa, device); - vp_vdpa->notify =map_capability (vp_vdpa, notify); - vp_vdpa->common =map_capability (vp_vdpa, common); + vp_vdpa->device =map_capability (vp_vdpa, device,NULL ); + vp_vdpa->notify =map_capability (vp_vdpa, notify, ¬ify_pa); + vp_vdpa->common =map_capability (vp_vdpa, common,NULL ); vp_vdpa->id = virtio_id; ret =vdpa_register_device (&vp_vdpa->vdpa);
@@ -545,6 +564,8 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) vp_vdpa->vring[i].irq =-1 ; vp_vdpa->vring[i].notify = vp_vdpa->notify + notify_off * vp_vdpa->notify_off_multiplier; + vp_vdpa->vring[i].notify_pa = notify_pa + + notify_off * vp_vdpa->notify_off_multiplier; }return 0 ;
Signature · 1 line
2.20.1