virtio_mmio: hypervisor specific interfaces for MMIO
cover virtio_mmio: hypervisor specific interfaces for MMIO
The Type-1 hypervisor we are dealing with does not allow for MMIO transport.
[1] summarizes some of the problems we have in making virtio work on such hypervisors. This patch proposes a solution for transport problem viz how we can do config space IO on such a hypervisor. Hypervisor specific methods introduced allows for seamless IO of config space.
This patch is meant to seek comments. If its considered to be in right direction, will work on making it more complete and send the next version!
1. https://lkml.org/lkml/2020/4/28/427
Srivatsa Vaddagiri (1):
virtio: Introduce MMIO ops
drivers/virtio/virtio_mmio.c | 131 ++++++++++++++++++++++++++----------------- include/linux/virtio.h | 14 +++++ 2 files changed, 94 insertions(+), 51 deletions(-)
Signature · 5 lines
2.7.4 -- QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
PATCH 1/1 virtio: Introduce MMIO ops
Some hypervisors may not support MMIO transport i.e trap config space access and have it be handled by backend driver. They may allow other ways to interact with backend such as message-queue or doorbell API. This patch allows for hypervisor specific methods for config space IO.
--- drivers/virtio/virtio_mmio.c | 131 ++++++++++++++++++++++++++----------------- include/linux/virtio.h | 14 +++++ 2 files changed, 94 insertions(+), 51 deletions(-)
drivers/virtio/virtio_mmio.c +80 -51
@@ -100,7 +100,35 @@ struct virtio_mmio_vq_info {struct list_head node; }; +#ifdef CONFIG_VIRTIO_MMIO_OPS +static struct virtio_mmio_ops *mmio_ops; + +#define virtio_readb (a) mmio_ops->mmio_readl((a)) +#define virtio_readw (a) mmio_ops->mmio_readl((a)) +#define virtio_readl (a) mmio_ops->mmio_readl((a)) +#define virtio_writeb (val, a) mmio_ops->mmio_writeb((val), (a)) +#define virtio_writew (val, a) mmio_ops->mmio_writew((val), (a)) +#define virtio_writel (val, a) mmio_ops->mmio_writel((val), (a)) + +int register_virtio_mmio_ops (struct virtio_mmio_ops *ops) +{ +pr_info ("Registered %s as mmio ops\n" , ops->name); + mmio_ops = ops; + +return 0 ; +} + +#else /* CONFIG_VIRTIO_MMIO_OPS */ + +#define virtio_readb (a) readb((a)) +#define virtio_readw (a) readw((a)) +#define virtio_readl (a) readl((a)) +#define virtio_writeb (val, a) writeb((val), (a)) +#define virtio_writew (val, a) writew((val), (a)) +#define virtio_writel (val, a) writel((val), (a)) + +#endif /* CONFIG_VIRTIO_MMIO_OPS */ /* Configuration interface */
@@ -109,12 +137,12 @@ static u64 vm_get_features(struct virtio_device *vdev)struct virtio_mmio_device *vm_dev =to_virtio_mmio_device (vdev);u64 features; -writel (1 , vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES_SEL ); - features =readl (vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES ); +virtio_writel (1 , vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES_SEL ); + features =virtio_readl (vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES ); features <<=32 ; -writel (0 , vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES_SEL ); - features |=readl (vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES ); +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES_SEL ); + features |=virtio_readl (vm_dev->base +VIRTIO_MMIO_DEVICE_FEATURES );return features; }
@@ -133,12 +161,12 @@ static int vm_finalize_features(struct virtio_device *vdev)return -EINVAL ; } -writel (1 , vm_dev->base +VIRTIO_MMIO_DRIVER_FEATURES_SEL ); -writel ((u32)(vdev->features >>32 ), +virtio_writel (1 , vm_dev->base +VIRTIO_MMIO_DRIVER_FEATURES_SEL ); +virtio_writel ((u32)(vdev->features >>32 ), vm_dev->base +VIRTIO_MMIO_DRIVER_FEATURES ); -writel (0 , vm_dev->base +VIRTIO_MMIO_DRIVER_FEATURES_SEL ); -writel ((u32 )vdev->features, +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_DRIVER_FEATURES_SEL ); +virtio_writel ((u32 )vdev->features, vm_dev->base +VIRTIO_MMIO_DRIVER_FEATURES );return 0 ;
@@ -158,25 +186,25 @@ static void vm_get(struct virtio_device *vdev, unsigned offset,int i;for (i =0 ; i < len; i++) - ptr[i] =readb (base + offset + i); + ptr[i] =virtio_readb (base + offset + i);return ; }switch (len) {case 1 : - b =readb (base + offset); + b =virtio_readb (base + offset);memcpy (buf, &b,sizeof b);break ;case 2 : - w =cpu_to_le16 (readw (base + offset)); + w =cpu_to_le16 (virtio_readw (base + offset));memcpy (buf, &w,sizeof w);break ;case 4 : - l =cpu_to_le32 (readl (base + offset)); + l =cpu_to_le32 (virtio_readl (base + offset));memcpy (buf, &l,sizeof l);break ;case 8 : - l =cpu_to_le32 (readl (base + offset)); + l =cpu_to_le32 (virtio_readl (base + offset));memcpy (buf, &l,sizeof l); l =cpu_to_le32 (ioread32 (base + offset +sizeof l));memcpy (buf +sizeof l, &l,sizeof l);
@@ -200,7 +228,7 @@ static void vm_set(struct virtio_device *vdev, unsigned offset,int i;for (i =0 ; i < len; i++) -writeb (ptr[i], base + offset + i); +virtio_writeb (ptr[i], base + offset + i);return ; }
@@ -208,21 +236,21 @@ static void vm_set(struct virtio_device *vdev, unsigned offset,switch (len) {case 1 :memcpy (&b, buf,sizeof b); -writeb (b, base + offset); +virtio_writeb (b, base + offset);break ;case 2 :memcpy (&w, buf,sizeof w); -writew (le16_to_cpu (w), base + offset); +virtio_writew (le16_to_cpu (w), base + offset);break ;case 4 :memcpy (&l, buf,sizeof l); -writel (le32_to_cpu (l), base + offset); +virtio_writel (le32_to_cpu (l), base + offset);break ;case 8 :memcpy (&l, buf,sizeof l); -writel (le32_to_cpu (l), base + offset); +virtio_writel (le32_to_cpu (l), base + offset);memcpy (&l, buf +sizeof l,sizeof l); -writel (le32_to_cpu (l), base + offset +sizeof l); +virtio_writel (le32_to_cpu (l), base + offset +sizeof l);break ;default :BUG ();
@@ -236,14 +264,14 @@ static u32 vm_generation(struct virtio_device *vdev)if (vm_dev->version ==1 )return 0 ;else -return readl (vm_dev->base +VIRTIO_MMIO_CONFIG_GENERATION ); +return virtio_readl (vm_dev->base +VIRTIO_MMIO_CONFIG_GENERATION ); }static u8 vm_get_status (struct virtio_device *vdev) {struct virtio_mmio_device *vm_dev =to_virtio_mmio_device (vdev); -return readl (vm_dev->base +VIRTIO_MMIO_STATUS ) &0xff ; +return virtio_readl (vm_dev->base +VIRTIO_MMIO_STATUS ) &0xff ; }static void vm_set_status (struct virtio_device *vdev,u8 status)
@@ -253,7 +281,7 @@ static void vm_set_status(struct virtio_device *vdev, u8 status)/* We should never be setting status to 0. */ BUG_ON (status ==0 ); -writel (status, vm_dev->base +VIRTIO_MMIO_STATUS ); +virtio_writel (status, vm_dev->base +VIRTIO_MMIO_STATUS ); }static void vm_reset (struct virtio_device *vdev)
@@ -261,7 +289,7 @@ static void vm_reset(struct virtio_device *vdev)struct virtio_mmio_device *vm_dev =to_virtio_mmio_device (vdev);/* 0 status means a reset. */ -writel (0 , vm_dev->base +VIRTIO_MMIO_STATUS ); +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_STATUS ); }
@@ -275,7 +303,7 @@ static bool vm_notify(struct virtqueue *vq)/* We write the queue's selector into the notification register to * signal the other end */ -writel (vq->index, vm_dev->base +VIRTIO_MMIO_QUEUE_NOTIFY ); +virtio_writel (vq->index, vm_dev->base +VIRTIO_MMIO_QUEUE_NOTIFY );return true; }
@@ -289,8 +317,8 @@ static irqreturn_t vm_interrupt(int irq, void *opaque)irqreturn_t ret =IRQ_NONE ;/* Read and acknowledge interrupts */ - status =readl (vm_dev->base +VIRTIO_MMIO_INTERRUPT_STATUS ); -writel (status, vm_dev->base +VIRTIO_MMIO_INTERRUPT_ACK ); + status =virtio_readl (vm_dev->base +VIRTIO_MMIO_INTERRUPT_STATUS ); +virtio_writel (status, vm_dev->base +VIRTIO_MMIO_INTERRUPT_ACK );if (unlikely (status &VIRTIO_MMIO_INT_CONFIG )) {virtio_config_changed (&vm_dev->vdev);
@@ -321,12 +349,12 @@ static void vm_del_vq(struct virtqueue *vq)spin_unlock_irqrestore (&vm_dev->lock, flags);/* Select and deactivate the queue */ -writel (index, vm_dev->base +VIRTIO_MMIO_QUEUE_SEL ); +virtio_writel (index, vm_dev->base +VIRTIO_MMIO_QUEUE_SEL );if (vm_dev->version ==1 ) { -writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_PFN ); +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_PFN ); }else { -writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_READY ); -WARN_ON (readl (vm_dev->base +VIRTIO_MMIO_QUEUE_READY )); +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_READY ); +WARN_ON (virtio_readl (vm_dev->base +VIRTIO_MMIO_QUEUE_READY )); }vring_del_virtqueue (vq);
@@ -360,10 +388,10 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,return NULL ;/* Select the queue we're interested in */ -writel (index, vm_dev->base +VIRTIO_MMIO_QUEUE_SEL ); +virtio_writel (index, vm_dev->base +VIRTIO_MMIO_QUEUE_SEL );/* Queue shouldn't already be set up. */ -if (readl (vm_dev->base + (vm_dev->version ==1 ? +if (virtio_readl (vm_dev->base + (vm_dev->version ==1 ?VIRTIO_MMIO_QUEUE_PFN :VIRTIO_MMIO_QUEUE_READY ))) { err = -ENOENT ; goto error_available;
@@ -376,7 +404,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, goto error_kmalloc; } - num =readl (vm_dev->base +VIRTIO_MMIO_QUEUE_NUM_MAX ); + num =virtio_readl (vm_dev->base +VIRTIO_MMIO_QUEUE_NUM_MAX );if (num ==0 ) { err = -ENOENT ; goto error_new_virtqueue;
@@ -391,7 +419,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, }/* Activate the queue */ -writel (virtqueue_get_vring_size (vq), vm_dev->base +VIRTIO_MMIO_QUEUE_NUM ); +virtio_writel (virtqueue_get_vring_size (vq), vm_dev->base +VIRTIO_MMIO_QUEUE_NUM );if (vm_dev->version ==1 ) {u64 q_pfn =virtqueue_get_desc_addr (vq) >>PAGE_SHIFT ;
@@ -408,27 +436,27 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, goto error_bad_pfn; } -writel (PAGE_SIZE , vm_dev->base +VIRTIO_MMIO_QUEUE_ALIGN ); -writel (q_pfn, vm_dev->base +VIRTIO_MMIO_QUEUE_PFN ); +virtio_writel (PAGE_SIZE , vm_dev->base +VIRTIO_MMIO_QUEUE_ALIGN ); +virtio_writel (q_pfn, vm_dev->base +VIRTIO_MMIO_QUEUE_PFN ); }else {u64 addr; addr =virtqueue_get_desc_addr (vq); -writel ((u32 )addr, vm_dev->base +VIRTIO_MMIO_QUEUE_DESC_LOW ); -writel ((u32)(addr >>32 ), +virtio_writel ((u32 )addr, vm_dev->base +VIRTIO_MMIO_QUEUE_DESC_LOW ); +virtio_writel ((u32)(addr >>32 ), vm_dev->base +VIRTIO_MMIO_QUEUE_DESC_HIGH ); addr =virtqueue_get_avail_addr (vq); -writel ((u32 )addr, vm_dev->base +VIRTIO_MMIO_QUEUE_AVAIL_LOW ); -writel ((u32)(addr >>32 ), +virtio_writel ((u32 )addr, vm_dev->base +VIRTIO_MMIO_QUEUE_AVAIL_LOW ); +virtio_writel ((u32)(addr >>32 ), vm_dev->base +VIRTIO_MMIO_QUEUE_AVAIL_HIGH ); addr =virtqueue_get_used_addr (vq); -writel ((u32 )addr, vm_dev->base +VIRTIO_MMIO_QUEUE_USED_LOW ); -writel ((u32)(addr >>32 ), +virtio_writel ((u32 )addr, vm_dev->base +VIRTIO_MMIO_QUEUE_USED_LOW ); +virtio_writel ((u32)(addr >>32 ), vm_dev->base +VIRTIO_MMIO_QUEUE_USED_HIGH ); -writel (1 , vm_dev->base +VIRTIO_MMIO_QUEUE_READY ); +virtio_writel (1 , vm_dev->base +VIRTIO_MMIO_QUEUE_READY ); } vq->priv = info;
@@ -444,10 +472,10 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,vring_del_virtqueue (vq); error_new_virtqueue:if (vm_dev->version ==1 ) { -writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_PFN ); +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_PFN ); }else { -writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_READY ); -WARN_ON (readl (vm_dev->base +VIRTIO_MMIO_QUEUE_READY )); +virtio_writel (0 , vm_dev->base +VIRTIO_MMIO_QUEUE_READY ); +WARN_ON (virtio_readl (vm_dev->base +VIRTIO_MMIO_QUEUE_READY )); }kfree (info); error_kmalloc:
@@ -514,6 +542,7 @@ static const struct virtio_config_ops virtio_mmio_config_ops = { .bus_name = vm_bus_name, }; +static const struct virtio_mmio_ops virtio_mmio_default_ops;static void virtio_mmio_release_dev (struct device *_d) {
@@ -550,21 +579,21 @@ static int virtio_mmio_probe(struct platform_device *pdev)return PTR_ERR (vm_dev->base);/* Check magic value */ - magic =readl (vm_dev->base +VIRTIO_MMIO_MAGIC_VALUE ); + magic =virtio_readl (vm_dev->base +VIRTIO_MMIO_MAGIC_VALUE );if (magic != ('v' |'i' <<8 |'r' <<16 |'t' <<24 )) {dev_warn (&pdev->dev,"Wrong magic value 0x%08lx!\n" , magic);return -ENODEV ; }/* Check device version */ - vm_dev->version =readl (vm_dev->base +VIRTIO_MMIO_VERSION ); + vm_dev->version =virtio_readl (vm_dev->base +VIRTIO_MMIO_VERSION );if (vm_dev->version <1 || vm_dev->version >2 ) {dev_err (&pdev->dev,"Version %ld not supported!\n" , vm_dev->version);return -ENXIO ; } - vm_dev->vdev.id.device =readl (vm_dev->base +VIRTIO_MMIO_DEVICE_ID ); + vm_dev->vdev.id.device =virtio_readl (vm_dev->base +VIRTIO_MMIO_DEVICE_ID );if (vm_dev->vdev.id.device ==0 ) {/* * virtio-mmio device with an ID 0 is a (dummy) placeholder
@@ -572,10 +601,10 @@ static int virtio_mmio_probe(struct platform_device *pdev)*/ return -ENODEV ; } - vm_dev->vdev.id.vendor =readl (vm_dev->base +VIRTIO_MMIO_VENDOR_ID ); + vm_dev->vdev.id.vendor =virtio_readl (vm_dev->base +VIRTIO_MMIO_VENDOR_ID );if (vm_dev->version ==1 ) { -writel (PAGE_SIZE , vm_dev->base +VIRTIO_MMIO_GUEST_PAGE_SIZE ); +virtio_writel (PAGE_SIZE , vm_dev->base +VIRTIO_MMIO_GUEST_PAGE_SIZE ); rc =dma_set_mask (&pdev->dev,DMA_BIT_MASK (64 ));/*
include/linux/virtio.h +14 -0
@@ -119,6 +119,20 @@ struct virtio_device {void *priv; }; +#define MAX_MMIO_NAME_SIZE 32 + +struct virtio_mmio_ops { +char name[MAX_MMIO_NAME_SIZE ]; +u8 (*mmio_readb)(const volatile void __iomem *addr); +u16 (*mmio_readw)(const volatile void __iomem *addr); +u32 (*mmio_readl)(const volatile void __iomem *addr); +void (*mmio_writeb)(u8 val,volatile void __iomem *addr); +void (*mmio_writew)(u16 val,volatile void __iomem *addr); +void (*mmio_writel)(u32 val,volatile void __iomem *addr); +}; + +extern int register_virtio_mmio_ops (struct virtio_mmio_ops *ops); +static inline struct virtio_device *dev_to_virtio (struct device *_dev) { return container_of(_dev, struct virtio_device, dev);
Signature · 6 lines
2.7.4 -- QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Michael S. Tsirkin
maintainerQuoted 1 line · Srivatsa Vaddagiri
The Type-1 hypervisor we are dealing with does not allow for MMIO transport.
How about PCI then?
Signature · 1 line
MST
Quoted 5 lines · Srivatsa Vaddagiri
The Type-1 hypervisor we are dealing with does not allow for MMIO transport.
[1] summarizes some of the problems we have in making virtio work on such hypervisors. This patch proposes a solution for transport problem viz how we can do config space IO on such a hypervisor. Hypervisor specific methods introduced allows for seamless IO of config space.
Seamless huh? You'd hope that might obviate the need for extra patches...
Quoted 2 lines
This patch is meant to seek comments. If its considered to be in right direction, will work on making it more complete and send the next version!
What's stopping you from implementing the trapping support in the hypervisor? Unlike the other patches you sent out, where the guest memory is not accessible to the host, there doesn't seem to be any advantage to not having trapping support, or am I missing something here?
Will
Re: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
In reply to Srivatsa VaddagiriQuoted 20 lines · Srivatsa Vaddagiri
Some hypervisors may not support MMIO transport i.e trap config space access and have it be handled by backend driver. They may allow other ways to interact with backend such as message-queue or doorbell API. This patch allows for hypervisor specific methods for config space IO.
Signed-off-by Srivatsa Vaddagiri <[email protected]>--- drivers/virtio/virtio_mmio.c | 131 ++++++++++++++++++++++++++----------------- include/linux/virtio.h | 14 +++++ 2 files changed, 94 insertions(+), 51 deletions(-)drivers/virtio/virtio_mmio.c +0
-0@@ -100,7 +100,35 @@ struct virtio_mmio_vq_info {struct list_head node; };
drivers/virtio/virtio_mmio.c struct virtio_mmio_vq_info { · line 112
Quoted 10 lines
drivers/virtio/virtio_mmio.c +9
-0+#ifdef CONFIG_VIRTIO_MMIO_OPS +static struct virtio_mmio_ops *mmio_ops; + +#define virtio_readb (a) mmio_ops->mmio_readl((a)) +#define virtio_readw (a) mmio_ops->mmio_readl((a)) +#define virtio_readl (a) mmio_ops->mmio_readl((a)) +#define virtio_writeb (val, a) mmio_ops->mmio_writeb((val), (a)) +#define virtio_writew (val, a) mmio_ops->mmio_writew((val), (a)) +#define virtio_writel (val, a) mmio_ops->mmio_writel((val), (a))
How exactly are these ops hooked up? I'm envisaging something like:
ops = spec_compliant_ops; [...] if (firmware_says_hypervisor_is_buggy()) ops = magic_qcom_ops;
am I wrong?
drivers/virtio/virtio_mmio.c struct virtio_mmio_vq_info { · line 117
Quoted 4 lines
drivers/virtio/virtio_mmio.c +4
-0+int register_virtio_mmio_ops (struct virtio_mmio_ops *ops) +{ +pr_info ("Registered %s as mmio ops\n" , ops->name); + mmio_ops = ops;
Not looking good, and really defeats the point of standardising this stuff imo.
Will
Quoted 7 lines · Will Deacon
Quoted 2 lines
This patch is meant to seek comments. If its considered to be in right direction, will work on making it more complete and send the next version!
What's stopping you from implementing the trapping support in the hypervisor? Unlike the other patches you sent out, where the guest memory is not accessible to the host, there doesn't seem to be any advantage to not having trapping support, or am I missing something here?
Hi Will,
I have had this discussion with hypervisor folks. They seem to be
concerned about complexity of having a VM's fault be handled in another untrusted VM. They are not keen to add MMIO support.
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Re: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
In reply to Will DeaconQuoted 13 lines · Will Deacon
drivers/virtio/virtio_mmio.c struct virtio_mmio_vq_info { · line 112
Quoted 10 lines
drivers/virtio/virtio_mmio.c +9
-0+#ifdef CONFIG_VIRTIO_MMIO_OPS +static struct virtio_mmio_ops *mmio_ops; + +#define virtio_readb (a) mmio_ops->mmio_readl((a)) +#define virtio_readw (a) mmio_ops->mmio_readl((a)) +#define virtio_readl (a) mmio_ops->mmio_readl((a)) +#define virtio_writeb (val, a) mmio_ops->mmio_writeb((val), (a)) +#define virtio_writew (val, a) mmio_ops->mmio_writew((val), (a)) +#define virtio_writel (val, a) mmio_ops->mmio_writel((val), (a))How exactly are these ops hooked up? I'm envisaging something like:
Quoted 6 lines
ops = spec_compliant_ops; [...] if (firmware_says_hypervisor_is_buggy()) ops = magic_qcom_ops;am I wrong?
If CONFIG_VIRTIO_MMIO_OPS is defined, then I expect this to be unconditionally set to 'magic_qcom_ops' that uses hypervisor-supported interface for IO (for
example: message_queue_send() and message_queue_recevie() hypercalls).
Quoted 7 lines
drivers/virtio/virtio_mmio.c struct virtio_mmio_vq_info { · line 117
Quoted 4 lines
drivers/virtio/virtio_mmio.c +4
-0+int register_virtio_mmio_ops (struct virtio_mmio_ops *ops) +{ +pr_info ("Registered %s as mmio ops\n" , ops->name); + mmio_ops = ops;Not looking good, and really defeats the point of standardising this stuff imo.
Ok. I guess the other option is to standardize on a new virtio transport (like ivshmem2-virtio)?
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Hi Vatsa,
Quoted 5 lines · Srivatsa Vaddagiri
Quoted 3 lines · Will Deacon
Quoted 2 lines
This patch is meant to seek comments. If its considered to be in right direction, will work on making it more complete and send the next version!
Quoted 8 lines
Quoted 4 lines
What's stopping you from implementing the trapping support in the hypervisor? Unlike the other patches you sent out, where the guest memory is not accessible to the host, there doesn't seem to be any advantage to not having trapping support, or am I missing something here?
I have had this discussion with hypervisor folks. They seem to be
concerned about complexity of having a VM's fault be handled in another untrusted VM. They are not keen to add MMIO support.
Right, but I'm concerned about forking the implementation from the spec and I'm not keen to add these hooks ;)
What does your hook actually do? I'm assuming an HVC? If so, then where the fault is handled seems to be unrelated and whether the guest exit is due to an HVC or a stage-2 fault should be immaterial. In other words, I don't follow why the trapping mechanism necessitates the way in which the fault is handled.
Will
Quoted 4 lines · Michael S. Tsirkin
Quoted 1 line · Srivatsa Vaddagiri
The Type-1 hypervisor we are dealing with does not allow for MMIO transport.
How about PCI then?
Correct me if I am wrong, but basically virtio_pci uses the same low-level primitive as readl/writel on a platform such as ARM64? So similar issues there also.
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Re: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
In reply to Srivatsa VaddagiriQuoted 20 lines · Srivatsa Vaddagiri
Quoted 18 lines · Will Deacon
drivers/virtio/virtio_mmio.c struct virtio_mmio_vq_info { · line 112
Quoted 10 lines
drivers/virtio/virtio_mmio.c +9
-0+#ifdef CONFIG_VIRTIO_MMIO_OPS +static struct virtio_mmio_ops *mmio_ops; + +#define virtio_readb (a) mmio_ops->mmio_readl((a)) +#define virtio_readw (a) mmio_ops->mmio_readl((a)) +#define virtio_readl (a) mmio_ops->mmio_readl((a)) +#define virtio_writeb (val, a) mmio_ops->mmio_writeb((val), (a)) +#define virtio_writew (val, a) mmio_ops->mmio_writew((val), (a)) +#define virtio_writel (val, a) mmio_ops->mmio_writel((val), (a))How exactly are these ops hooked up? I'm envisaging something like:
ops = spec_compliant_ops; [...] if (firmware_says_hypervisor_is_buggy()) ops = magic_qcom_ops;
Quoted 5 lines
Quoted 1 line
am I wrong?
If
CONFIG_VIRTIO_MMIO_OPSis defined, then I expect this to be unconditionally set to 'magic_qcom_ops' that uses hypervisor-supported interface for IO (for
example:message_queue_send()andmessage_queue_recevie()hypercalls).
Hmm, but then how would such a kernel work as a guest under all the spec-compliant hypervisors out there?
Quoted 10 lines
Quoted 7 lines
drivers/virtio/virtio_mmio.c struct virtio_mmio_vq_info { · line 117
Quoted 4 lines
drivers/virtio/virtio_mmio.c +4
-0+int register_virtio_mmio_ops (struct virtio_mmio_ops *ops) +{ +pr_info ("Registered %s as mmio ops\n" , ops->name); + mmio_ops = ops;Not looking good, and really defeats the point of standardising this stuff imo.
Ok. I guess the other option is to standardize on a new virtio transport (like ivshmem2-virtio)?
I haven't looked at that, but I suppose it depends on what your hypervisor folks are willing to accomodate.
Will
Quoted 3 lines · Michael S. Tsirkin
Quoted 1 line · Srivatsa Vaddagiri
The Type-1 hypervisor we are dealing with does not allow for MMIO transport.
How about PCI then?
Or maybe you can use virtio-vdpa.
Thanks
Quoted 8 lines · Will Deacon
Hi Vatsa,
Quoted 5 lines · Srivatsa Vaddagiri
Quoted 4 lines
What's stopping you from implementing the trapping support in the hypervisor? Unlike the other patches you sent out, where the guest memory is not accessible to the host, there doesn't seem to be any advantage to not having trapping support, or am I missing something here?
Quoted 8 lines
Quoted 3 lines
I have had this discussion with hypervisor folks. They seem to be
concerned about complexity of having a VM's fault be handled in another untrusted VM. They are not keen to add MMIO support.Right, but I'm concerned about forking the implementation from the spec and I'm not keen to add these hooks ;)
What does your hook actually do? I'm assuming an HVC?
Yes, it will issue message-queue related hypercalls
Quoted 3 lines
If so, then where the
fault is handled seems to be unrelated and whether the guest exit is due to an HVC or a stage-2 fault should be immaterial.
A stage-2 fault would be considered fatal normally and result in termination of guest. Modifying that behavior to allow resumption in case of virtio config space access, especially including the untrusted VM in this flow, is perhaps the concern. HVC calls OTOH are more vetted interfaces that the hypervisor has to do nothing additional to handle.
Quoted 3 lines
In other words, I don't
follow why the trapping mechanism necessitates the way in which the fault is handled.
Let me check with our hypervisor folks again. Thanks for your inputs.
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Re: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
In reply to Will DeaconQuoted 7 lines · Will Deacon
Quoted 3 lines · Srivatsa Vaddagiri
If
CONFIG_VIRTIO_MMIO_OPSis defined, then I expect this to be unconditionally set to 'magic_qcom_ops' that uses hypervisor-supported interface for IO (for
example:message_queue_send()andmessage_queue_recevie()hypercalls).Hmm, but then how would such a kernel work as a guest under all the spec-compliant hypervisors out there?
Ok I see your point and yes for better binary compatibility, the ops have to be set based on runtime detection of hypervisor capabilities.
Quoted 5 lines
Quoted 2 lines
Ok. I guess the other option is to standardize on a new virtio transport (like ivshmem2-virtio)?
I haven't looked at that, but I suppose it depends on what your hypervisor folks are willing to accomodate.
I believe ivshmem2_virtio requires hypervisor to support PCI device emulation (for life-cycle management of VMs), which our hypervisor may not support. A simple shared memory and doorbell or message-queue based transport will work for us.
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Re: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
Quoted 16 lines · Srivatsa Vaddagiri
Quoted 7 lines · Will Deacon
Quoted 3 lines · Srivatsa Vaddagiri
If
CONFIG_VIRTIO_MMIO_OPSis defined, then I expect this to be unconditionally set to 'magic_qcom_ops' that uses hypervisor-supported interface for IO (for
example:message_queue_send()andmessage_queue_recevie()hypercalls).Hmm, but then how would such a kernel work as a guest under all the spec-compliant hypervisors out there?
Ok I see your point and yes for better binary compatibility, the ops have to be set based on runtime detection of hypervisor capabilities.
Quoted 3 lines
Quoted 2 lines
Ok. I guess the other option is to standardize on a new virtio transport (like ivshmem2-virtio)?
Quoted 7 lines
Quoted 2 lines
I haven't looked at that, but I suppose it depends on what your hypervisor folks are willing to accomodate.
I believe
ivshmem2_virtiorequires hypervisor to support PCI device emulation (for life-cycle management of VMs), which our hypervisor may not support. A simple shared memory and doorbell or message-queue based transport will work for us.
As written in our private conversation, a mapping of the ivshmem2 device discovery to platform mechanism (device tree etc.) and maybe even the register access for doorbell and life-cycle management to something hypercall-like would be imaginable. What would count more from virtio perspective is a common mapping on a shared memory transport.
That said, I also warned about all the features that PCI already defined (such as message-based interrupts) which you may have to add when going a different way for the shared memory device.
Jan
Signature · 2 lines
Siemens AG, Corporate Technology, CT RDA IOT SES-DE Corporate Competence Center Embedded Linux
Re: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
Quoted 10 lines · Jan Kiszka
Quoted 4 lines
I believe
ivshmem2_virtiorequires hypervisor to support PCI device emulation (for life-cycle management of VMs), which our hypervisor may not support. A simple shared memory and doorbell or message-queue based transport will work for us.As written in our private conversation, a mapping of the ivshmem2 device discovery to platform mechanism (device tree etc.) and maybe even the register access for doorbell and life-cycle management to something hypercall-like would be imaginable. What would count more from virtio perspective is a common mapping on a shared memory transport.
Yes that sounds simpler for us.
Quoted 3 lines
That said, I also warned about all the features that PCI already defined (such as message-based interrupts) which you may have to add when going a different way for the shared memory device.
Is it really required to present this shared memory as belonging to a PCI device? I would expect the device-tree to indicate the presence of this shared memory region, which we should be able to present to ivshmem2 as shared memory region to use (along with some handles for doorbell or message queue use).
I understand the usefulness of modeling the shared memory as part of device so that hypervisor can send events related to peers going down or coming up. In our case, there will be other means to discover those events and avoiding this requirement on hypervisor (to emulate PCI) will simplify the solution for us.
Any idea when we can expect virtio over ivshmem2 to become available?!
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation
Michael S. Tsirkin
maintainerRe: [RFC/PATCH 1/1] virtio: Introduce MMIO ops
Quoted 4 lines · Srivatsa Vaddagiri
Quoted 2 lines · Jan Kiszka
Quoted 2 lines
I believe
ivshmem2_virtiorequires hypervisor to support PCI device emulation (for life-cycle management of VMs), which our hypervisor may not support.
PCI is mostly just 2 registers. One sets the affected device, one the data to read/write.
Quoted 10 lines
A
Quoted 8 lines
Quoted 2 lines
simple shared memory and doorbell or message-queue based transport will work for us.
As written in our private conversation, a mapping of the ivshmem2 device discovery to platform mechanism (device tree etc.) and maybe even the register access for doorbell and life-cycle management to something hypercall-like would be imaginable. What would count more from virtio perspective is a common mapping on a shared memory transport.
Quoted 8 lines
Yes that sounds simpler for us.
Quoted 3 lines
That said, I also warned about all the features that PCI already defined (such as message-based interrupts) which you may have to add when going a different way for the shared memory device.
Is it really required to present this shared memory as belonging to a PCI device?
But then you will go on and add MSI, and NUMA, and security, and and and ...
Quoted 10 lines
I would expect the device-tree to indicate the presence of this shared memory region, which we should be able to present to ivshmem2 as shared memory region to use (along with some handles for doorbell or message queue use).
I understand the usefulness of modeling the shared memory as part of device so that hypervisor can send events related to peers going down or coming up. In our case, there will be other means to discover those events and avoiding this requirement on hypervisor (to emulate PCI) will simplify the solution for us.
Any idea when we can expect virtio over ivshmem2 to become available?!
Check out the virtio spec. Right at the beginning it states:
These devices are
found in virtual environments, yet by design they look like physical devices to the guest within the virtual machine - and this document treats them as such. This similarity allows the guest to use standard drivers and discovery mechanisms
Quoted 3 lines
Signature · 2 lines
QUALCOMM INDIA, on behalf of Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, hosted by The Linux Foundation