virtio-pci: new config layout: using memory BAR
Subsystems (2)
Michael S. Tsirkin
maintainerRFC PATCH 1/1 virtio-pci: new config layout: using memory BAR
This adds support for new config, and is designed to work with the new layout code in Rusty's new layout branch.
At the moment all fields are in the same memory BAR (bar 2). This will be used to test performance and compare memory, io and hypercall latency.
Compiles but does not work yet.
Migration isn't handled yet.
It's not clear what do queue_enable/queue_disable
fields do, not yet implemented.
Gateway for config access with config cycles
not yet implemented.
Sending out for early review/flames.
--- hw/virtio/virtio-pci.c | 393 +++++++++++++++++++++++++++++++++++++++++++-- hw/virtio/virtio-pci.h | 55 +++++++ hw/virtio/virtio.c | 20 +++ include/hw/virtio/virtio.h | 4 + 4 files changed, 458 insertions(+), 14 deletions(-)
hw/virtio/virtio-pci.c +379 -14
@@ -259,6 +259,26 @@ static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy) proxy->ioeventfd_started = false; } +static void virtio_pci_set_status (VirtIOPCIProxy *proxy,uint8_t val) +{ +VirtIODevice *vdev = proxy->vdev; + +if (!(val &VIRTIO_CONFIG_S_DRIVER_OK )) { +virtio_pci_stop_ioeventfd (proxy); + } + +virtio_set_status (vdev, val &0xFF ); + +if (val &VIRTIO_CONFIG_S_DRIVER_OK ) { +virtio_pci_start_ioeventfd (proxy); + } + +if (vdev->status ==0 ) { +virtio_reset (proxy->vdev); +msix_unuse_all_vectors (&proxy->pci_dev); + } +} +static void virtio_ioport_write (void *opaque,uint32_t addr,uint32_t val) {VirtIOPCIProxy *proxy = opaque;
@@ -293,20 +313,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) }break ;case VIRTIO_PCI_STATUS : -if (!(val &VIRTIO_CONFIG_S_DRIVER_OK )) { -virtio_pci_stop_ioeventfd (proxy); - } - -virtio_set_status (vdev, val &0xFF ); - -if (val &VIRTIO_CONFIG_S_DRIVER_OK ) { -virtio_pci_start_ioeventfd (proxy); - } - -if (vdev->status ==0 ) { -virtio_reset (proxy->vdev); -msix_unuse_all_vectors (&proxy->pci_dev); - } +virtio_pci_set_status (proxy, val);/* Linux before 2.6.34 sets the device as OK without enabling the PCI device bus master bit. In this case we need to disable
@@ -455,6 +462,226 @@ static void virtio_pci_config_write(void *opaque, hwaddr addr, } } +static uint64_t virtio_pci_config_common_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select; +case offsetof(struct virtio_pci_common_cfg , device_feature): +/* TODO: 64-bit features */ +return proxy->device_feature_select ?0 : proxy->host_features; +case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +return proxy->guest_feature_select; +case offsetof(struct virtio_pci_common_cfg , guest_feature): +/* TODO: 64-bit features */ +return proxy->guest_feature_select ?0 : vdev->guest_features; +case offsetof(struct virtio_pci_common_cfg , msix_config): +return vdev->config_vector; +case offsetof(struct virtio_pci_common_cfg , num_queues): +/* TODO: more exact limit? */ +return VIRTIO_PCI_QUEUE_MAX ; +case offsetof(struct virtio_pci_common_cfg , device_status): +return vdev->status; + +/* About a specific virtqueue. */ +case offsetof(struct virtio_pci_common_cfg , queue_select): +return vdev->queue_sel; +case offsetof(struct virtio_pci_common_cfg , queue_size): +return virtio_queue_get_num (vdev, vdev->queue_sel); +case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +return virtio_queue_vector (vdev, vdev->queue_sel); +case offsetof(struct virtio_pci_common_cfg , queue_enable): +/* TODO */ +return 0 ; +case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +return vdev->queue_sel; +case offsetof(struct virtio_pci_common_cfg , queue_desc): +return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) >>32 ; +case offsetof(struct virtio_pci_common_cfg , queue_avail): +return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) >>32 ; +case offsetof(struct virtio_pci_common_cfg , queue_used): +return virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +return virtio_queue_get_used_addr (vdev, vdev->queue_sel) >>32 ; +default : +return 0 ; + } +} + +static void virtio_pci_config_common_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; +uint64_t high = ~low; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): + proxy->device_feature_select = val; +break ; +case offsetof(struct virtio_pci_common_cfg , device_feature): +break ; +case offsetof(struct virtio_pci_common_cfg , guest_feature_select): + proxy->guest_feature_select = val; +break ; +case offsetof(struct virtio_pci_common_cfg , guest_feature): +/* TODO: 64-bit features */ +if (!proxy->guest_feature_select) { +virtio_set_features (vdev, val); + } +break ; +case offsetof(struct virtio_pci_common_cfg , msix_config): + vdev->config_vector = val; +break ; +case offsetof(struct virtio_pci_common_cfg , num_queues): +break ; +case offsetof(struct virtio_pci_common_cfg , device_status): +virtio_pci_set_status (proxy, val); +break ; +/* About a specific virtqueue. */ +case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (val <VIRTIO_PCI_QUEUE_MAX ); + vdev->queue_sel = val; +break ; +case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (val && val <0x8ffff && !(val & (val -1 ))); +virtio_queue_set_num (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +virtio_queue_set_vector (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_enable): +/* TODO */ +break ; +case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +break ; +case offsetof(struct virtio_pci_common_cfg , queue_desc): + val &= low; + val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : + val = val <<32 ; + val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_avail): + val &= low; + val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : + val = val <<32 ; + val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_used): + val &= low; + val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_used_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : + val = val <<32 ; + val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_used_addr (vdev, vdev->queue_sel, val); +break ; +default : +break ; + } +} + +static uint64_t virtio_pci_config_isr_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; +uint8_t ret; + +/* reading from the ISR also clears it. */ + ret = vdev->isr; + vdev->isr =0 ; +qemu_set_irq (proxy->pci_dev.irq[0 ],0 ); +return ret; +} + +static void virtio_pci_config_isr_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +} + +static uint64_t virtio_pci_config_notify_read (void *opaque,hwaddr addr, +unsigned size) +{ +return 0 ; +} + +static void virtio_pci_config_notify_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; +if (val <VIRTIO_PCI_QUEUE_MAX ) { +virtio_queue_notify (vdev, val); + } +} + +static uint64_t virtio_pci_config_device_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +uint64_t val =0 ; +switch (size) { +case 1 : + val =virtio_config_readb (proxy->vdev, addr); +break ; +case 2 : + val =virtio_config_readw (proxy->vdev, addr); +break ; +case 4 : +/* Most devices don't have 64 bit config fields. +* Block is an exception: first 8 bytes include +* a 64 bit capacity field. +*/ +if (VIRTIO_HOST_IS_BIG_ENDIAN && + proxy->vdev->device_id ==VIRTIO_ID_BLOCK && addr <8 ) { +/* Swap first two words */ + addr ^=0x4 ; + } + val =virtio_config_readl (proxy->vdev, addr); +break ; + } +return val; +} + +static void virtio_pci_config_device_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +switch (size) { +case 1 : +virtio_config_writeb (proxy->vdev, addr, val); +break ; +case 2 : +virtio_config_writew (proxy->vdev, addr, val); +break ; +case 4 : +virtio_config_writel (proxy->vdev, addr, val); +break ; + } +} +static const MemoryRegionOps virtio_pci_config_ops = { .read = virtio_pci_config_read, .write = virtio_pci_config_write,
@@ -465,6 +692,46 @@ static const MemoryRegionOps virtio_pci_config_ops = { .endianness = DEVICE_LITTLE_ENDIAN, }; +static const MemoryRegionOps virtio_pci_config_common_ops = { + .read = virtio_pci_config_common_read, + .write = virtio_pci_config_common_write, + .impl = { + .min_access_size =1 , + .max_access_size =4 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_isr_ops = { + .read = virtio_pci_config_isr_read, + .write = virtio_pci_config_isr_write, + .impl = { + .min_access_size =1 , + .max_access_size =1 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_notify_ops = { + .read = virtio_pci_config_notify_read, + .write = virtio_pci_config_notify_write, + .impl = { + .min_access_size =2 , + .max_access_size =2 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_device_ops = { + .read = virtio_pci_config_device_read, + .write = virtio_pci_config_device_write, + .impl = { + .min_access_size =1 , + .max_access_size =4 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; +static void virtio_write_config (PCIDevice *pci_dev,uint32_t address,uint32_t val,int len) {
@@ -949,11 +1216,49 @@ static const TypeInfo virtio_9p_pci_info = {* virtio-pci: This is the PCIDevice which has a virtio-pci-bus. */ +#define VIRTIO_PCI_CONFIG_BAR_NUM 2 + +static void *virtio_pci_cap_add (VirtIOPCIProxy *proxy, +const char *name, +MemoryRegion *reg, +const MemoryRegionOps *ops, +unsigned offset, +unsigned size, +unsigned cap_type, +unsigned cap_length) +{ +struct virtio_pci_cap *cap; +int c; +memory_region_init_io (reg, ops, proxy, name, size); +memory_region_add_subregion (&proxy->config_bar, offset, reg); + +if (!cap_length) { + cap_length =sizeof *cap; + } + c =pci_add_capability (&proxy->pci_dev,PCI_CAP_ID_VNDR ,0 , cap_length); +assert (c >0 ); + cap = (void *)proxy->pci_dev.config + c; + cap->cap_length = cap_length; + cap->cfg_type = cap_type; + cap->bar =VIRTIO_PCI_CONFIG_BAR_NUM ; + cap->offset = offset; + cap->length = size; + +return cap; +} + +static void virtio_pci_cap_del (VirtIOPCIProxy *proxy,MemoryRegion *reg) +{ +memory_region_del_subregion (&proxy->config_bar, reg); +memory_region_destroy (reg); +} +/* This is called by virtio-bus just after the device is plugged. */ static void virtio_pci_device_plugged (DeviceState *d) {VirtIOPCIProxy *proxy =VIRTIO_PCI (d);VirtioBusState *bus = &proxy->bus; +struct virtio_pci_notify_cap *notify_cap;uint8_t *config;uint32_t size;
@@ -986,6 +1291,59 @@ static void virtio_pci_device_plugged(DeviceState *d)pci_register_bar (&proxy->pci_dev,0 ,PCI_BASE_ADDRESS_SPACE_IO , &proxy->bar); +/* New config layout */ +/* Four capabilities: common, notify, isr, device cfg */ +#define VIRTIO_PCI_CONFIG_BAR_SIZE 0x1000 +#define VIRTIO_PCI_CONFIG_COMMON 0x0 +#define VIRTIO_PCI_CONFIG_NOTIFY 0x100 +#define VIRTIO_PCI_CONFIG_ISR 0x200 +#define VIRTIO_PCI_CONFIG_DEVICE 0x300 + +/* Make sure we left enough space for all vqs */ +assert (VIRTIO_PCI_CONFIG_ISR -VIRTIO_PCI_CONFIG_NOTIFY >= +VIRTIO_PCI_QUEUE_MAX *2 ); +assert (VIRTIO_PCI_CONFIG_BAR_SIZE -VIRTIO_PCI_CONFIG_DEVICE >= + proxy->vdev->config_len); + +memory_region_init (&proxy->config_bar,"virtio-pci-config" , +VIRTIO_PCI_CONFIG_BAR_SIZE ); + +virtio_pci_cap_add (proxy,"virtio-pci-common" , + &proxy->config_common, + &virtio_pci_config_common_ops, +VIRTIO_PCI_CONFIG_COMMON , +sizeof (struct virtio_pci_common_cfg ), +VIRTIO_PCI_CAP_COMMON_CFG , +0 ); + notify_cap =virtio_pci_cap_add (proxy,"virtio-pci-notify" , + &proxy->config_notify, + &virtio_pci_config_notify_ops, +VIRTIO_PCI_CONFIG_NOTIFY , +VIRTIO_PCI_QUEUE_MAX *2 , +VIRTIO_PCI_CAP_NOTIFY_CFG , +sizeof *notify_cap); + notify_cap->notify_off_multiplier =0x2 ; +virtio_pci_cap_add (proxy,"virtio-pci-isr" , + &proxy->config_isr, + &virtio_pci_config_isr_ops, +VIRTIO_PCI_CONFIG_ISR , +0x1 , +VIRTIO_PCI_CAP_ISR_CFG , +0 ); +if (proxy->vdev->config_len) { +virtio_pci_cap_add (proxy,"virtio-pci-device" , + &proxy->config_device, + &virtio_pci_config_device_ops, +VIRTIO_PCI_CONFIG_DEVICE , + proxy->vdev->config_len, +VIRTIO_PCI_CAP_DEVICE_CFG , +0 ); + } + +pci_register_bar (&proxy->pci_dev,VIRTIO_PCI_CONFIG_BAR_NUM , +PCI_BASE_ADDRESS_SPACE_MEMORY , + &proxy->config_bar); +if (!kvm_has_many_ioeventfds ()) { proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD ; }
@@ -1012,6 +1370,13 @@ static void virtio_pci_exit(PCIDevice *pci_dev)VirtIOPCIProxy *proxy =VIRTIO_PCI (pci_dev);virtio_pci_stop_ioeventfd (proxy);memory_region_destroy (&proxy->bar); +virtio_pci_cap_del (proxy, &proxy->config_common); +virtio_pci_cap_del (proxy, &proxy->config_notify); +virtio_pci_cap_del (proxy, &proxy->config_isr); +if (&proxy->vdev->config_len) { +virtio_pci_cap_del (proxy, &proxy->config_device); + } +memory_region_destroy (&proxy->config_bar);msix_uninit_exclusive_bar (pci_dev); }
hw/virtio/virtio-pci.h +55 -0
71 lines not on this page · raw message
hw/virtio/virtio.c +20 -0
@@ -672,6 +672,11 @@ int virtio_queue_get_num(VirtIODevice *vdev, int n)return vdev->vq[n].vring.num; } +void virtio_queue_set_num (VirtIODevice *vdev,int n,int num) +{ + vdev->vq[n].vring.num = num; +} +int virtio_queue_get_id (VirtQueue *vq) {VirtIODevice *vdev = vq->vdev;
@@ -987,6 +992,21 @@ hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)return vdev->vq[n].vring.used; } +void virtio_queue_set_desc_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.desc = addr; +} + +void virtio_queue_set_avail_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.avail = addr; +} + +void virtio_queue_set_used_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.used = addr; +} +hwaddr virtio_queue_get_ring_addr (VirtIODevice *vdev,int n) {return vdev->vq[n].vring.desc;
include/hw/virtio/virtio.h +4 -0
@@ -199,6 +199,7 @@ void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data);void virtio_queue_set_addr (VirtIODevice *vdev,int n,hwaddr addr);hwaddr virtio_queue_get_addr (VirtIODevice *vdev,int n);int virtio_queue_get_num (VirtIODevice *vdev,int n); +void virtio_queue_set_num (VirtIODevice *vdev,int n,int num);void virtio_queue_notify (VirtIODevice *vdev,int n);uint16_t virtio_queue_vector (VirtIODevice *vdev,int n);void virtio_queue_set_vector (VirtIODevice *vdev,int n,uint16_t vector);
@@ -226,6 +227,9 @@ typedef struct VirtIORNGConf VirtIORNGConf;hwaddr virtio_queue_get_desc_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_avail_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_used_addr (VirtIODevice *vdev,int n); +void virtio_queue_set_desc_addr (VirtIODevice *vdev,int n,hwaddr addr); +void virtio_queue_set_avail_addr (VirtIODevice *vdev,int n,hwaddr addr); +void virtio_queue_set_used_addr (VirtIODevice *vdev,int n,hwaddr addr);hwaddr virtio_queue_get_ring_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_desc_size (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_avail_size (VirtIODevice *vdev,int n);
Signature · 1 line
MST
Quoted 84 lines · Michael S. Tsirkin
This adds support for new config, and is designed to work with the new layout code in Rusty's new layout branch.
At the moment all fields are in the same memory BAR (bar 2). This will be used to test performance and compare memory, io and hypercall latency.
Compiles but does not work yet.
Migration isn't handled yet.It's not clear what do queue_enable/queue_disable
fields do, not yet implemented.Gateway for config access with config cycles
not yet implemented.Sending out for early review/flames.
Signed-off-by Michael S. Tsirkin <[email protected]>--- hw/virtio/virtio-pci.c | 393 +++++++++++++++++++++++++++++++++++++++++++-- hw/virtio/virtio-pci.h | 55 +++++++ hw/virtio/virtio.c | 20 +++ include/hw/virtio/virtio.h | 4 + 4 files changed, 458 insertions(+), 14 deletions(-)hw/virtio/virtio-pci.c +22
-14@@ -259,6 +259,26 @@ static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy) proxy->ioeventfd_started = false; } +static void virtio_pci_set_status (VirtIOPCIProxy *proxy,uint8_t val) +{ +VirtIODevice *vdev = proxy->vdev; + +if (!(val &VIRTIO_CONFIG_S_DRIVER_OK )) { +virtio_pci_stop_ioeventfd (proxy); + } + +virtio_set_status (vdev, val &0xFF ); + +if (val &VIRTIO_CONFIG_S_DRIVER_OK ) { +virtio_pci_start_ioeventfd (proxy); + } + +if (vdev->status ==0 ) { +virtio_reset (proxy->vdev); +msix_unuse_all_vectors (&proxy->pci_dev); + } +} +static void virtio_ioport_write (void *opaque,uint32_t addr,uint32_t val) {VirtIOPCIProxy *proxy = opaque;@@ -293,20 +313,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) }break ;case VIRTIO_PCI_STATUS : -if (!(val &VIRTIO_CONFIG_S_DRIVER_OK )) { -virtio_pci_stop_ioeventfd (proxy); - } - -virtio_set_status (vdev, val &0xFF ); - -if (val &VIRTIO_CONFIG_S_DRIVER_OK ) { -virtio_pci_start_ioeventfd (proxy); - } - -if (vdev->status ==0 ) { -virtio_reset (proxy->vdev); -msix_unuse_all_vectors (&proxy->pci_dev); - } +virtio_pci_set_status (proxy, val);/* Linux before 2.6.34 sets the device as OK without enabling the PCI device bus master bit. In this case we need to disable @@ -455,6 +462,226 @@ static void virtio_pci_config_write(void *opaque, hwaddr addr, } } +static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr,
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 10 lines
hw/virtio/virtio-pci.c +10
-0+ unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;
Oh dear no... Please use defines like the rest of QEMU.
From a QEMU pov, take a look at:
https://github.com/aliguori/qemu/commit/587c35c1a3fe90f6af0f97927047ef4d3182a659
And:
https://github.com/aliguori/qemu/commit/01ba80a23cf2eb1e15056f82b44b94ec381565cb
Which lets virtio-pci be subclassable and then remaps the config space to BAR2.
Haven't looked at the proposed new ring layout yet.
Regards,
Anthony Liguori
Quoted 527 lines
hw/virtio/virtio-pci.c +347
-0+case offsetof(struct virtio_pci_common_cfg , device_feature): +/* TODO: 64-bit features */ +return proxy->device_feature_select ?0 : proxy->host_features; +case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +return proxy->guest_feature_select; +case offsetof(struct virtio_pci_common_cfg , guest_feature): +/* TODO: 64-bit features */ +return proxy->guest_feature_select ?0 : vdev->guest_features; +case offsetof(struct virtio_pci_common_cfg , msix_config): +return vdev->config_vector; +case offsetof(struct virtio_pci_common_cfg , num_queues): +/* TODO: more exact limit? */ +return VIRTIO_PCI_QUEUE_MAX ; +case offsetof(struct virtio_pci_common_cfg , device_status): +return vdev->status; + +/* About a specific virtqueue. */ +case offsetof(struct virtio_pci_common_cfg , queue_select): +return vdev->queue_sel; +case offsetof(struct virtio_pci_common_cfg , queue_size): +return virtio_queue_get_num (vdev, vdev->queue_sel); +case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +return virtio_queue_vector (vdev, vdev->queue_sel); +case offsetof(struct virtio_pci_common_cfg , queue_enable): +/* TODO */ +return 0 ; +case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +return vdev->queue_sel; +case offsetof(struct virtio_pci_common_cfg , queue_desc): +return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) >>32 ; +case offsetof(struct virtio_pci_common_cfg , queue_avail): +return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) >>32 ; +case offsetof(struct virtio_pci_common_cfg , queue_used): +return virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +return virtio_queue_get_used_addr (vdev, vdev->queue_sel) >>32 ; +default : +return 0 ; + } +} + +static void virtio_pci_config_common_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; +uint64_t high = ~low; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): + proxy->device_feature_select = val; +break ; +case offsetof(struct virtio_pci_common_cfg , device_feature): +break ; +case offsetof(struct virtio_pci_common_cfg , guest_feature_select): + proxy->guest_feature_select = val; +break ; +case offsetof(struct virtio_pci_common_cfg , guest_feature): +/* TODO: 64-bit features */ +if (!proxy->guest_feature_select) { +virtio_set_features (vdev, val); + } +break ; +case offsetof(struct virtio_pci_common_cfg , msix_config): + vdev->config_vector = val; +break ; +case offsetof(struct virtio_pci_common_cfg , num_queues): +break ; +case offsetof(struct virtio_pci_common_cfg , device_status): +virtio_pci_set_status (proxy, val); +break ; +/* About a specific virtqueue. */ +case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (val <VIRTIO_PCI_QUEUE_MAX ); + vdev->queue_sel = val; +break ; +case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (val && val <0x8ffff && !(val & (val -1 ))); +virtio_queue_set_num (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +virtio_queue_set_vector (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_enable): +/* TODO */ +break ; +case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +break ; +case offsetof(struct virtio_pci_common_cfg , queue_desc): + val &= low; + val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : + val = val <<32 ; + val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_avail): + val &= low; + val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : + val = val <<32 ; + val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_used): + val &= low; + val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_used_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : + val = val <<32 ; + val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_used_addr (vdev, vdev->queue_sel, val); +break ; +default : +break ; + } +} + +static uint64_t virtio_pci_config_isr_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; +uint8_t ret; + +/* reading from the ISR also clears it. */ + ret = vdev->isr; + vdev->isr =0 ; +qemu_set_irq (proxy->pci_dev.irq[0 ],0 ); +return ret; +} + +static void virtio_pci_config_isr_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +} + +static uint64_t virtio_pci_config_notify_read (void *opaque,hwaddr addr, +unsigned size) +{ +return 0 ; +} + +static void virtio_pci_config_notify_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; +if (val <VIRTIO_PCI_QUEUE_MAX ) { +virtio_queue_notify (vdev, val); + } +} + +static uint64_t virtio_pci_config_device_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +uint64_t val =0 ; +switch (size) { +case 1 : + val =virtio_config_readb (proxy->vdev, addr); +break ; +case 2 : + val =virtio_config_readw (proxy->vdev, addr); +break ; +case 4 : +/* Most devices don't have 64 bit config fields. +* Block is an exception: first 8 bytes include +* a 64 bit capacity field. +*/ +if (VIRTIO_HOST_IS_BIG_ENDIAN && + proxy->vdev->device_id ==VIRTIO_ID_BLOCK && addr <8 ) { +/* Swap first two words */ + addr ^=0x4 ; + } + val =virtio_config_readl (proxy->vdev, addr); +break ; + } +return val; +} + +static void virtio_pci_config_device_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +switch (size) { +case 1 : +virtio_config_writeb (proxy->vdev, addr, val); +break ; +case 2 : +virtio_config_writew (proxy->vdev, addr, val); +break ; +case 4 : +virtio_config_writel (proxy->vdev, addr, val); +break ; + } +} +static const MemoryRegionOps virtio_pci_config_ops = { .read = virtio_pci_config_read, .write = virtio_pci_config_write,@@ -465,6 +692,46 @@ static const MemoryRegionOps virtio_pci_config_ops = { .endianness = DEVICE_LITTLE_ENDIAN, }; +static const MemoryRegionOps virtio_pci_config_common_ops = { + .read = virtio_pci_config_common_read, + .write = virtio_pci_config_common_write, + .impl = { + .min_access_size =1 , + .max_access_size =4 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_isr_ops = { + .read = virtio_pci_config_isr_read, + .write = virtio_pci_config_isr_write, + .impl = { + .min_access_size =1 , + .max_access_size =1 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_notify_ops = { + .read = virtio_pci_config_notify_read, + .write = virtio_pci_config_notify_write, + .impl = { + .min_access_size =2 , + .max_access_size =2 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_device_ops = { + .read = virtio_pci_config_device_read, + .write = virtio_pci_config_device_write, + .impl = { + .min_access_size =1 , + .max_access_size =4 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; +static void virtio_write_config (PCIDevice *pci_dev,uint32_t address,uint32_t val,int len) {@@ -949,11 +1216,49 @@ static const TypeInfo virtio_9p_pci_info = {* virtio-pci: This is the PCIDevice which has a virtio-pci-bus. */ +#define VIRTIO_PCI_CONFIG_BAR_NUM 2 + +static void *virtio_pci_cap_add (VirtIOPCIProxy *proxy, +const char *name, +MemoryRegion *reg, +const MemoryRegionOps *ops, +unsigned offset, +unsigned size, +unsigned cap_type, +unsigned cap_length) +{ +struct virtio_pci_cap *cap; +int c; +memory_region_init_io (reg, ops, proxy, name, size); +memory_region_add_subregion (&proxy->config_bar, offset, reg); + +if (!cap_length) { + cap_length =sizeof *cap; + } + c =pci_add_capability (&proxy->pci_dev,PCI_CAP_ID_VNDR ,0 , cap_length); +assert (c >0 ); + cap = (void *)proxy->pci_dev.config + c; + cap->cap_length = cap_length; + cap->cfg_type = cap_type; + cap->bar =VIRTIO_PCI_CONFIG_BAR_NUM ; + cap->offset = offset; + cap->length = size; + +return cap; +} + +static void virtio_pci_cap_del (VirtIOPCIProxy *proxy,MemoryRegion *reg) +{ +memory_region_del_subregion (&proxy->config_bar, reg); +memory_region_destroy (reg); +} +/* This is called by virtio-bus just after the device is plugged. */ static void virtio_pci_device_plugged (DeviceState *d) {VirtIOPCIProxy *proxy =VIRTIO_PCI (d);VirtioBusState *bus = &proxy->bus; +struct virtio_pci_notify_cap *notify_cap;uint8_t *config;uint32_t size;@@ -986,6 +1291,59 @@ static void virtio_pci_device_plugged(DeviceState *d)pci_register_bar (&proxy->pci_dev,0 ,PCI_BASE_ADDRESS_SPACE_IO , &proxy->bar); +/* New config layout */ +/* Four capabilities: common, notify, isr, device cfg */ +#define VIRTIO_PCI_CONFIG_BAR_SIZE 0x1000 +#define VIRTIO_PCI_CONFIG_COMMON 0x0 +#define VIRTIO_PCI_CONFIG_NOTIFY 0x100 +#define VIRTIO_PCI_CONFIG_ISR 0x200 +#define VIRTIO_PCI_CONFIG_DEVICE 0x300 + +/* Make sure we left enough space for all vqs */ +assert (VIRTIO_PCI_CONFIG_ISR -VIRTIO_PCI_CONFIG_NOTIFY >= +VIRTIO_PCI_QUEUE_MAX *2 ); +assert (VIRTIO_PCI_CONFIG_BAR_SIZE -VIRTIO_PCI_CONFIG_DEVICE >= + proxy->vdev->config_len); + +memory_region_init (&proxy->config_bar,"virtio-pci-config" , +VIRTIO_PCI_CONFIG_BAR_SIZE ); + +virtio_pci_cap_add (proxy,"virtio-pci-common" , + &proxy->config_common, + &virtio_pci_config_common_ops, +VIRTIO_PCI_CONFIG_COMMON , +sizeof (struct virtio_pci_common_cfg ), +VIRTIO_PCI_CAP_COMMON_CFG , +0 ); + notify_cap =virtio_pci_cap_add (proxy,"virtio-pci-notify" , + &proxy->config_notify, + &virtio_pci_config_notify_ops, +VIRTIO_PCI_CONFIG_NOTIFY , +VIRTIO_PCI_QUEUE_MAX *2 , +VIRTIO_PCI_CAP_NOTIFY_CFG , +sizeof *notify_cap); + notify_cap->notify_off_multiplier =0x2 ; +virtio_pci_cap_add (proxy,"virtio-pci-isr" , + &proxy->config_isr, + &virtio_pci_config_isr_ops, +VIRTIO_PCI_CONFIG_ISR , +0x1 , +VIRTIO_PCI_CAP_ISR_CFG , +0 ); +if (proxy->vdev->config_len) { +virtio_pci_cap_add (proxy,"virtio-pci-device" , + &proxy->config_device, + &virtio_pci_config_device_ops, +VIRTIO_PCI_CONFIG_DEVICE , + proxy->vdev->config_len, +VIRTIO_PCI_CAP_DEVICE_CFG , +0 ); + } + +pci_register_bar (&proxy->pci_dev,VIRTIO_PCI_CONFIG_BAR_NUM , +PCI_BASE_ADDRESS_SPACE_MEMORY , + &proxy->config_bar); +if (!kvm_has_many_ioeventfds ()) { proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD ; }@@ -1012,6 +1370,13 @@ static void virtio_pci_exit(PCIDevice *pci_dev)VirtIOPCIProxy *proxy =VIRTIO_PCI (pci_dev);virtio_pci_stop_ioeventfd (proxy);memory_region_destroy (&proxy->bar); +virtio_pci_cap_del (proxy, &proxy->config_common); +virtio_pci_cap_del (proxy, &proxy->config_notify); +virtio_pci_cap_del (proxy, &proxy->config_isr); +if (&proxy->vdev->config_len) { +virtio_pci_cap_del (proxy, &proxy->config_device); + } +memory_region_destroy (&proxy->config_bar);msix_uninit_exclusive_bar (pci_dev); }hw/virtio/virtio-pci.h +55
-0@@ -84,6 +84,11 @@ struct VirtIOPCIProxy {PCIDevice pci_dev;VirtIODevice *vdev;MemoryRegion bar; +MemoryRegion config_bar; +MemoryRegion config_common; +MemoryRegion config_isr; +MemoryRegion config_notify; +MemoryRegion config_device;uint32_t flags;uint32_t class_code;uint32_t nvectors;@@ -93,6 +98,8 @@ struct VirtIOPCIProxy {VirtIOIRQFD *vector_irqfd;int nvqs_with_notifiers;VirtioBusState bus; +unsigned device_feature_select; +unsigned guest_feature_select; };@@ -203,4 +210,52 @@ struct VirtIORngPCI {/* Virtio ABI version, if we increment this, we break the guest driver. */ #define VIRTIO_PCI_ABI_VERSION 0 +/* IDs for different capabilities. Must all exist. */ +/* Common configuration */ +#define VIRTIO_PCI_CAP_COMMON_CFG 1 +/* Notifications */ +#define VIRTIO_PCI_CAP_NOTIFY_CFG 2 +/* ISR access */ +#define VIRTIO_PCI_CAP_ISR_CFG 3 +/* Device specific confiuration */ +#define VIRTIO_PCI_CAP_DEVICE_CFG 4 + +/* This is the PCI capability header: */ +struct virtio_pci_cap { +uint8_t cap_vndr;/* Generic PCI field: PCI_CAP_ID_VNDR */ +uint8_t cap_next;/* Generic PCI field: next ptr. */ +uint8_t cap_length;/* Generic PCI field: capability length. */ +uint8_t cfg_type;/* One of the VIRTIO_PCI_CAP_*_CFG. */ +uint8_t bar;/* Where to find it. */ +uint32_t offset;/* Offset within bar. */ +uint32_t length;/* Length. */ +}; + +struct virtio_pci_notify_cap { +struct virtio_pci_cap cap; +uint32_t notify_off_multiplier;/* Multiplier for queue_notify_off. */ +}; + +/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */ +struct virtio_pci_common_cfg { +/* About the whole device. */ +uint32_t device_feature_select;/* read-write */ +uint32_t device_feature;/* read-only */ +uint32_t guest_feature_select;/* read-write */ +uint32_t guest_feature;/* read-only */ +uint16_t msix_config;/* read-write */ +uint16_t num_queues;/* read-only */ +uint8_t device_status;/* read-write */ +uint8_t unused1; + +/* About a specific virtqueue. */ +uint16_t queue_select;/* read-write */ +uint16_t queue_size;/* read-write, power of 2. */ +uint16_t queue_msix_vector;/* read-write */ +uint16_t queue_enable;/* read-write */ +uint16_t queue_notify_off;/* read-only */ +uint64_t queue_desc;/* read-write */ +uint64_t queue_avail;/* read-write */ +uint64_t queue_used;/* read-write */ +};#endif hw/virtio/virtio.c +20
-0@@ -672,6 +672,11 @@ int virtio_queue_get_num(VirtIODevice *vdev, int n)return vdev->vq[n].vring.num; } +void virtio_queue_set_num (VirtIODevice *vdev,int n,int num) +{ + vdev->vq[n].vring.num = num; +} +int virtio_queue_get_id (VirtQueue *vq) {VirtIODevice *vdev = vq->vdev;@@ -987,6 +992,21 @@ hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)return vdev->vq[n].vring.used; } +void virtio_queue_set_desc_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.desc = addr; +} + +void virtio_queue_set_avail_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.avail = addr; +} + +void virtio_queue_set_used_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.used = addr; +} +hwaddr virtio_queue_get_ring_addr (VirtIODevice *vdev,int n) {return vdev->vq[n].vring.desc;include/hw/virtio/virtio.h +4
-0@@ -199,6 +199,7 @@ void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data);void virtio_queue_set_addr (VirtIODevice *vdev,int n,hwaddr addr);hwaddr virtio_queue_get_addr (VirtIODevice *vdev,int n);int virtio_queue_get_num (VirtIODevice *vdev,int n); +void virtio_queue_set_num (VirtIODevice *vdev,int n,int num);void virtio_queue_notify (VirtIODevice *vdev,int n);uint16_t virtio_queue_vector (VirtIODevice *vdev,int n);void virtio_queue_set_vector (VirtIODevice *vdev,int n,uint16_t vector);@@ -226,6 +227,9 @@ typedef struct VirtIORNGConf VirtIORNGConf;hwaddr virtio_queue_get_desc_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_avail_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_used_addr (VirtIODevice *vdev,int n); +void virtio_queue_set_desc_addr (VirtIODevice *vdev,int n,hwaddr addr); +void virtio_queue_set_avail_addr (VirtIODevice *vdev,int n,hwaddr addr); +void virtio_queue_set_used_addr (VirtIODevice *vdev,int n,hwaddr addr);hwaddr virtio_queue_get_ring_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_desc_size (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_avail_size (VirtIODevice *vdev,int n);Signature · 5 lines
MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 84 lines · Michael S. Tsirkin
This adds support for new config, and is designed to work with the new layout code in Rusty's new layout branch.
At the moment all fields are in the same memory BAR (bar 2). This will be used to test performance and compare memory, io and hypercall latency.
Compiles but does not work yet.
Migration isn't handled yet.It's not clear what do queue_enable/queue_disable
fields do, not yet implemented.Gateway for config access with config cycles
not yet implemented.Sending out for early review/flames.
Signed-off-by Michael S. Tsirkin <[email protected]>--- hw/virtio/virtio-pci.c | 393 +++++++++++++++++++++++++++++++++++++++++++-- hw/virtio/virtio-pci.h | 55 +++++++ hw/virtio/virtio.c | 20 +++ include/hw/virtio/virtio.h | 4 + 4 files changed, 458 insertions(+), 14 deletions(-)hw/virtio/virtio-pci.c +22
-14@@ -259,6 +259,26 @@ static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy) proxy->ioeventfd_started = false; } +static void virtio_pci_set_status (VirtIOPCIProxy *proxy,uint8_t val) +{ +VirtIODevice *vdev = proxy->vdev; + +if (!(val &VIRTIO_CONFIG_S_DRIVER_OK )) { +virtio_pci_stop_ioeventfd (proxy); + } + +virtio_set_status (vdev, val &0xFF ); + +if (val &VIRTIO_CONFIG_S_DRIVER_OK ) { +virtio_pci_start_ioeventfd (proxy); + } + +if (vdev->status ==0 ) { +virtio_reset (proxy->vdev); +msix_unuse_all_vectors (&proxy->pci_dev); + } +} +static void virtio_ioport_write (void *opaque,uint32_t addr,uint32_t val) {VirtIOPCIProxy *proxy = opaque;@@ -293,20 +313,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) }break ;case VIRTIO_PCI_STATUS : -if (!(val &VIRTIO_CONFIG_S_DRIVER_OK )) { -virtio_pci_stop_ioeventfd (proxy); - } - -virtio_set_status (vdev, val &0xFF ); - -if (val &VIRTIO_CONFIG_S_DRIVER_OK ) { -virtio_pci_start_ioeventfd (proxy); - } - -if (vdev->status ==0 ) { -virtio_reset (proxy->vdev); -msix_unuse_all_vectors (&proxy->pci_dev); - } +virtio_pci_set_status (proxy, val);/* Linux before 2.6.34 sets the device as OK without enabling the PCI device bus master bit. In this case we need to disable @@ -455,6 +462,226 @@ static void virtio_pci_config_write(void *opaque, hwaddr addr, } } +static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr,
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 10 lines
hw/virtio/virtio-pci.c +10
-0+ unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;
Oh dear no... Please use defines like the rest of QEMU.
Quoted 1 line
From a QEMU pov, take a look at:
https://github.com/aliguori/qemu/commit/587c35c1a3fe90f6af0f97927047ef4d3182a659
And:
https://github.com/aliguori/qemu/commit/01ba80a23cf2eb1e15056f82b44b94ec381565cb
Which lets virtio-pci be subclassable and then remaps the config space to BAR2.
Haven't looked at the proposed new ring layout yet.
Regards,
Anthony Liguori
Quoted 527 lines
hw/virtio/virtio-pci.c +347
-0+case offsetof(struct virtio_pci_common_cfg , device_feature): +/* TODO: 64-bit features */ +return proxy->device_feature_select ?0 : proxy->host_features; +case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +return proxy->guest_feature_select; +case offsetof(struct virtio_pci_common_cfg , guest_feature): +/* TODO: 64-bit features */ +return proxy->guest_feature_select ?0 : vdev->guest_features; +case offsetof(struct virtio_pci_common_cfg , msix_config): +return vdev->config_vector; +case offsetof(struct virtio_pci_common_cfg , num_queues): +/* TODO: more exact limit? */ +return VIRTIO_PCI_QUEUE_MAX ; +case offsetof(struct virtio_pci_common_cfg , device_status): +return vdev->status; + +/* About a specific virtqueue. */ +case offsetof(struct virtio_pci_common_cfg , queue_select): +return vdev->queue_sel; +case offsetof(struct virtio_pci_common_cfg , queue_size): +return virtio_queue_get_num (vdev, vdev->queue_sel); +case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +return virtio_queue_vector (vdev, vdev->queue_sel); +case offsetof(struct virtio_pci_common_cfg , queue_enable): +/* TODO */ +return 0 ; +case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +return vdev->queue_sel; +case offsetof(struct virtio_pci_common_cfg , queue_desc): +return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) >>32 ; +case offsetof(struct virtio_pci_common_cfg , queue_avail): +return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) >>32 ; +case offsetof(struct virtio_pci_common_cfg , queue_used): +return virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low; +case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +return virtio_queue_get_used_addr (vdev, vdev->queue_sel) >>32 ; +default : +return 0 ; + } +} + +static void virtio_pci_config_common_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; +uint64_t high = ~low; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): + proxy->device_feature_select = val; +break ; +case offsetof(struct virtio_pci_common_cfg , device_feature): +break ; +case offsetof(struct virtio_pci_common_cfg , guest_feature_select): + proxy->guest_feature_select = val; +break ; +case offsetof(struct virtio_pci_common_cfg , guest_feature): +/* TODO: 64-bit features */ +if (!proxy->guest_feature_select) { +virtio_set_features (vdev, val); + } +break ; +case offsetof(struct virtio_pci_common_cfg , msix_config): + vdev->config_vector = val; +break ; +case offsetof(struct virtio_pci_common_cfg , num_queues): +break ; +case offsetof(struct virtio_pci_common_cfg , device_status): +virtio_pci_set_status (proxy, val); +break ; +/* About a specific virtqueue. */ +case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (val <VIRTIO_PCI_QUEUE_MAX ); + vdev->queue_sel = val; +break ; +case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (val && val <0x8ffff && !(val & (val -1 ))); +virtio_queue_set_num (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +virtio_queue_set_vector (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_enable): +/* TODO */ +break ; +case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +break ; +case offsetof(struct virtio_pci_common_cfg , queue_desc): + val &= low; + val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : + val = val <<32 ; + val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_avail): + val &= low; + val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : + val = val <<32 ; + val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_used): + val &= low; + val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & high; +virtio_queue_set_used_addr (vdev, vdev->queue_sel, val); +break ; +case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : + val = val <<32 ; + val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low; +virtio_queue_set_used_addr (vdev, vdev->queue_sel, val); +break ; +default : +break ; + } +} + +static uint64_t virtio_pci_config_isr_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; +uint8_t ret; + +/* reading from the ISR also clears it. */ + ret = vdev->isr; + vdev->isr =0 ; +qemu_set_irq (proxy->pci_dev.irq[0 ],0 ); +return ret; +} + +static void virtio_pci_config_isr_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +} + +static uint64_t virtio_pci_config_notify_read (void *opaque,hwaddr addr, +unsigned size) +{ +return 0 ; +} + +static void virtio_pci_config_notify_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; +if (val <VIRTIO_PCI_QUEUE_MAX ) { +virtio_queue_notify (vdev, val); + } +} + +static uint64_t virtio_pci_config_device_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +uint64_t val =0 ; +switch (size) { +case 1 : + val =virtio_config_readb (proxy->vdev, addr); +break ; +case 2 : + val =virtio_config_readw (proxy->vdev, addr); +break ; +case 4 : +/* Most devices don't have 64 bit config fields. +* Block is an exception: first 8 bytes include +* a 64 bit capacity field. +*/ +if (VIRTIO_HOST_IS_BIG_ENDIAN && + proxy->vdev->device_id ==VIRTIO_ID_BLOCK && addr <8 ) { +/* Swap first two words */ + addr ^=0x4 ; + } + val =virtio_config_readl (proxy->vdev, addr); +break ; + } +return val; +} + +static void virtio_pci_config_device_write (void *opaque,hwaddr addr, +uint64_t val,unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +switch (size) { +case 1 : +virtio_config_writeb (proxy->vdev, addr, val); +break ; +case 2 : +virtio_config_writew (proxy->vdev, addr, val); +break ; +case 4 : +virtio_config_writel (proxy->vdev, addr, val); +break ; + } +} +static const MemoryRegionOps virtio_pci_config_ops = { .read = virtio_pci_config_read, .write = virtio_pci_config_write,@@ -465,6 +692,46 @@ static const MemoryRegionOps virtio_pci_config_ops = { .endianness = DEVICE_LITTLE_ENDIAN, }; +static const MemoryRegionOps virtio_pci_config_common_ops = { + .read = virtio_pci_config_common_read, + .write = virtio_pci_config_common_write, + .impl = { + .min_access_size =1 , + .max_access_size =4 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_isr_ops = { + .read = virtio_pci_config_isr_read, + .write = virtio_pci_config_isr_write, + .impl = { + .min_access_size =1 , + .max_access_size =1 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_notify_ops = { + .read = virtio_pci_config_notify_read, + .write = virtio_pci_config_notify_write, + .impl = { + .min_access_size =2 , + .max_access_size =2 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; + +static const MemoryRegionOps virtio_pci_config_device_ops = { + .read = virtio_pci_config_device_read, + .write = virtio_pci_config_device_write, + .impl = { + .min_access_size =1 , + .max_access_size =4 , + }, + .endianness =DEVICE_LITTLE_ENDIAN , +}; +static void virtio_write_config (PCIDevice *pci_dev,uint32_t address,uint32_t val,int len) {@@ -949,11 +1216,49 @@ static const TypeInfo virtio_9p_pci_info = {* virtio-pci: This is the PCIDevice which has a virtio-pci-bus. */ +#define VIRTIO_PCI_CONFIG_BAR_NUM 2 + +static void *virtio_pci_cap_add (VirtIOPCIProxy *proxy, +const char *name, +MemoryRegion *reg, +const MemoryRegionOps *ops, +unsigned offset, +unsigned size, +unsigned cap_type, +unsigned cap_length) +{ +struct virtio_pci_cap *cap; +int c; +memory_region_init_io (reg, ops, proxy, name, size); +memory_region_add_subregion (&proxy->config_bar, offset, reg); + +if (!cap_length) { + cap_length =sizeof *cap; + } + c =pci_add_capability (&proxy->pci_dev,PCI_CAP_ID_VNDR ,0 , cap_length); +assert (c >0 ); + cap = (void *)proxy->pci_dev.config + c; + cap->cap_length = cap_length; + cap->cfg_type = cap_type; + cap->bar =VIRTIO_PCI_CONFIG_BAR_NUM ; + cap->offset = offset; + cap->length = size; + +return cap; +} + +static void virtio_pci_cap_del (VirtIOPCIProxy *proxy,MemoryRegion *reg) +{ +memory_region_del_subregion (&proxy->config_bar, reg); +memory_region_destroy (reg); +} +/* This is called by virtio-bus just after the device is plugged. */ static void virtio_pci_device_plugged (DeviceState *d) {VirtIOPCIProxy *proxy =VIRTIO_PCI (d);VirtioBusState *bus = &proxy->bus; +struct virtio_pci_notify_cap *notify_cap;uint8_t *config;uint32_t size;@@ -986,6 +1291,59 @@ static void virtio_pci_device_plugged(DeviceState *d)pci_register_bar (&proxy->pci_dev,0 ,PCI_BASE_ADDRESS_SPACE_IO , &proxy->bar); +/* New config layout */ +/* Four capabilities: common, notify, isr, device cfg */ +#define VIRTIO_PCI_CONFIG_BAR_SIZE 0x1000 +#define VIRTIO_PCI_CONFIG_COMMON 0x0 +#define VIRTIO_PCI_CONFIG_NOTIFY 0x100 +#define VIRTIO_PCI_CONFIG_ISR 0x200 +#define VIRTIO_PCI_CONFIG_DEVICE 0x300 + +/* Make sure we left enough space for all vqs */ +assert (VIRTIO_PCI_CONFIG_ISR -VIRTIO_PCI_CONFIG_NOTIFY >= +VIRTIO_PCI_QUEUE_MAX *2 ); +assert (VIRTIO_PCI_CONFIG_BAR_SIZE -VIRTIO_PCI_CONFIG_DEVICE >= + proxy->vdev->config_len); + +memory_region_init (&proxy->config_bar,"virtio-pci-config" , +VIRTIO_PCI_CONFIG_BAR_SIZE ); + +virtio_pci_cap_add (proxy,"virtio-pci-common" , + &proxy->config_common, + &virtio_pci_config_common_ops, +VIRTIO_PCI_CONFIG_COMMON , +sizeof (struct virtio_pci_common_cfg ), +VIRTIO_PCI_CAP_COMMON_CFG , +0 ); + notify_cap =virtio_pci_cap_add (proxy,"virtio-pci-notify" , + &proxy->config_notify, + &virtio_pci_config_notify_ops, +VIRTIO_PCI_CONFIG_NOTIFY , +VIRTIO_PCI_QUEUE_MAX *2 , +VIRTIO_PCI_CAP_NOTIFY_CFG , +sizeof *notify_cap); + notify_cap->notify_off_multiplier =0x2 ; +virtio_pci_cap_add (proxy,"virtio-pci-isr" , + &proxy->config_isr, + &virtio_pci_config_isr_ops, +VIRTIO_PCI_CONFIG_ISR , +0x1 , +VIRTIO_PCI_CAP_ISR_CFG , +0 ); +if (proxy->vdev->config_len) { +virtio_pci_cap_add (proxy,"virtio-pci-device" , + &proxy->config_device, + &virtio_pci_config_device_ops, +VIRTIO_PCI_CONFIG_DEVICE , + proxy->vdev->config_len, +VIRTIO_PCI_CAP_DEVICE_CFG , +0 ); + } + +pci_register_bar (&proxy->pci_dev,VIRTIO_PCI_CONFIG_BAR_NUM , +PCI_BASE_ADDRESS_SPACE_MEMORY , + &proxy->config_bar); +if (!kvm_has_many_ioeventfds ()) { proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD ; }@@ -1012,6 +1370,13 @@ static void virtio_pci_exit(PCIDevice *pci_dev)VirtIOPCIProxy *proxy =VIRTIO_PCI (pci_dev);virtio_pci_stop_ioeventfd (proxy);memory_region_destroy (&proxy->bar); +virtio_pci_cap_del (proxy, &proxy->config_common); +virtio_pci_cap_del (proxy, &proxy->config_notify); +virtio_pci_cap_del (proxy, &proxy->config_isr); +if (&proxy->vdev->config_len) { +virtio_pci_cap_del (proxy, &proxy->config_device); + } +memory_region_destroy (&proxy->config_bar);msix_uninit_exclusive_bar (pci_dev); }hw/virtio/virtio-pci.h +55
-0@@ -84,6 +84,11 @@ struct VirtIOPCIProxy {PCIDevice pci_dev;VirtIODevice *vdev;MemoryRegion bar; +MemoryRegion config_bar; +MemoryRegion config_common; +MemoryRegion config_isr; +MemoryRegion config_notify; +MemoryRegion config_device;uint32_t flags;uint32_t class_code;uint32_t nvectors;@@ -93,6 +98,8 @@ struct VirtIOPCIProxy {VirtIOIRQFD *vector_irqfd;int nvqs_with_notifiers;VirtioBusState bus; +unsigned device_feature_select; +unsigned guest_feature_select; };@@ -203,4 +210,52 @@ struct VirtIORngPCI {/* Virtio ABI version, if we increment this, we break the guest driver. */ #define VIRTIO_PCI_ABI_VERSION 0 +/* IDs for different capabilities. Must all exist. */ +/* Common configuration */ +#define VIRTIO_PCI_CAP_COMMON_CFG 1 +/* Notifications */ +#define VIRTIO_PCI_CAP_NOTIFY_CFG 2 +/* ISR access */ +#define VIRTIO_PCI_CAP_ISR_CFG 3 +/* Device specific confiuration */ +#define VIRTIO_PCI_CAP_DEVICE_CFG 4 + +/* This is the PCI capability header: */ +struct virtio_pci_cap { +uint8_t cap_vndr;/* Generic PCI field: PCI_CAP_ID_VNDR */ +uint8_t cap_next;/* Generic PCI field: next ptr. */ +uint8_t cap_length;/* Generic PCI field: capability length. */ +uint8_t cfg_type;/* One of the VIRTIO_PCI_CAP_*_CFG. */ +uint8_t bar;/* Where to find it. */ +uint32_t offset;/* Offset within bar. */ +uint32_t length;/* Length. */ +}; + +struct virtio_pci_notify_cap { +struct virtio_pci_cap cap; +uint32_t notify_off_multiplier;/* Multiplier for queue_notify_off. */ +}; + +/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */ +struct virtio_pci_common_cfg { +/* About the whole device. */ +uint32_t device_feature_select;/* read-write */ +uint32_t device_feature;/* read-only */ +uint32_t guest_feature_select;/* read-write */ +uint32_t guest_feature;/* read-only */ +uint16_t msix_config;/* read-write */ +uint16_t num_queues;/* read-only */ +uint8_t device_status;/* read-write */ +uint8_t unused1; + +/* About a specific virtqueue. */ +uint16_t queue_select;/* read-write */ +uint16_t queue_size;/* read-write, power of 2. */ +uint16_t queue_msix_vector;/* read-write */ +uint16_t queue_enable;/* read-write */ +uint16_t queue_notify_off;/* read-only */ +uint64_t queue_desc;/* read-write */ +uint64_t queue_avail;/* read-write */ +uint64_t queue_used;/* read-write */ +};#endif hw/virtio/virtio.c +20
-0@@ -672,6 +672,11 @@ int virtio_queue_get_num(VirtIODevice *vdev, int n)return vdev->vq[n].vring.num; } +void virtio_queue_set_num (VirtIODevice *vdev,int n,int num) +{ + vdev->vq[n].vring.num = num; +} +int virtio_queue_get_id (VirtQueue *vq) {VirtIODevice *vdev = vq->vdev;@@ -987,6 +992,21 @@ hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)return vdev->vq[n].vring.used; } +void virtio_queue_set_desc_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.desc = addr; +} + +void virtio_queue_set_avail_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.avail = addr; +} + +void virtio_queue_set_used_addr (VirtIODevice *vdev,int n,hwaddr addr) +{ + vdev->vq[n].vring.used = addr; +} +hwaddr virtio_queue_get_ring_addr (VirtIODevice *vdev,int n) {return vdev->vq[n].vring.desc;include/hw/virtio/virtio.h +4
-0@@ -199,6 +199,7 @@ void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data);void virtio_queue_set_addr (VirtIODevice *vdev,int n,hwaddr addr);hwaddr virtio_queue_get_addr (VirtIODevice *vdev,int n);int virtio_queue_get_num (VirtIODevice *vdev,int n); +void virtio_queue_set_num (VirtIODevice *vdev,int n,int num);void virtio_queue_notify (VirtIODevice *vdev,int n);uint16_t virtio_queue_vector (VirtIODevice *vdev,int n);void virtio_queue_set_vector (VirtIODevice *vdev,int n,uint16_t vector);@@ -226,6 +227,9 @@ typedef struct VirtIORNGConf VirtIORNGConf;hwaddr virtio_queue_get_desc_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_avail_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_used_addr (VirtIODevice *vdev,int n); +void virtio_queue_set_desc_addr (VirtIODevice *vdev,int n,hwaddr addr); +void virtio_queue_set_avail_addr (VirtIODevice *vdev,int n,hwaddr addr); +void virtio_queue_set_used_addr (VirtIODevice *vdev,int n,hwaddr addr);hwaddr virtio_queue_get_ring_addr (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_desc_size (VirtIODevice *vdev,int n);hwaddr virtio_queue_get_avail_size (VirtIODevice *vdev,int n);Signature · 5 lines
MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 4 lines · Anthony Liguori
Quoted 4 lines
(quoted diff) +0
-0@@ -455,6 +462,226 @@ static void virtio_pci_config_write(void *opaque, hwaddr addr, } }
Quoted 13 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 11 lines
hw/virtio/virtio-pci.c +11
-0+static uint64_t virtio_pci_config_common_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
I see about 138 examples in qemu.
Anyway, that's the way Rusty wrote it in the kernel header - I started with defines.
If you convince Rusty to switch I can switch too, but I prefer reusing same structures as linux even if for now I've given up on reusing same headers.
Quoted 10 lines
Quoted 1 line
From a QEMU pov, take a look at:
https://github.com/aliguori/qemu/commit/587c35c1a3fe90f6af0f97927047ef4d3182a659
And:
https://github.com/aliguori/qemu/commit/01ba80a23cf2eb1e15056f82b44b94ec381565cb
Which lets virtio-pci be subclassable and then remaps the config space to BAR2.
Interesting. Have the spec anywhere?
You are saying this is going to conflict because of BAR2 usage, yes?
So let's only do this virtio-fb only for new layout, so we don't need to maintain compatibility. In particular, we are working on making memory BAR access fast for virtio devices
in a generic way. At the moment they are measureably slower than PIO on x86.
Quoted 3 lines
Haven't looked at the proposed new ring layout yet.
Regards,
No new ring layout. It's new config layout.
Signature · 1 line
MST
Paolo Bonzini
maintainerIl 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
Quoted 1 line
I see about 138 examples in qemu.
Almost all of them are about fields in the host, not the guest, and none of them are in case statements.
Paolo
Quoted 8 lines
Anyway, that's the way Rusty wrote it in the kernel header - I started with defines.
If you convince Rusty to switch I can switch too, but I prefer reusing same structures as linux even if for now I've given up on reusing same headers.
Quoted 19 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
Quoted 4 lines
(quoted diff) +0
-0@@ -455,6 +462,226 @@ static void virtio_pci_config_write(void *opaque, hwaddr addr, } }Quoted 13 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 11 lines
hw/virtio/virtio-pci.c +11
-0+static uint64_t virtio_pci_config_common_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Quoted 2 lines
Any good reason not to use offsetof?
I see about 138 examples in qemu.
There are exactly zero:
$ find . -name "*.c" -exec grep -l "case offset" {} \;
$Quoted 3 lines
Anyway, that's the way Rusty wrote it in the kernel header - I started with defines.
If you convince Rusty to switch I can switch too,
We have 300+ devices in QEMU that use #defines. We're not using this kind of thing just because you want to copy code from the kernel.
Quoted 11 lines
Quoted 8 lines
https://github.com/aliguori/qemu/commit/587c35c1a3fe90f6af0f97927047ef4d3182a659
And:
https://github.com/aliguori/qemu/commit/01ba80a23cf2eb1e15056f82b44b94ec381565cb
Which lets virtio-pci be subclassable and then remaps the config space to BAR2.
Interesting. Have the spec anywhere?
Not yet, but working on that.
Quoted 2 lines
You are saying this is going to conflict because
of BAR2 usage, yes?
No, this whole thing is flexible. I had to use BAR2 because BAR0 has to be the vram mapping. It also had to be an MMIO bar.
The new layout might make it easier to implement a device like this. I shared it mainly because I wanted to show the subclassing idea vs. just tacking an option onto the existing virtio-pci code in QEMU.
Regards,
Anthony Liguori
Quoted 20 lines
So let's only do this virtio-fb only for new layout, so we don't need to maintain compatibility. In particular, we are working on making memory BAR access fast for virtio devices
in a generic way. At the moment they are measureably slower than PIO on x86.Quoted 3 lines
Haven't looked at the proposed new ring layout yet.
Regards,
No new ring layout. It's new config layout.
Signature · 5 lines
MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 29 lines · Anthony Liguori
Quoted 19 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
Quoted 4 lines
(quoted diff) +0
-0@@ -455,6 +462,226 @@ static void virtio_pci_config_write(void *opaque, hwaddr addr, } }Quoted 13 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 11 lines
hw/virtio/virtio-pci.c +11
-0+static uint64_t virtio_pci_config_common_read (void *opaque,hwaddr addr, +unsigned size) +{ +VirtIOPCIProxy *proxy = opaque; +VirtIODevice *vdev = proxy->vdev; + +uint64_t low =0xffffffffull ; + +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Quoted 2 lines
Any good reason not to use offsetof?
I see about 138 examples in qemu.There are exactly zero:
$ find . -name "*.c" -exec grep -l "case offset" {} \; $
Quoted 6 lines
Quoted 3 lines
Anyway, that's the way Rusty wrote it in the kernel header - I started with defines.
If you convince Rusty to switch I can switch too,We have 300+ devices in QEMU that use #defines. We're not using this kind of thing just because you want to copy code from the kernel.
Rusty, let's switch to defines?
Quoted 14 lines
Quoted 11 lines
Quoted 8 lines
https://github.com/aliguori/qemu/commit/587c35c1a3fe90f6af0f97927047ef4d3182a659
And:
https://github.com/aliguori/qemu/commit/01ba80a23cf2eb1e15056f82b44b94ec381565cb
Which lets virtio-pci be subclassable and then remaps the config space to BAR2.
Interesting. Have the spec anywhere?
Not yet, but working on that.
Quoted 7 lines
Quoted 2 lines
You are saying this is going to conflict because
of BAR2 usage, yes?No, this whole thing is flexible. I had to use BAR2 because BAR0 has to be the vram mapping. It also had to be an MMIO bar.
The new layout might make it easier to implement a device like this.
Absolutely, you can put thing anywhere you like.
Quoted 7 lines
I
shared it mainly because I wanted to show the subclassing idea vs. just tacking an option onto the existing virtio-pci code in QEMU.Regards,
Anthony Liguori
I don't think a subclass will work for the new layout. This should be completely transparent to users, just have more devices work with more flexibility.
Quoted 20 lines
Quoted 20 lines
So let's only do this virtio-fb only for new layout, so we don't need to maintain compatibility. In particular, we are working on making memory BAR access fast for virtio devices
in a generic way. At the moment they are measureably slower than PIO on x86.Quoted 3 lines
Haven't looked at the proposed new ring layout yet.
Regards,
No new ring layout. It's new config layout.
Signature · 5 lines
MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 6 lines · Paolo Bonzini
Il 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;
Quoted 6 lines
Quoted 2 lines
Quoted 1 line
Quoted 1 line
Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
As long as we use the offsetof() that comes with the C implementation (ie. we don't hack it up ourselves), we should be safe; ISO C99 elegantly defines offsetof() in "7.17 Common definitions <stddef.h>" p3 as
The macros are [...]
offsetof(type, member-designator)
which expands to an integer constant expression that has type *size_t*, the value of which is the offset in bytes, to the structure member (designated by /member-designator/), from the beginning of its structure (designated by /type/). The type and member designator shall be such that given
static type t;
then the expression &(t.member-designator) evaluates to an address constant. (If the specified member is a bit-field, the behavior is undefined.)
("Address constant" is described in 6.6p9 but quoting even that here doesn't seem necesary.)
Quoted 1 line
From 6.8.4.2p3, "The expression of each case label shall be an integer
constant expression [...]".
So,
(a) if we use the standard offsetof(),
(b) and you can also write, for example,
static struct virtio_pci_common_cfg t; static void *p = &t.device_feature_select;
then the construct seems valid.
(Consistently with the above mention of UB if the specified member is a
bit-field: the address-of operator's constraints also forbid a bit-field object as operand.)
Laszlo
Quoted 5 lines · Anthony Liguori
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 2 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +2
-0+case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
So AFAICT the question is, do we put the required
#define VIRTIO_PCI_CFG_FEATURE_SEL \
(offsetof(struct virtio_pci_common_cfg, device_feature_select))
etc. in the kernel headers or qemu?
Quoted 1 line
Haven't looked at the proposed new ring layout yet.
No change, but there's an open question on whether we should nail it to little endian (or define the endian by the transport).
Of course, I can't rule out that the 1.0 standard *may* decide to frob the ring layout somehow, but I'd think it would require a compelling reason. I suggest that's 2.0 material...
Cheers,
Rusty.
Quoted 6 lines · Paolo Bonzini
Il 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;
Quoted 6 lines
Quoted 2 lines
Quoted 1 line
Quoted 1 line
Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
It's defined to yield an integer constant expression in the ISO standard (and I think ANSI too, though that's not at hand):
7.19, para 3:
...offsetof(type, member-designator)
which expands to an integer constant expression that has type
size_t, ...The real question is whether compilers qemu cares about meet the standard (there's some evidence that older compilers fail this). If not, we'll have to define them as raw offsets...
Cheers,
Rusty.
Quoted 6 lines · Paolo Bonzini
Il 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;
Quoted 6 lines
Quoted 2 lines
Quoted 1 line
Quoted 1 line
Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
It's defined to yield an integer constant expression in the ISO standard (and I think ANSI too, though that's not at hand):
7.19, para 3:
...offsetof(type, member-designator)
which expands to an integer constant expression that has type
size_t, ...The real question is whether compilers qemu cares about meet the standard (there's some evidence that older compilers fail this). If not, we'll have to define them as raw offsets...
Cheers,
Rusty.
Paolo Bonzini
maintainerIl 29/05/2013 06:33, Rusty Russell ha scritto:
Quoted 8 lines
Quoted 7 lines · Paolo Bonzini
Il 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
Any good reason not to use offsetof?
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
It's defined to yield an integer constant expression in the ISO standard (and I think ANSI too, though that's not at hand):
It's not in C89. The oldest compiler QEMU cares about is GCC 4.2. I don't know if it has a builtin offsetof, probably it does. But I'm not sure about other users of virtio headers.
Paolo
Quoted 14 lines
7.19, para 3: ...offsetof(type, member-designator) which expands to an integer constant expression that has type size_t, ...The real question is whether compilers qemu cares about meet the standard (there's some evidence that older compilers fail this). If not, we'll have to define them as raw offsets...
Cheers,
Rusty.
Michael S. Tsirkin
maintainerQuoted 15 lines · Paolo Bonzini
Il 29/05/2013 06:33, Rusty Russell ha scritto:
Quoted 14 lines
Quoted 12 lines · Paolo Bonzini
Il 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
Quoted 5 lines
Quoted 2 lines
It's defined to yield an integer constant expression in the ISO standard (and I think ANSI too, though that's not at hand):
It's not in C89. The oldest compiler QEMU cares about is GCC 4.2. I don't know if it has a builtin offsetof, probably it does.
Yes, I think __builtin_offsetof was added in 4.0:
http://gcc.gnu.org/onlinedocs/gcc-4.0.0/gcc/Offsetof.html
Quoted 13 lines
But I'm not sure about other users of virtio headers.
Paolo
Quoted 10 lines
7.19, para 3: ...offsetof(type, member-designator) which expands to an integer constant expression that has type size_t, ...The real question is whether compilers qemu cares about meet the standard (there's some evidence that older compilers fail this). If not, we'll have to define them as raw offsets...
So I think the question is whether we care about GCC 3. We used to when mingw in debian stable was GCC 3, but not anymore ...
Quoted 4 lines
Quoted 4 lines
Cheers,
Rusty.
Michael S. Tsirkin
maintainerQuoted 12 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Review
Quoted 2 lines · Michael S. Tsirkin
(quoted diff) +2
-0+ case offsetof(struct virtio_pci_common_cfg, device_feature_select): + return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
Quoted 6 lines
So AFAICT the question is, do we put the required
#define
VIRTIO_PCI_CFG_FEATURE_SEL\
(offsetof(struct virtio_pci_common_cfg, device_feature_select))etc. in the kernel headers or qemu?
I forgot that we need to validate size (different fields have different sizes so memory core does not validate for us).
And that is much cleaner if we use offsetof directly:
You can see that you are checking the correct size matching the offset, at a glance.
--->
virtio: new layout: add offset validation
---
hw/virtio/virtio-pci.c +38 -0
@@ -467,51 +467,70 @@ static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr, {VirtIOPCIProxy *proxy = opaque;VirtIODevice *vdev = proxy->vdev; +struct virtio_pci_common_cfg cfg;uint64_t low =0xffffffffull ;switch (addr) {case offsetof(struct virtio_pci_common_cfg , device_feature_select): +assert (size ==sizeof cfg.device_feature_select);return proxy->device_feature_select;case offsetof(struct virtio_pci_common_cfg , device_feature): +assert (size ==sizeof cfg.device_feature);/* TODO: 64-bit features */ return proxy->device_feature_select ?0 : proxy->host_features;case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +assert (size ==sizeof cfg.guest_feature_select);return proxy->guest_feature_select;case offsetof(struct virtio_pci_common_cfg , guest_feature): +assert (size ==sizeof cfg.guest_feature);/* TODO: 64-bit features */ return proxy->guest_feature_select ?0 : vdev->guest_features;case offsetof(struct virtio_pci_common_cfg , msix_config): +assert (size ==sizeof cfg.msix_config);return vdev->config_vector;case offsetof(struct virtio_pci_common_cfg , num_queues): +assert (size ==sizeof cfg.num_queues);/* TODO: more exact limit? */ return VIRTIO_PCI_QUEUE_MAX ;case offsetof(struct virtio_pci_common_cfg , device_status): +assert (size ==sizeof cfg.device_status);return vdev->status;/* About a specific virtqueue. */ case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (size ==sizeof cfg.queue_select);return vdev->queue_sel;case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (size ==sizeof cfg.queue_size);return virtio_queue_get_num (vdev, vdev->queue_sel);case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +assert (size ==sizeof cfg.queue_msix_vector);return virtio_queue_vector (vdev, vdev->queue_sel);case offsetof(struct virtio_pci_common_cfg , queue_enable): +assert (size ==sizeof cfg.queue_enable);/* TODO */ return 0 ;case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +assert (size ==sizeof cfg.queue_notify_off);return vdev->queue_sel;case offsetof(struct virtio_pci_common_cfg , queue_desc): +assert (size ==4 );return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low;case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +assert (size ==4 );return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) >>32 ;case offsetof(struct virtio_pci_common_cfg , queue_avail): +assert (size ==4 );return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low;case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +assert (size ==4 );return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) >>32 ;case offsetof(struct virtio_pci_common_cfg , queue_used): +assert (size ==4 );return virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low;case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +assert (size ==4 );return virtio_queue_get_used_addr (vdev, vdev->queue_sel) >>32 ;default :return 0 ;
@@ -523,76 +542,95 @@ static void virtio_pci_config_common_write(void *opaque, hwaddr addr, {VirtIOPCIProxy *proxy = opaque;VirtIODevice *vdev = proxy->vdev; +struct virtio_pci_common_cfg cfg;uint64_t low =0xffffffffull ;uint64_t high = ~low;switch (addr) {case offsetof(struct virtio_pci_common_cfg , device_feature_select): +assert (size ==sizeof cfg.device_feature_select); proxy->device_feature_select = val;break ;case offsetof(struct virtio_pci_common_cfg , device_feature): +assert (size ==sizeof cfg.device_feature);break ;case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +assert (size ==sizeof cfg.guest_feature_select); proxy->guest_feature_select = val;break ;case offsetof(struct virtio_pci_common_cfg , guest_feature): +assert (size ==sizeof cfg.guest_feature);/* TODO: 64-bit features */ if (!proxy->guest_feature_select) {virtio_set_features (vdev, val); }break ;case offsetof(struct virtio_pci_common_cfg , msix_config): +assert (size ==sizeof cfg.msix_config); vdev->config_vector = val;break ;case offsetof(struct virtio_pci_common_cfg , num_queues): +assert (size ==sizeof cfg.num_queues);break ;case offsetof(struct virtio_pci_common_cfg , device_status): +assert (size ==sizeof cfg.device_status);virtio_pci_set_status (proxy, val);break ;/* About a specific virtqueue. */ case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (size ==sizeof cfg.queue_select);assert (val <VIRTIO_PCI_QUEUE_MAX ); vdev->queue_sel = val;break ;case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (size ==sizeof cfg.queue_size);assert (val && val <0x8ffff && !(val & (val -1 )));virtio_queue_set_num (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +assert (size ==sizeof cfg.queue_msix_vector);virtio_queue_set_vector (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_enable): +assert (size ==sizeof cfg.queue_enable);/* TODO */ break ;case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +assert (size ==sizeof cfg.queue_notify_off);break ;case offsetof(struct virtio_pci_common_cfg , queue_desc): +assert (size ==4 ); val &= low; val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & high;virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +assert (size ==4 ); val = val <<32 ; val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low;virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_avail): +assert (size ==4 ); val &= low; val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & high;virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +assert (size ==4 ); val = val <<32 ; val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low;virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_used): +assert (size ==4 ); val &= low; val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & high;virtio_queue_set_used_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +assert (size ==4 ); val = val <<32 ; val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low;virtio_queue_set_used_addr (vdev, vdev->queue_sel, val);
Paolo Bonzini
maintainerIl 29/05/2013 10:24, Michael S. Tsirkin ha scritto:
Quoted 21 lines · Rusty Russell
Quoted 12 lines
Quoted 5 lines · Anthony Liguori
Review
Quoted 2 lines · Michael S. Tsirkin
(quoted diff) +2
-0+ case offsetof(struct virtio_pci_common_cfg, device_feature_select): + return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
Quoted 6 lines
So AFAICT the question is, do we put the required
#define
VIRTIO_PCI_CFG_FEATURE_SEL\
(offsetof(struct virtio_pci_common_cfg, device_feature_select))etc. in the kernel headers or qemu?
Quoted 7 lines
I forgot that we need to validate size (different fields have different sizes so memory core does not validate for us).
And that is much cleaner if we use offsetof directly:
You can see that you are checking the correct size matching the offset, at a glance.
I wonder if we can use and possibly extend Peter Crosthwaite's "register API" to support this better.
Paolo
Quoted 180 lines
--->
virtio: new layout: add offset validation
Signed-off-by Michael S. Tsirkin <[email protected]>---
hw/virtio/virtio-pci.c +38
-0@@ -467,51 +467,70 @@ static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr, {VirtIOPCIProxy *proxy = opaque;VirtIODevice *vdev = proxy->vdev; +struct virtio_pci_common_cfg cfg;uint64_t low =0xffffffffull ;switch (addr) {case offsetof(struct virtio_pci_common_cfg , device_feature_select): +assert (size ==sizeof cfg.device_feature_select);return proxy->device_feature_select;case offsetof(struct virtio_pci_common_cfg , device_feature): +assert (size ==sizeof cfg.device_feature);/* TODO: 64-bit features */ return proxy->device_feature_select ?0 : proxy->host_features;case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +assert (size ==sizeof cfg.guest_feature_select);return proxy->guest_feature_select;case offsetof(struct virtio_pci_common_cfg , guest_feature): +assert (size ==sizeof cfg.guest_feature);/* TODO: 64-bit features */ return proxy->guest_feature_select ?0 : vdev->guest_features;case offsetof(struct virtio_pci_common_cfg , msix_config): +assert (size ==sizeof cfg.msix_config);return vdev->config_vector;case offsetof(struct virtio_pci_common_cfg , num_queues): +assert (size ==sizeof cfg.num_queues);/* TODO: more exact limit? */ return VIRTIO_PCI_QUEUE_MAX ;case offsetof(struct virtio_pci_common_cfg , device_status): +assert (size ==sizeof cfg.device_status);return vdev->status;/* About a specific virtqueue. */ case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (size ==sizeof cfg.queue_select);return vdev->queue_sel;case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (size ==sizeof cfg.queue_size);return virtio_queue_get_num (vdev, vdev->queue_sel);case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +assert (size ==sizeof cfg.queue_msix_vector);return virtio_queue_vector (vdev, vdev->queue_sel);case offsetof(struct virtio_pci_common_cfg , queue_enable): +assert (size ==sizeof cfg.queue_enable);/* TODO */ return 0 ;case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +assert (size ==sizeof cfg.queue_notify_off);return vdev->queue_sel;case offsetof(struct virtio_pci_common_cfg , queue_desc): +assert (size ==4 );return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low;case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +assert (size ==4 );return virtio_queue_get_desc_addr (vdev, vdev->queue_sel) >>32 ;case offsetof(struct virtio_pci_common_cfg , queue_avail): +assert (size ==4 );return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low;case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +assert (size ==4 );return virtio_queue_get_avail_addr (vdev, vdev->queue_sel) >>32 ;case offsetof(struct virtio_pci_common_cfg , queue_used): +assert (size ==4 );return virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low;case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +assert (size ==4 );return virtio_queue_get_used_addr (vdev, vdev->queue_sel) >>32 ;default :return 0 ;@@ -523,76 +542,95 @@ static void virtio_pci_config_common_write(void *opaque, hwaddr addr, {VirtIOPCIProxy *proxy = opaque;VirtIODevice *vdev = proxy->vdev; +struct virtio_pci_common_cfg cfg;uint64_t low =0xffffffffull ;uint64_t high = ~low;switch (addr) {case offsetof(struct virtio_pci_common_cfg , device_feature_select): +assert (size ==sizeof cfg.device_feature_select); proxy->device_feature_select = val;break ;case offsetof(struct virtio_pci_common_cfg , device_feature): +assert (size ==sizeof cfg.device_feature);break ;case offsetof(struct virtio_pci_common_cfg , guest_feature_select): +assert (size ==sizeof cfg.guest_feature_select); proxy->guest_feature_select = val;break ;case offsetof(struct virtio_pci_common_cfg , guest_feature): +assert (size ==sizeof cfg.guest_feature);/* TODO: 64-bit features */ if (!proxy->guest_feature_select) {virtio_set_features (vdev, val); }break ;case offsetof(struct virtio_pci_common_cfg , msix_config): +assert (size ==sizeof cfg.msix_config); vdev->config_vector = val;break ;case offsetof(struct virtio_pci_common_cfg , num_queues): +assert (size ==sizeof cfg.num_queues);break ;case offsetof(struct virtio_pci_common_cfg , device_status): +assert (size ==sizeof cfg.device_status);virtio_pci_set_status (proxy, val);break ;/* About a specific virtqueue. */ case offsetof(struct virtio_pci_common_cfg , queue_select): +assert (size ==sizeof cfg.queue_select);assert (val <VIRTIO_PCI_QUEUE_MAX ); vdev->queue_sel = val;break ;case offsetof(struct virtio_pci_common_cfg , queue_size): +assert (size ==sizeof cfg.queue_size);assert (val && val <0x8ffff && !(val & (val -1 )));virtio_queue_set_num (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_msix_vector): +assert (size ==sizeof cfg.queue_msix_vector);virtio_queue_set_vector (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_enable): +assert (size ==sizeof cfg.queue_enable);/* TODO */ break ;case offsetof(struct virtio_pci_common_cfg , queue_notify_off): +assert (size ==sizeof cfg.queue_notify_off);break ;case offsetof(struct virtio_pci_common_cfg , queue_desc): +assert (size ==4 ); val &= low; val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & high;virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_desc) +4 : +assert (size ==4 ); val = val <<32 ; val |=virtio_queue_get_desc_addr (vdev, vdev->queue_sel) & low;virtio_queue_set_desc_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_avail): +assert (size ==4 ); val &= low; val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & high;virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_avail) +4 : +assert (size ==4 ); val = val <<32 ; val |=virtio_queue_get_avail_addr (vdev, vdev->queue_sel) & low;virtio_queue_set_avail_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_used): +assert (size ==4 ); val &= low; val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & high;virtio_queue_set_used_addr (vdev, vdev->queue_sel, val);break ;case offsetof(struct virtio_pci_common_cfg , queue_used) +4 : +assert (size ==4 ); val = val <<32 ; val |=virtio_queue_get_used_addr (vdev, vdev->queue_sel) & low;virtio_queue_set_used_addr (vdev, vdev->queue_sel, val);
Quoted 6 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +0
-0@@ -467,51 +467,70 @@ static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr, {
hw/virtio/virtio-pci.c static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr, · line 477
Quoted 10 lines
hw/virtio/virtio-pci.c +2
-0VirtIOPCIProxy *proxy = opaque;VirtIODevice *vdev = proxy->vdev; +struct virtio_pci_common_cfg cfg;uint64_t low =0xffffffffull ;switch (addr) {case offsetof(struct virtio_pci_common_cfg , device_feature_select): +assert (size ==sizeof cfg.device_feature_select);return proxy->device_feature_select;
Asserting is definitely the wrong thing here, since the guest can trigger it.
If you really want to use offsetof like this you're going to need to decorate the structs with QEMU_PACKED.
thanks
-- PMM
Quoted 11 lines · Paolo Bonzini
Il 29/05/2013 06:33, Rusty Russell ha scritto:
Quoted 10 lines
Quoted 9 lines · Paolo Bonzini
Il 28/05/2013 19:32, Michael S. Tsirkin ha scritto:
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 4 lines
hw/virtio/virtio-pci.c +4
-0+ +switch (addr) { +case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
Any good reason not to use offsetof?
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
I'm not sure it's portable to use it in case labels. IIRC, the definition ((int)&(((T *)0)->field)) is not a valid C integer constant expression. Laszlo?
It's defined to yield an integer constant expression in the ISO standard (and I think ANSI too, though that's not at hand):
It's not in C89.
It is, see 7.1.6 Common definitions <stddef.h>.
Quoted 2 lines
The oldest compiler QEMU cares about is GCC 4.2. I don't know if it has a builtin offsetof, probably it does.
Talking about nothing else but the ISO C standard(s), it doesn't matter how the C implementation deals with offsetof() internally. "C implementation" in this sense includes both compiler and standard library.
If the standard library header (<stddef.h> or something internal included by it) expands the offsetof() macro to replacement text that does pointer black magic, magic that would be otherwise nonportable or undefined, it is alright as long as the accompanying compiler guarantees that the replacement text will work.
That is, offsetof() gives the C implementor leeway to implement it in wherever "distribution" between compiler and standard library; the main thing is that the C program use the one offsetof() provided by the C implementation.
Of course in the FLOSS world OS distros might mix and match gcc and glibc statistically randomly; normally some documentation should enable the user to put his/her system into ISO C or even SUSv[1234] mode.
(
An interesting example for, say, "non-unified implementation" is "getconf" vs. "c99". I'll pick SUSv3 for this example.
http://pubs.opengroup.org/onlinepubs/000095399/utilities/getconf.html
http://pubs.opengroup.org/onlinepubs/000095399/utilities/c99.html
In a nutshell one can interrogate getconf for CFLAGS, LDFLAGS and LIBS so that c99 builds a program in ILP32_OFF32 / ILP32_OFFBIG / LP64_OFF64 / LPBIG_OFFBIG mode ("programming environment").
On my x86_64 RHEL-6.4 laptop, "getconf" is part of glibc (glibc-common-2.12-1.107.el6.x86_64), while c99 is part of gcc (gcc-4.4.7-3.el6.x86_64).
Supposing I'd like to build a 32-bit program with 64-bit off_t:
getconf _POSIX_V6_ILP32_OFFBIG
1
getconf -v POSIX_V6_ILP32_OFFBIG POSIX_V6_ILP32_OFFBIG_CFLAGS -m32 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
getconf -v POSIX_V6_ILP32_OFFBIG POSIX_V6_ILP32_OFFBIG_LDFLAGS -m32
getconf -v POSIX_V6_ILP32_OFFBIG POSIX_V6_ILP32_OFFBIG_LIBS [empty string]
However if I try to actually build a program using these flags:
#define _XOPEN_SOURCE 600
int main(void) { return 0; }
c99 \
-m32 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 \
-o mytest \
-m32 \
mytest.cI get
/usr/bin/ld: crt1.o: No such file: No such file or directory
collect2: ld returned 1 exit status
unless I install "glibc-devel.i686".
This problem is rooted in "getconf" (a glibc utility) being unaware of RHEL packaging choices: if the system can't guarantee the final c99 invocation to succeed, then the very first "getconf" invocation should write "-1\n" or "undefined\n" to stdout.
(I'm aware that RHEL-6 is not certified UNIX 03; this is just an example for problems caused by various parts of a standard's (quasi-)implementation coming from separate projects. In that sense, caring about the offsetof() internals may have merit.)
)
Laszlo
Michael S. Tsirkin
maintainerQuoted 13 lines · Peter Maydell
Quoted 12 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +1
-0@@ -467,51 +467,70 @@ static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr, {VirtIOPCIProxy *proxy = opaque;VirtIODevice *vdev = proxy->vdev; +struct virtio_pci_common_cfg cfg;uint64_t low =0xffffffffull ;
Quoted 7 lines
hw/virtio/virtio-pci.c static uint64_t virtio_pci_config_common_read(void *opaque, hwaddr addr, · line 477
Quoted 4 lines
hw/virtio/virtio-pci.c +1
-0switch (addr) {case offsetof(struct virtio_pci_common_cfg , device_feature_select): +assert (size ==sizeof cfg.device_feature_select);return proxy->device_feature_select;Asserting is definitely the wrong thing here, since the guest can trigger it.
So?
It's a driver bug. It can reset or crash guest with the same effect, and it likely will if we let it continue.
assert makes sure we don't let it escalate into some
hard to debug security problem.
Quoted 5 lines
If you really want to use offsetof like this you're going to need to decorate the structs with
QEMU_PACKED.thanks
-- PMM
Nope.
These structs are carefully designed not to have any padding.
And if there was a bug and there was some padding, we still can't fix it with PACKED because this structure
is used to interact with the guest code which does not have the packed attribute.
Signature · 1 line
MST
Quoted 5 lines · Michael S. Tsirkin
Quoted 2 lines · Peter Maydell
Asserting is definitely the wrong thing here, since the guest can trigger it.
So?
So "guest should not be able to crash QEMU" is a standard
rule: assert() is for QEMU bugs, not guest bugs. Virtio isn't any different to any other device model.
Quoted 2 lines
It's a driver bug. It can reset or crash guest with the same effect, and it likely will if we let it continue.
Letting a misbehaving guest crash itself is fine. Killing QEMU isn't.
Quoted 2 lines
assert makes sure we don't let it escalate into some hard to debug security problem.
If you want to make guest bugs easier to spot and debug this is what qemu_log_mask(LOG_GUEST_ERROR,...) is for.
Quoted 5 lines
Quoted 2 lines
If you really want to use offsetof like this you're going to need to decorate the structs with
QEMU_PACKED.Nope.
These structs are carefully designed not to have any padding.
...on every compiler and OS combination that QEMU builds for?
Quoted 4 lines
And if there was a bug and there was some padding, we still can't fix it with PACKED because this structure
is used to interact with the guest code which does not have the packed attribute.
The guest code has to use a set of structure offsets and sizes which is fixed by the virtio ABI -- how it implements this is up to the guest (and if it misimplements it that is a guest bug and not our problem). Note also that the ABI is not defined by a set of C source struct definitions (except in as far as they are accompanied by a set of restrictions on alignment, padding, etc that completely determine the numerical alignments and offsets). How QEMU implements the set of offsets and sizes specified by the ABI is definitely our problem, and is exactly what this discussion is about. The simplest and safest way to get the offsets right on all platforms is just to use a set of #defines, which is why that's what almost all of our device models do. Where we do define a struct matching guest memory layout, we tag it with QEMU_PACKED as our best means of ensuring consistency on all hosts.
thanks
-- PMM
Michael S. Tsirkin
maintainerQuoted 9 lines · Peter Maydell
Quoted 5 lines · Michael S. Tsirkin
Quoted 2 lines · Peter Maydell
Asserting is definitely the wrong thing here, since the guest can trigger it.
So?
So "guest should not be able to crash QEMU" is a standard
rule:assert()is for QEMU bugs, not guest bugs.
It's quite likely a QEMU bug -
guests normally do fixed-size accesses so it's hard for them to access a field with a wrong size.
It is guest triggerable but this does not contradict the fact that it never happens in practice.
Quoted 7 lines
Virtio isn't any different to any other device model.
Quoted 2 lines
It's a driver bug. It can reset or crash guest with the same effect, and it likely will if we let it continue.
Letting a misbehaving guest crash itself is fine. Killing QEMU isn't.
*Why* isn't it?
It has the advantage of making sure the misbehaving system is stopped before it does any damage.
Why keep QEMU running even though we know
there's a memory corruption somewhere?
Quoted 5 lines
Quoted 2 lines
assert makes sure we don't let it escalate into some hard to debug security problem.
If you want to make guest bugs easier to spot and debug this is what
qemu_log_mask(LOG_GUEST_ERROR,...)is for.
We really want something that would also stop guest and stop device model as well - we don't know where the bug is.
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
If you really want to use offsetof like this you're going to need to decorate the structs with
QEMU_PACKED.Nope.
These structs are carefully designed not to have any padding....on every compiler and OS combination that QEMU builds for?
Yes. All the way back to EGCS and before.
GCC has been like this for many many years.
Quoted 5 lines
Quoted 4 lines
And if there was a bug and there was some padding, we still can't fix it with PACKED because this structure
is used to interact with the guest code which does not have the packed attribute.
Quoted 8 lines
The guest code has to use a set of structure offsets and sizes which is fixed by the virtio ABI -- how it implements this is up to the guest (and if it misimplements it that is a guest bug and not our problem). Note also that the ABI is not defined by a set of C source struct definitions (except in as far as they are accompanied by a set of restrictions on alignment, padding, etc that completely determine the numerical alignments and offsets).
In practical terms, we should all talk and agree on what's best for driver *and* QEMU, not have QEMU just ignore driver and do it's own thing.
In practical terms, virtio in QEMU should use exactly same code to define layout as virtio in guest. Preferably same header file, we'll get there too, once we comple the discussion over the bikeshed color.
Deviating from driver in random ways is an endless source of hard to debug issues, and it's a very practical consideration because virtio spec is constantly being extended (unlike hardware which is mostly fixed).
Quoted 6 lines
How QEMU implements the set of offsets and sizes specified by the ABI is definitely our problem, and is exactly what this discussion is about. The simplest and safest way to get the offsets right on all platforms is just to use a set of #defines, which is why that's what almost all of our device models do.
Maybe you don't feel safe when you see offsetof.
I review a struct and see fields are aligned
properly at a glance and I feel safe.
But I don't feel safe when I see headers in guest and qemu have different code.
Quoted 6 lines
Where we do define a struct matching
guest memory layout, we tag it withQEMU_PACKEDas our best means of ensuring consistency on all hosts.thanks
-- PMM
Further, packed structures produce terrible code in GCC, in all versions that QEMU cares about :).
Signature · 1 line
MST
Paolo Bonzini
maintainerIl 29/05/2013 14:16, Michael S. Tsirkin ha scritto:
Quoted 9 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
If you really want to use offsetof like this you're going to need to decorate the structs with
QEMU_PACKED.Nope.
These structs are carefully designed not to have any padding....on every compiler and OS combination that QEMU builds for?
Yes. All the way back to EGCS and before.
GCC has been like this for many many years.
I would still prefer to have QEMU_PACKED (or __attribute((__packed__)) in the kernel).
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 4 lines
Quoted 4 lines
And if there was a bug and there was some padding, we still can't fix it with PACKED because this structure
is used to interact with the guest code which does not have the packed attribute.The guest code has to use a set of structure offsets and sizes which is fixed by the virtio ABI -- how it implements this is up to the guest (and if it misimplements it that is a guest bug and not our problem).
On the other hand, encouraging both userspace and guest to reuse a single set of headers means that the bad offset becomes a spec bug more than a guest bug.
Quoted 4 lines
Deviating from driver in random ways is an endless source of hard to debug issues, and it's a very practical consideration because virtio spec is constantly being extended (unlike hardware which is mostly fixed).
Agreed---but the driver should use __attribute__((__packed__)) too.
Paolo
Michael S. Tsirkin
maintainerQuoted 19 lines · Paolo Bonzini
Il 29/05/2013 14:16, Michael S. Tsirkin ha scritto:
Quoted 9 lines
Quoted 7 lines
Quoted 7 lines
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Quoted 2 lines
If you really want to use offsetof like this you're going to need to decorate the structs with
QEMU_PACKED.Nope.
These structs are carefully designed not to have any padding....on every compiler and OS combination that QEMU builds for?
Yes. All the way back to EGCS and before.
GCC has been like this for many many years.I would still prefer to have
QEMU_PACKED(or __attribute((__packed__)) in the kernel).Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 4 lines
And if there was a bug and there was some padding, we still can't fix it with PACKED because this structure
is used to interact with the guest code which does not have the packed attribute.
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
The guest code has to use a set of structure offsets and sizes which is fixed by the virtio ABI -- how it implements this is up to the guest (and if it misimplements it that is a guest bug and not our problem).
On the other hand, encouraging both userspace and guest to reuse a single set of headers means that the bad offset becomes a spec bug more than a guest bug.
Heh
Quoted 8 lines
Quoted 4 lines
Deviating from driver in random ways is an endless source of hard to debug issues, and it's a very practical consideration because virtio spec is constantly being extended (unlike hardware which is mostly fixed).
Agreed---but the driver should use __attribute__((__packed__)) too.
Paolo
For these specific structures I don't mind, we never dereference them anyway. If you do dereference a structure, using packed is generally a mistake. In particular because GCC generates code that is much slower.
You can also get nasty surprises e.g.
struct foo {
char a;
int b;
} __attribute__((packed));struct foo foo;
int *a = &foo.a;
Last line compiles fine but isn't legal C.
So I think there are two cases with packed:
1. it does not change logical behaviour but results in bad code generated
2. it does change logical behaviour and leads to unsafe code
There aren't many good reasons to use packed: if you must treat unaligned data, best to use constants.
Signature · 1 line
MST
Quoted 8 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 2 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +2
-0+case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
I think beauty is in the eye of the beholder here...
Pretty much every device we have has a switch statement like this. Consistency wins when it comes to qualitative arguments like this.
Quoted 4 lines
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
From an API design point of view, here are the problems I see:
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
2) If we every introduce anything like latching, this doesn't work out so well anymore because it's hard to express in a single C structure the register layout at that point. Perhaps a union could be used but padding may make it a bit challenging.
3) It suspect it's harder to review because a subtle change could more easily have broad impact. If someone changed the type of a field from u32 to u16, it changes the offset of every other field. That's not terribly obvious in the patch itself unless you understand how the structure is used elsewhere.
This may not be a problem for virtio because we all understand that the structures are part of an ABI, but if we used this pattern more in QEMU, it would be a lot less obvious.
Quoted 6 lines
So AFAICT the question is, do we put the required
#define
VIRTIO_PCI_CFG_FEATURE_SEL\
(offsetof(struct virtio_pci_common_cfg, device_feature_select))etc. in the kernel headers or qemu?
I'm pretty sure we would end up just having our own integer defines. We carry our own virtio headers today because we can't easily import the kernel headers.
Quoted 7 lines
Quoted 1 line
Haven't looked at the proposed new ring layout yet.
No change, but there's an open question on whether we should nail it to little endian (or define the endian by the transport).
Of course, I can't rule out that the 1.0 standard *may* decide to frob the ring layout somehow,
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no?
Any new config layout would be 2.0 material, right?
Re: the new config layout, I don't think we would want to use it for anything but new devices. Forcing a guest driver change is a really big deal and I see no reason to do that unless there's a compelling reason to.
So we're stuck with the 1.0 config layout for a very long time.
Regards,
Anthony Liguori
Quoted 10 lines
but I'd think it would require a compelling
reason. I suggest that's 2.0 material...Cheers,
Rusty.--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 19 lines · Anthony Liguori
Quoted 8 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
hw/virtio/virtio-pci.c static void virtio_pci_config_write(void *opaque, hwaddr addr, · line 475
Quoted 2 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +2
-0+case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
I think beauty is in the eye of the beholder here...
Pretty much every device we have has a switch statement like this. Consistency wins when it comes to qualitative arguments like this.
Quoted 2 lines
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
Quoted 6 lines
Quoted 1 line
From an API design point of view, here are the problems I see:
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
You expect a compiler to pad this structure:
struct foo {
uint8_t a;
uint8_t b;
uint16_t c;
uint32_t d;
};I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
Quoted 4 lines
2) If we every introduce anything like latching, this doesn't work out so well anymore because it's hard to express in a single C structure the register layout at that point. Perhaps a union could be used but padding may make it a bit challenging.
Then linux won't use it either.
Quoted 9 lines
3) It suspect it's harder to review because a subtle change could more easily have broad impact. If someone changed the type of a field from u32 to u16, it changes the offset of every other field. That's not terribly obvious in the patch itself unless you understand how the structure is used elsewhere.
This may not be a problem for virtio because we all understand that the structures are part of an ABI, but if we used this pattern more in QEMU, it would be a lot less obvious.
So let's not use it more in QEMU.
Quoted 10 lines
Quoted 6 lines
So AFAICT the question is, do we put the required
#define
VIRTIO_PCI_CFG_FEATURE_SEL\
(offsetof(struct virtio_pci_common_cfg, device_feature_select))etc. in the kernel headers or qemu?
I'm pretty sure we would end up just having our own integer defines. We carry our own virtio headers today because we can't easily import the kernel headers.
Yes we can easily import them.
And at least we copy headers verbatim.
Quoted 11 lines
Quoted 7 lines
Quoted 1 line
Haven't looked at the proposed new ring layout yet.
No change, but there's an open question on whether we should nail it to little endian (or define the endian by the transport).
Of course, I can't rule out that the 1.0 standard *may* decide to frob the ring layout somehow,
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Quoted 2 lines
Re: the new config layout, I don't think we would want to use it for anything but new devices. Forcing a guest driver change
There's no forcing.
If you look at the patches closely, you will see that we still support the old layout on BAR0.
Quoted 3 lines
is a really big
deal and I see no reason to do that unless there's a compelling reason to.
There are many a compelling reasons, and they are well known limitations of virtio PCI:
- PCI spec compliance (madates device operation with IO memory disabled).
- support 64 bit addressing
- add more than 32 feature bits.
- individually disable queues.
- sanely support cross-endian systems.
- support very small (<1 PAGE) for virtio rings.
- support a separate page for each vq kick.
- make each device place config at flexible offset.
Addressing any one of these would cause us to add a substantially new way to operate virtio devices.
And since it's a guest change anyway, it seemed like a good time to do the new layout and fix everything in one go.
And they are needed like yesterday.
Quoted 5 lines
So we're stuck with the 1.0 config layout for a very long time.
Regards,
Anthony Liguori
Absolutely. This patch let us support both which will allow for a gradual transition over the next 10 years or so.
Quoted 9 lines
Quoted 9 lines
reason. I suggest that's 2.0 material...
Cheers,
Rusty.--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 15 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
Structure alignment is a platform ABI choice. Oddly enough people choose operating systems on more criteria than the amount of padding their ABI mandates in structures.
In any case it's really tedious to read a structure and wade through working out whether it's going to have all its members naturally aligned, especially once it's more than a handful of members in size. And getting it wrong is "silent portability problem".
thanks
-- PMM
Quoted 15 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
Structure alignment is a platform ABI choice. Oddly enough people choose operating systems on more criteria than the amount of padding their ABI mandates in structures.
In any case it's really tedious to read a structure and wade through working out whether it's going to have all its members naturally aligned, especially once it's more than a handful of members in size. And getting it wrong is "silent portability problem".
thanks
-- PMM
Paolo Bonzini
maintainerIl 29/05/2013 15:24, Michael S. Tsirkin ha scritto:
Quoted 9 lines
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
You know the virtio-pci config structures are padded, but not all of them are. For example, virtio_balloon_stat is not padded and indeed has an __attribute__((__packed__)) in the spec.
For this reason I prefer to have the attribute everywhere. So people don't have to wonder why it's here and not there.
Paolo
Michael S. Tsirkin
maintainerQuoted 14 lines · Paolo Bonzini
Il 29/05/2013 15:24, Michael S. Tsirkin ha scritto:
Quoted 9 lines
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
Quoted 8 lines
You know the virtio-pci config structures are padded, but not all of them are. For example,
virtio_balloon_statis not padded and indeed has an __attribute__((__packed__)) in the spec.For this reason I prefer to have the attribute everywhere. So people don't have to wonder why it's here and not there.
Paolo
FWIW I think it was a mistake to lay the balloon structure out like that. We should have padded it manually. __attribute__((__packed__)) is really easy to misuse.
If you get into such a situation, just use offset enums ...
Signature · 1 line
MST
Quoted 15 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Quoted 8 lines
Quoted 3 lines
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Adding a new BAR most certainly requires bumping the revision ID or changing the device ID, no?
Didn't we run into this problem with the virtio-win drivers with just the BAR size changing?
Quoted 12 lines
Quoted 2 lines
Re: the new config layout, I don't think we would want to use it for anything but new devices. Forcing a guest driver change
There's no forcing.
If you look at the patches closely, you will see that we still support the old layout on BAR0.Quoted 3 lines
is a really big
deal and I see no reason to do that unless there's a compelling reason to.
Quoted 5 lines
There are many a compelling reasons, and they are well known limitations of virtio PCI:
- PCI spec compliance (madates device operation with IO memory disabled).
PCI express spec. We are fully compliant with the PCI spec. And what's the user visible advantage of pointing an emulated virtio device behind a PCI-e bus verses a legacy PCI bus?
This is a very good example because if we have to disable BAR0, then it's an ABI breaker plan and simple.
Quoted 1 line
- support 64 bit addressing
We currently support 44-bit addressing for the ring. While I agree we need to bump it, there's no immediate problem with 44-bit addressing.
Quoted 6 lines
- add more than 32 feature bits.
- individually disable queues.
- sanely support cross-endian systems.
- support very small (<1 PAGE) for virtio rings.
- support a separate page for each vq kick.
- make each device place config at flexible offset.
None of these things are holding us back today.
I'm not saying we shouldn't introduce a new device. But adoption of that device will be slow and realistically will be limited to new devices only.
We'll be supporting both devices for a very, very long time.
Compatibility is the fundamental value that we provide. We need to go out of our way to make sure that existing guests work and work as well as possible.
Sticking virtio devices behind a PCI-e bus just for the hell of it isn't a compelling reason to break existing guests.
Regards,
Anthony Liguori
Quoted 31 lines
Addressing any one of these would cause us to add a substantially new way to operate virtio devices.
And since it's a guest change anyway, it seemed like a good time to do the new layout and fix everything in one go.
And they are needed like yesterday.
Quoted 5 lines
So we're stuck with the 1.0 config layout for a very long time.
Regards,
Anthony Liguori
Absolutely. This patch let us support both which will allow for a gradual transition over the next 10 years or so.
Quoted 9 lines
Quoted 9 lines
reason. I suggest that's 2.0 material...
Cheers,
Rusty.--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 15 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Quoted 8 lines
Quoted 3 lines
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Adding a new BAR most certainly requires bumping the revision ID or changing the device ID, no?
Didn't we run into this problem with the virtio-win drivers with just the BAR size changing?
Quoted 12 lines
Quoted 2 lines
Re: the new config layout, I don't think we would want to use it for anything but new devices. Forcing a guest driver change
There's no forcing.
If you look at the patches closely, you will see that we still support the old layout on BAR0.Quoted 3 lines
is a really big
deal and I see no reason to do that unless there's a compelling reason to.
Quoted 5 lines
There are many a compelling reasons, and they are well known limitations of virtio PCI:
- PCI spec compliance (madates device operation with IO memory disabled).
PCI express spec. We are fully compliant with the PCI spec. And what's the user visible advantage of pointing an emulated virtio device behind a PCI-e bus verses a legacy PCI bus?
This is a very good example because if we have to disable BAR0, then it's an ABI breaker plan and simple.
Quoted 1 line
- support 64 bit addressing
We currently support 44-bit addressing for the ring. While I agree we need to bump it, there's no immediate problem with 44-bit addressing.
Quoted 6 lines
- add more than 32 feature bits.
- individually disable queues.
- sanely support cross-endian systems.
- support very small (<1 PAGE) for virtio rings.
- support a separate page for each vq kick.
- make each device place config at flexible offset.
None of these things are holding us back today.
I'm not saying we shouldn't introduce a new device. But adoption of that device will be slow and realistically will be limited to new devices only.
We'll be supporting both devices for a very, very long time.
Compatibility is the fundamental value that we provide. We need to go out of our way to make sure that existing guests work and work as well as possible.
Sticking virtio devices behind a PCI-e bus just for the hell of it isn't a compelling reason to break existing guests.
Regards,
Anthony Liguori
Quoted 31 lines
Addressing any one of these would cause us to add a substantially new way to operate virtio devices.
And since it's a guest change anyway, it seemed like a good time to do the new layout and fix everything in one go.
And they are needed like yesterday.
Quoted 5 lines
So we're stuck with the 1.0 config layout for a very long time.
Regards,
Anthony Liguori
Absolutely. This patch let us support both which will allow for a gradual transition over the next 10 years or so.
Quoted 9 lines
Quoted 9 lines
reason. I suggest that's 2.0 material...
Cheers,
Rusty.--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Paolo Bonzini
Il 29/05/2013 15:24, Michael S. Tsirkin ha scritto:
Quoted 9 lines
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 7 lines
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
You know the virtio-pci config structures are padded, but not all of them are. For example,
virtio_balloon_statis not padded and indeed has an __attribute__((__packed__)) in the spec.
Not that these structures are actually used for something.
We store the config in these structures so they are actually used for something.
The proposed structures only serve as a way to express offsets. You would never actually have a variable of this type.
Regards,
Anthony Liguori
Quoted 9 lines
For this reason I prefer to have the attribute everywhere. So people don't have to wonder why it's here and not there.
Paolo
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Paolo Bonzini
Il 29/05/2013 15:24, Michael S. Tsirkin ha scritto:
Quoted 9 lines
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 7 lines
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
You know the virtio-pci config structures are padded, but not all of them are. For example,
virtio_balloon_statis not padded and indeed has an __attribute__((__packed__)) in the spec.
Not that these structures are actually used for something.
We store the config in these structures so they are actually used for something.
The proposed structures only serve as a way to express offsets. You would never actually have a variable of this type.
Regards,
Anthony Liguori
Quoted 9 lines
For this reason I prefer to have the attribute everywhere. So people don't have to wonder why it's here and not there.
Paolo
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 17 lines · Anthony Liguori
Quoted 15 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };
Quoted 6 lines
Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Great. Let's make the bikeshed blue then?
Quoted 11 lines
Quoted 8 lines
Quoted 3 lines
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Adding a new BAR most certainly requires bumping the revision ID or changing the device ID, no?
No, why would it?
If a device dropped BAR0, that would be a good reason to bump revision ID.
We don't do this yet.
Quoted 2 lines
Didn't we run into this problem with the virtio-win drivers with just the BAR size changing?
Because they had a bug: they validated BAR0 size. AFAIK they don't care what happens with other bars.
Quoted 15 lines
Quoted 15 lines
Quoted 2 lines
Re: the new config layout, I don't think we would want to use it for anything but new devices. Forcing a guest driver change
There's no forcing.
If you look at the patches closely, you will see that we still support the old layout on BAR0.Quoted 3 lines
is a really big
deal and I see no reason to do that unless there's a compelling reason to.There are many a compelling reasons, and they are well known limitations of virtio PCI:
Quoted 6 lines
Quoted 2 lines
- PCI spec compliance (madates device operation with IO memory disabled).
PCI express spec. We are fully compliant with the PCI spec. And what's the user visible advantage of pointing an emulated virtio device behind a PCI-e bus verses a legacy PCI bus?
Native hotplug support.
Quoted 2 lines
This is a very good example because if we have to disable BAR0, then it's an ABI breaker plan and simple.
Not we. The BIOS can disable IO BAR: it can do this already but the device won't be functional.
Quoted 4 lines
Quoted 1 line
- support 64 bit addressing
We currently support 44-bit addressing for the ring. While I agree we need to bump it, there's no immediate problem with 44-bit addressing.
I heard developers (though not users) complaining.
Quoted 8 lines
Quoted 6 lines
- add more than 32 feature bits.
- individually disable queues.
- sanely support cross-endian systems.
- support very small (<1 PAGE) for virtio rings.
- support a separate page for each vq kick.
- make each device place config at flexible offset.None of these things are holding us back today.
All of them do, to bigger or lesser degree.
Quoted 5 lines
I'm not saying we shouldn't introduce a new device. But adoption of that device will be slow and realistically will be limited to new devices only.
We'll be supporting both devices for a very, very long time.
This is true for any new feature. What are you trying to say here? We won't add new features to old config: for once, we have run out of feature bits.
Quoted 3 lines
Compatibility is the fundamental value that we provide. We need to go out of our way to make sure that existing guests work and work as well as possible.
What are you trying to say?
There's nothing here that breaks compatibility.
Have you looked at the patch?
I'm wasting my time arguing on the mailing list, but once I tear myself away from this occupation, I intend to verify that I can run an old guest on qemu with this patch without issues.
Quoted 6 lines
Sticking virtio devices behind a PCI-e bus just for the hell of it isn't a compelling reason to break existing guests.
Regards,
Anthony Liguori
That's why my patch does not break existing guests.
Quoted 32 lines
Quoted 32 lines
Addressing any one of these would cause us to add a substantially new way to operate virtio devices.
And since it's a guest change anyway, it seemed like a good time to do the new layout and fix everything in one go.
And they are needed like yesterday.
Quoted 5 lines
So we're stuck with the 1.0 config layout for a very long time.
Regards,
Anthony Liguori
Absolutely. This patch let us support both which will allow for a gradual transition over the next 10 years or so.
Quoted 9 lines
Quoted 9 lines
reason. I suggest that's 2.0 material...
Cheers,
Rusty.--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Paolo Bonzini
maintainerIl 29/05/2013 16:30, Michael S. Tsirkin ha scritto:
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Great. Let's make the bikeshed blue then?
Yes, let's make it blue, but please promise to check into Peter's register API so we can remove the case offsetof.
I checked that it works on RHEL5, which is 4.1 and probably the oldest compiler we care about (any other 4.1 lacks the __sync builtins; upstream added them in 4.2).
Paolo
Michael S. Tsirkin
maintainerQuoted 7 lines · Paolo Bonzini
Il 29/05/2013 16:30, Michael S. Tsirkin ha scritto:
Quoted 3 lines
Quoted 2 lines
Quoted 2 lines
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Great. Let's make the bikeshed blue then?
Yes, let's make it blue, but please promise to check into Peter's register API so we can remove the case offsetof.
I think it's overkill here. virtio was designed to be easy to implement with a simple switch statement.
Quoted 5 lines
I checked that it works on RHEL5, which is 4.1 and probably the oldest compiler we care about (any other 4.1 lacks the
__syncbuiltins; upstream added them in 4.2).Paolo
Are you sure? Documentation says 4.0 ...
Signature · 1 line
MST
Quoted 10 lines · Michael S. Tsirkin
Quoted 7 lines · Anthony Liguori
Quoted 3 lines · Michael S. Tsirkin
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Great. Let's make the bikeshed blue then?
It's fun to argue about stuff like this and I certainly have an opinion, but I honestly don't care all that much about the offsetof thing. However...
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 4 lines
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Quoted 4 lines
Quoted 2 lines
Adding a new BAR most certainly requires bumping the revision ID or changing the device ID, no?
No, why would it?
If we change the programming interface for a device in a way that is incompatible, we are required to change the revision ID and/or device ID.
Quoted 3 lines
If a device dropped BAR0, that would be a good reason to bump revision ID.
We don't do this yet.
But we have to drop BAR0 to put it behind a PCI express bus, right?
If that's the case, then device that's exposed on the PCI express bus must use a different device ID and/or revision ID.
That means a new driver is needed in the guest.
Quoted 5 lines
Quoted 2 lines
Didn't we run into this problem with the virtio-win drivers with just the BAR size changing?
Because they had a bug: they validated BAR0 size. AFAIK they don't care what happens with other bars.
I think there's a grey area with respect to the assumptions a device can make about the programming interface.
But very concretely, we cannot expose virtio-pci-net via PCI express with BAR0 disabled because that will result in existing virtio-pci Linux drivers breaking.
Quoted 2 lines
Not we. The BIOS can disable IO BAR: it can do this already but the device won't be functional.
But the only way to expose the device over PCI express is to disable the IO BAR, right?
Regards,
Anthony Liguori
Quoted 10 lines · Michael S. Tsirkin
Quoted 7 lines · Anthony Liguori
Quoted 3 lines · Michael S. Tsirkin
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Great. Let's make the bikeshed blue then?
It's fun to argue about stuff like this and I certainly have an opinion, but I honestly don't care all that much about the offsetof thing. However...
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 4 lines
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Quoted 4 lines
Quoted 2 lines
Adding a new BAR most certainly requires bumping the revision ID or changing the device ID, no?
No, why would it?
If we change the programming interface for a device in a way that is incompatible, we are required to change the revision ID and/or device ID.
Quoted 3 lines
If a device dropped BAR0, that would be a good reason to bump revision ID.
We don't do this yet.
But we have to drop BAR0 to put it behind a PCI express bus, right?
If that's the case, then device that's exposed on the PCI express bus must use a different device ID and/or revision ID.
That means a new driver is needed in the guest.
Quoted 5 lines
Quoted 2 lines
Didn't we run into this problem with the virtio-win drivers with just the BAR size changing?
Because they had a bug: they validated BAR0 size. AFAIK they don't care what happens with other bars.
I think there's a grey area with respect to the assumptions a device can make about the programming interface.
But very concretely, we cannot expose virtio-pci-net via PCI express with BAR0 disabled because that will result in existing virtio-pci Linux drivers breaking.
Quoted 2 lines
Not we. The BIOS can disable IO BAR: it can do this already but the device won't be functional.
But the only way to expose the device over PCI express is to disable the IO BAR, right?
Regards,
Anthony Liguori
Michael S. Tsirkin
maintainerQuoted 28 lines · Anthony Liguori
Quoted 10 lines · Michael S. Tsirkin
Quoted 7 lines · Anthony Liguori
Quoted 3 lines · Michael S. Tsirkin
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
There are other responses in the thread here and I don't really care to bikeshed on this issue.
Great. Let's make the bikeshed blue then?
It's fun to argue about stuff like this and I certainly have an opinion, but I honestly don't care all that much about the offsetof thing. However...
Quoted 10 lines
Quoted 10 lines
Quoted 9 lines
Quoted 4 lines
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no? Any new config layout would be 2.0 material, right?
Not as it's currently planned. Devices can choose to support a legacy layout in addition to the new one, and if you look at the patch you will see that that is exactly what it does.
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Adding a new BAR most certainly requires bumping the revision ID or changing the device ID, no?
No, why would it?
If we change the programming interface for a device in a way that is incompatible, we are required to change the revision ID and/or device ID.
It's compatible.
Quoted 5 lines
Quoted 3 lines
If a device dropped BAR0, that would be a good reason to bump revision ID.
We don't do this yet.But we have to drop BAR0 to put it behind a PCI express bus, right?
No, no.
The PCIe spec states that failure to allocate an I/O BAR should still allow the device to function.
That's *guest* allocating an I/O BAR.
Quoted 21 lines
If that's the case, then device that's exposed on the PCI express bus must use a different device ID and/or revision ID.
That means a new driver is needed in the guest.
Quoted 5 lines
Quoted 2 lines
Didn't we run into this problem with the virtio-win drivers with just the BAR size changing?
Because they had a bug: they validated BAR0 size. AFAIK they don't care what happens with other bars.
I think there's a grey area with respect to the assumptions a device can make about the programming interface.
But very concretely, we cannot expose virtio-pci-net via PCI express with BAR0 disabled because that will result in existing virtio-pci Linux drivers breaking.
Quoted 2 lines
Not we. The BIOS can disable IO BAR: it can do this already but the device won't be functional.
Quoted 6 lines
But the only way to expose the device over PCI express is to disable the IO BAR, right?
Regards,
Anthony Liguori
No :)
Michael S. Tsirkin
maintainerQuoted 5 lines · Anthony Liguori
Quoted 2 lines
Not we. The BIOS can disable IO BAR: it can do this already but the device won't be functional.
But the only way to expose the device over PCI express is to disable the IO BAR, right?
I think this is the source of the misunderstanding:
1.3.2.2. PCI Express Endpoint Rules
...
PCI Express Endpoint must not depend on operating system allocation of I/O resources claimed through BAR(s).
The real meaning here is *not* that an Endpoint should not have an I/O BAR. An Endpoint can have an I/O BAR, and PCI Express spec supports I/O transactions. The meaning is that an Endpoint should work
even if *the OS* decides to disable the I/O BAR.
Note: it's up to the guest. We support I/O
as a device with full compatibility for old guests, but must be prepared to handle a guest which does not enable I/O.
This likely means that as time goes on,
future OSes will start disabling I/O BARs,
relying on this not hurting functionality.
Signature · 1 line
MST
Quoted 7 lines · Anthony Liguori
Quoted 5 lines · Rusty Russell
Quoted 4 lines · Anthony Liguori
Quoted 2 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +2
-0+case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
I think beauty is in the eye of the beholder here...
Pretty much every device we have has a switch statement like this. Consistency wins when it comes to qualitative arguments like this.
I was agreeing with you here, actually.
Quoted 9 lines
Quoted 4 lines
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
From an API design point of view, here are the problems I see:
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
[ I argued in detail here, then deleted. Damn bikeshedding... ]
I think the best thing is to add the decoded integer versions as macros, and have a heap of BUILD_BUG_ON() in the kernel source to make sure they match.
Quoted 5 lines
3) It suspect it's harder to review because a subtle change could more easily have broad impact. If someone changed the type of a field from u32 to u16, it changes the offset of every other field. That's not terribly obvious in the patch itself unless you understand how the structure is used elsewhere.
MST's patch just did this, so point taken. (MST: I'm going to combine the cfg_type and bar bytes to fix this, patch coming).
Quoted 8 lines
Quoted 5 lines
No change, but there's an open question on whether we should nail it to little endian (or define the endian by the transport).
Of course, I can't rule out that the 1.0 standard *may* decide to frob the ring layout somehow,
Well, given that virtio is widely deployed today, I would think the 1.0 standard should strictly reflect what's deployed today, no?
That will be up to the committee. I think we want to fix some obvious pain points, though qemu will not benefit from them in the next 5 years.
Quoted 4 lines
Any new config layout would be 2.0 material, right?
Re: the new config layout, I don't think we would want to use it for anything but new devices.
There are enough concrete reasons that I think we want it for existing devices:
1) Negotiated ring size/alignment. Coreboot wants smaller, others want larger.
2) Remove assertion that it has to be an I/O bar. PowerPC wants this.
3) Notification location flexibility. MST wanted this for performance.
4) More feature bits.
Quoted 5 lines
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
It's a delicate balancing act. My plan is to accompany any changes in the standard with a qemu implementation, so we can see how painful those changes are. And if there are performance implications, measure them.
Cheers,
Rusty.
Michael S. Tsirkin
maintainerQuoted 29 lines · Rusty Russell
Quoted 7 lines · Anthony Liguori
Quoted 5 lines · Rusty Russell
Quoted 4 lines · Anthony Liguori
Quoted 2 lines · Michael S. Tsirkin
hw/virtio/virtio-pci.c +2
-0+case offsetof(struct virtio_pci_common_cfg , device_feature_select): +return proxy->device_feature_select;Quoted 8 lines
Quoted 3 lines
Quoted 1 line
Oh dear no... Please use defines like the rest of QEMU.
It is pretty ugly.
I think beauty is in the eye of the beholder here...
Pretty much every device we have has a switch statement like this. Consistency wins when it comes to qualitative arguments like this.
I was agreeing with you here, actually.
Quoted 9 lines
Quoted 4 lines
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
From an API design point of view, here are the problems I see:
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
Quoted 5 lines
[ I argued in detail here, then deleted. Damn bikeshedding... ]
I think the best thing is to add the decoded integer versions as macros, and have a heap of
BUILD_BUG_ON()in the kernel source to make sure they match.
On the qemu side, fine with me, all I want is ability to stay close to kernel headers.
Let's have XXX_SIZE macros as well, so access size can be easily validated?
On the kernel side, for 'struct virtio_pci_cap', since we only ever do offsetof on this struct, it might be easier to just use the numeric constants directly.
Signature · 1 line
MST
Michael S. Tsirkin
maintainerQuoted 14 lines · Paolo Bonzini
Il 29/05/2013 15:24, Michael S. Tsirkin ha scritto:
Quoted 9 lines
You expect a compiler to pad this structure:
struct foo { uint8_t a; uint8_t b; uint16_t c; uint32_t d; };Quoted 3 lines
I'm guessing any compiler that decides to waste memory in this way will quickly get dropped by users and then we won't worry about building QEMU with it.
Quoted 8 lines
You know the virtio-pci config structures are padded, but not all of them are. For example,
virtio_balloon_statis not padded and indeed has an __attribute__((__packed__)) in the spec.For this reason I prefer to have the attribute everywhere. So people don't have to wonder why it's here and not there.
Paolo
BTW we don't even do this consistently everywhere in QEMU.
It would have been better to have a rule to avoid packed as much as possible, then we'd have found the misaligned field bug in balloon before it's too late.
That would be a good rule to adopt I think:
any pack if something is misaligned, and document
the reason.
Signature · 1 line
MST
Michael S. Tsirkin
maintainerQuoted 10 lines · Rusty Russell
Quoted 9 lines
Quoted 4 lines
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
From an API design point of view, here are the problems I see:
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
Quoted 5 lines
[ I argued in detail here, then deleted. Damn bikeshedding... ]
I think the best thing is to add the decoded integer versions as macros, and have a heap of
BUILD_BUG_ON()in the kernel source to make sure they match.
Hmm you want to have both in the header?
That would confuse users for sure :)
I guess we could put in a big comment explaning
why do we have both, and which version should
userspace use. I tried to write it:
/*
* On all known C compilers, you can use the
* offsetof macro to find the correct offset of each field -
* if your compiler doesn't implement this correctly, use the
* integer versions below, instead.
* In that case please don't use the structures above, to avoid confusion. */
Signature · 1 line
MST
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
Quoted 11 lines
It's a delicate balancing act. My plan is to accompany any changes in the standard with a qemu implementation, so we can see how painful those changes are. And if there are performance implications, measure them.
Cheers,
Rusty.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 14 lines · Anthony Liguori
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
Quoted 7 lines
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
If you look at my patches you'll see how it works. Basically old guests use BAR0 new ones don't, so it's easy: BAR0 access means legacy guest. Only started testing but things seem to work fine with old guests so far.
I think we need a spec, not just driver code.
Rusty what's the plan? Want me to write it?
Quoted 11 lines
Quoted 11 lines
It's a delicate balancing act. My plan is to accompany any changes in the standard with a qemu implementation, so we can see how painful those changes are. And if there are performance implications, measure them.
Cheers,
Rusty.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 25 lines · Michael S. Tsirkin
Quoted 10 lines · Rusty Russell
Quoted 9 lines
Quoted 4 lines
Yet the structure definitions are descriptive, capturing layout, size and endianness in natural a format readable by any C programmer.
From an API design point of view, here are the problems I see:
1) C makes no guarantees about structure layout beyond the first member. Yes, if it's naturally aligned or has a packed attribute, GCC does the right thing. But this isn't kernel land anymore, portability matters and there are more compilers than GCC.
Quoted 5 lines
[ I argued in detail here, then deleted. Damn bikeshedding... ]
I think the best thing is to add the decoded integer versions as macros, and have a heap of
BUILD_BUG_ON()in the kernel source to make sure they match.Hmm you want to have both in the header?
That would confuse users for sure :)
I guess we could put in a big comment explaning
why do we have both, and which version should
userspace use. I tried to write it:
Quoted 7 lines
/*
* On all known C compilers, you can use the
* offsetof macro to find the correct offset of each field -
* if your compiler doesn't implement this correctly, use the
* integer versions below, instead.
* In that case please don't use the structures above, to avoid confusion. */
My version was a little less verbose :)
Subject: virtio_pci: macros for PCI layout offsets.
QEMU wants it, so why not? Trust, but verify.
drivers/virtio/virtio_pci_modern.c +60 -0
@@ -453,6 +453,64 @@ static void __iomem *map_capability(struct pci_dev *dev, int off, size_t minlen,return p; } +/* This is part of the ABI. Don't screw with it. */ +static inline void check_offsets (void ) +{ +/* Note: disk space was harmed in compilation of this function. */ +BUILD_BUG_ON (VIRTIO_PCI_CAP_VNDR != + offsetof(struct virtio_pci_cap , cap_vndr)); +BUILD_BUG_ON (VIRTIO_PCI_CAP_NEXT != + offsetof(struct virtio_pci_cap , cap_next)); +BUILD_BUG_ON (VIRTIO_PCI_CAP_LEN != + offsetof(struct virtio_pci_cap , cap_len)); +BUILD_BUG_ON (VIRTIO_PCI_CAP_TYPE_AND_BAR != + offsetof(struct virtio_pci_cap , type_and_bar)); +BUILD_BUG_ON (VIRTIO_PCI_CAP_OFFSET != + offsetof(struct virtio_pci_cap , offset)); +BUILD_BUG_ON (VIRTIO_PCI_CAP_LENGTH != + offsetof(struct virtio_pci_cap , length)); +BUILD_BUG_ON (VIRTIO_PCI_NOTIFY_CAP_MULT != + offsetof(struct virtio_pci_notify_cap , + notify_off_multiplier)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_DFSELECT != + offsetof(struct virtio_pci_common_cfg , + device_feature_select)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_DF != + offsetof(struct virtio_pci_common_cfg , device_feature)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_GFSELECT != + offsetof(struct virtio_pci_common_cfg , + guest_feature_select)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_GF != + offsetof(struct virtio_pci_common_cfg , guest_feature)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_MSIX != + offsetof(struct virtio_pci_common_cfg , msix_config)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_NUMQ != + offsetof(struct virtio_pci_common_cfg , num_queues)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_STATUS != + offsetof(struct virtio_pci_common_cfg , device_status)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_SELECT != + offsetof(struct virtio_pci_common_cfg , queue_select)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_SIZE != + offsetof(struct virtio_pci_common_cfg , queue_size)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_MSIX != + offsetof(struct virtio_pci_common_cfg , queue_msix_vector)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_ENABLE != + offsetof(struct virtio_pci_common_cfg , queue_enable)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_NOFF != + offsetof(struct virtio_pci_common_cfg , queue_notify_off)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_DESCLO != + offsetof(struct virtio_pci_common_cfg , queue_desc_lo)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_DESCHI != + offsetof(struct virtio_pci_common_cfg , queue_desc_hi)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_AVAILLO != + offsetof(struct virtio_pci_common_cfg , queue_avail_lo)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_AVAILHI != + offsetof(struct virtio_pci_common_cfg , queue_avail_hi)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_USEDLO != + offsetof(struct virtio_pci_common_cfg , queue_used_lo)); +BUILD_BUG_ON (VIRTIO_PCI_COMMON_Q_USEDHI != + offsetof(struct virtio_pci_common_cfg , queue_used_hi)); +}/* the PCI probing function */ static int virtio_pci_probe(struct pci_dev *pci_dev,
@@ -461,6 +519,8 @@ static int virtio_pci_probe(struct pci_dev *pci_dev,struct virtio_pci_device *vp_dev;int err, common, isr, notify, device; +check_offsets (); +/* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */ if (pci_dev->device <0x1000 || pci_dev->device >0x103f )return -ENODEV ;
include/uapi/linux/virtio_pci.h +30 -0
@@ -171,4 +171,34 @@ struct virtio_pci_common_cfg {__le32 queue_used_lo;/* read-write */ __le32 queue_used_hi; /* read-write */ }; + +/* Macro versions of offsets for the Old Timers! */ +#define VIRTIO_PCI_CAP_VNDR 0 +#define VIRTIO_PCI_CAP_NEXT 1 +#define VIRTIO_PCI_CAP_LEN 2 +#define VIRTIO_PCI_CAP_TYPE_AND_BAR 3 +#define VIRTIO_PCI_CAP_OFFSET 4 +#define VIRTIO_PCI_CAP_LENGTH 8 + +#define VIRTIO_PCI_NOTIFY_CAP_MULT 12 + +#define VIRTIO_PCI_COMMON_DFSELECT 0 +#define VIRTIO_PCI_COMMON_DF 4 +#define VIRTIO_PCI_COMMON_GFSELECT 8 +#define VIRTIO_PCI_COMMON_GF 12 +#define VIRTIO_PCI_COMMON_MSIX 16 +#define VIRTIO_PCI_COMMON_NUMQ 18 +#define VIRTIO_PCI_COMMON_STATUS 20 +#define VIRTIO_PCI_COMMON_Q_SELECT 22 +#define VIRTIO_PCI_COMMON_Q_SIZE 24 +#define VIRTIO_PCI_COMMON_Q_MSIX 26 +#define VIRTIO_PCI_COMMON_Q_ENABLE 28 +#define VIRTIO_PCI_COMMON_Q_NOFF 30 +#define VIRTIO_PCI_COMMON_Q_DESCLO 32 +#define VIRTIO_PCI_COMMON_Q_DESCHI 36 +#define VIRTIO_PCI_COMMON_Q_AVAILLO 40 +#define VIRTIO_PCI_COMMON_Q_AVAILHI 44 +#define VIRTIO_PCI_COMMON_Q_USEDLO 48 +#define VIRTIO_PCI_COMMON_Q_USEDHI 52 +#endif /* _UAPI_LINUX_VIRTIO_PCI_H */
Quoted 29 lines · Michael S. Tsirkin
Quoted 14 lines · Anthony Liguori
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
Quoted 7 lines
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
If you look at my patches you'll see how it works. Basically old guests use BAR0 new ones don't, so it's easy: BAR0 access means legacy guest. Only started testing but things seem to work fine with old guests so far.
Quoted 3 lines
I think we need a spec, not just driver code.
Rusty what's the plan? Want me to write it?
We need both, of course, but the spec work will happen in the OASIS WG. A draft is good, but let's not commit anything to upstream QEMU until we get the spec finalized. And that is proposed to be late this year.
Since I'm going to have to reformat the spec and adapt it into OASIS style anyway, perhaps you should prepare a description as a standalone text document. Easier to email and work with...
Now, the idea is that if you want to support 0.9 and 1.0 (or whatever we call them; I used the term "legacy" for existing implementations in the OASIS WG proposal), you add capabilities and don't point them into (the start of?) BAR0. Old drivers use BAR0 as now.
One trick to note: while drivers shouldn't use both old and new style on the same device, you need to allow it for kexec, particularly reset via BAR0.
Cheers,
Rusty.
Michael S. Tsirkin
maintainerQuoted 30 lines · Rusty Russell
Quoted 29 lines · Michael S. Tsirkin
Quoted 14 lines · Anthony Liguori
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
Quoted 7 lines
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
If you look at my patches you'll see how it works. Basically old guests use BAR0 new ones don't, so it's easy: BAR0 access means legacy guest. Only started testing but things seem to work fine with old guests so far.
Quoted 7 lines
Quoted 3 lines
I think we need a spec, not just driver code.
Rusty what's the plan? Want me to write it?
We need both, of course, but the spec work will happen in the OASIS WG. A draft is good, but let's not commit anything to upstream QEMU until we get the spec finalized. And that is proposed to be late this year.
Well that would be quite sad really.
This means we can't make virtio a spec compliant pci express device, and we can't add any more feature bits, so no
flexible buffer optimizations for virtio net.
There are probably more projects that will be blocked.
So how about we keep extending legacy layout for a bit longer:
- add a way to access device with MMIO
- use feature bit 31 to signal 64 bit features
(and shift device config accordingly)
No endian-ness rework, no per queue enable etc.
Then when we start discussions we will have a working express and working 64 bit feature support, and it will be that much easier to make it pretty it.
Quoted 15 lines
Since I'm going to have to reformat the spec and adapt it into OASIS style anyway, perhaps you should prepare a description as a standalone text document. Easier to email and work with...
Now, the idea is that if you want to support 0.9 and 1.0 (or whatever we call them; I used the term "legacy" for existing implementations in the OASIS WG proposal), you add capabilities and don't point them into (the start of?) BAR0. Old drivers use BAR0 as now.
One trick to note: while drivers shouldn't use both old and new style on the same device, you need to allow it for kexec, particularly reset via BAR0.
Cheers,
Rusty.
Quoted 45 lines · Michael S. Tsirkin
Quoted 30 lines · Rusty Russell
Quoted 29 lines · Michael S. Tsirkin
Quoted 14 lines · Anthony Liguori
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
Quoted 7 lines
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
If you look at my patches you'll see how it works. Basically old guests use BAR0 new ones don't, so it's easy: BAR0 access means legacy guest. Only started testing but things seem to work fine with old guests so far.
Quoted 7 lines
Quoted 3 lines
I think we need a spec, not just driver code.
Rusty what's the plan? Want me to write it?
We need both, of course, but the spec work will happen in the OASIS WG. A draft is good, but let's not commit anything to upstream QEMU until we get the spec finalized. And that is proposed to be late this year.
Well that would be quite sad really.
This means we can't make virtio a spec compliant pci express device, and we can't add any more feature bits, so no
flexible buffer optimizations for virtio net.
Quoted 6 lines
There are probably more projects that will be blocked.
So how about we keep extending legacy layout for a bit longer:
- add a way to access device with MMIO
- use feature bit 31 to signal 64 bit features
(and shift device config accordingly)
By my count, net still has 7 feature bits left, so I don't think the feature bits are likely to be a limitation in the next 6 months?
MMIO is a bigger problem. Linux guests are happy with it: does it break the Windows drivers?
Thanks,
Rusty.
Michael S. Tsirkin
maintainerQuoted 48 lines · Rusty Russell
Quoted 47 lines · Michael S. Tsirkin
Quoted 30 lines · Rusty Russell
Quoted 29 lines · Michael S. Tsirkin
Quoted 14 lines · Anthony Liguori
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
Quoted 7 lines
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
If you look at my patches you'll see how it works. Basically old guests use BAR0 new ones don't, so it's easy: BAR0 access means legacy guest. Only started testing but things seem to work fine with old guests so far.
Quoted 7 lines
Quoted 3 lines
I think we need a spec, not just driver code.
Rusty what's the plan? Want me to write it?
We need both, of course, but the spec work will happen in the OASIS WG. A draft is good, but let's not commit anything to upstream QEMU until we get the spec finalized. And that is proposed to be late this year.
Well that would be quite sad really.
This means we can't make virtio a spec compliant pci express device, and we can't add any more feature bits, so no
flexible buffer optimizations for virtio net.There are probably more projects that will be blocked.
Quoted 7 lines
Quoted 4 lines
So how about we keep extending legacy layout for a bit longer:
- add a way to access device with MMIO
- use feature bit 31 to signal 64 bit features
(and shift device config accordingly)By my count, net still has 7 feature bits left, so I don't think the feature bits are likely to be a limitation in the next 6 months?
Yes but you wanted a generic transport feature bit for flexible SG layout.
Are you happy with VIRTIO_NET_F_ANY_HEADER_SG then?
Quoted 5 lines
MMIO is a bigger problem. Linux guests are happy with it: does it break the Windows drivers?
Thanks,
Rusty.
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...
Signature · 1 line
MST
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines
By my count, net still has 7 feature bits left, so I don't think the feature bits are likely to be a limitation in the next 6 months?
Yes but you wanted a generic transport feature bit for flexible SG layout.
Are you happy withVIRTIO_NET_F_ANY_HEADER_SGthen?
If we need it within 6 months, I'd rather do that: this feature would then be assumed for 1.0, and reserved. We may do that for other features, too (the committee will have to review).
Quoted 6 lines
Quoted 5 lines
MMIO is a bigger problem. Linux guests are happy with it: does it break the Windows drivers?
Thanks,
Rusty.
Quoted 7 lines
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...
We might have something ready to deploy in 3 months, but realistically, I'd rather have it ready and tested outside the main git tree(s) and push it once the standard is committed.
Cheers,
Rusty.
Michael S. Tsirkin
maintainerQuoted 26 lines · Rusty Russell
Quoted 6 lines · Michael S. Tsirkin
Quoted 2 lines
By my count, net still has 7 feature bits left, so I don't think the feature bits are likely to be a limitation in the next 6 months?
Yes but you wanted a generic transport feature bit for flexible SG layout.
Are you happy withVIRTIO_NET_F_ANY_HEADER_SGthen?If we need it within 6 months, I'd rather do that: this feature would then be assumed for 1.0, and reserved. We may do that for other features, too (the committee will have to review).
Quoted 6 lines
Quoted 5 lines
MMIO is a bigger problem. Linux guests are happy with it: does it break the Windows drivers?
Thanks,
Rusty.Quoted 7 lines
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...
Quoted 7 lines
We might have something ready to deploy in 3 months,
but realistically,
I'd rather have it ready and tested outside the main git tree(s) and push it once the standard is committed.Cheers,
Rusty.
We have working code already. We don't need 3 months out of tree, this gets us no progress. To make it ready to deploy
we need it upstream and people testing it.
I think the issue is the layout change, you don't want the new config layout before we get standartization rolling, right? Okay how about this small patch to the linux guest (completely untested, just to give you the idea):
drivers/virtio/virtio_pci.c +31 -1
@@ -675,6 +675,33 @@ static void virtio_pci_release_dev(struct device *_d)*/ } +/* Map a BAR. But carefully: make sure we don't overlap the MSI-X table */ +static void __iomem *virtio_pci_iomap (struct pci_dev *pci_dev,int bar) +{ +int msix_cap =pci_find_capability (pci_dev,PCI_CAP_ID_MSIX ); +if (msix_cap) { +u32 offset; +u8 bir; +pci_read_config_dword (pci_dev, msix_cap +PCI_MSIX_TABLE , + &offset); + bir = (u8)(offset &PCI_MSIX_TABLE_BIR ); + offset &=PCI_MSIX_TABLE_OFFSET ; +/* Spec says table offset is in a 4K page all by itself */ +if (bir == bar && offset <4096 ) +return NULL ; + +pci_read_config_dword (pci_dev, msix_cap +PCI_MSIX_PBA , + &offset); + bir = (u8)(offset &PCI_MSIX_PBA_BIR ); + offset &=PCI_MSIX_PBA_OFFSET ; +/* Spec says table offset is in a 4K page all by itself. */ +if (bir == bar && offset <4096 ) +return NULL ; + } +/* 4K is enough for all devices at the moment. */ +return pci_iomap (pci_dev,0 ,4096 ); +} +/* the PCI probing function */ static int virtio_pci_probe (struct pci_dev *pci_dev,const struct pci_device_id *id)
@@ -716,7 +743,10 @@ static int virtio_pci_probe(struct pci_dev *pci_dev,if (err) goto out_enable_device; - vp_dev->ioaddr =pci_iomap (pci_dev,0 ,0 ); + vp_dev->ioaddr =virtio_pci_iomap (pci_dev,0 ); +/* Failed to map BAR0? Try with BAR1. */ +if (vp_dev->ioaddr ==NULL ) { + vp_dev->ioaddr =virtio_pci_iomap (pci_dev,1 );if (vp_dev->ioaddr ==NULL ) { err = -ENOMEM ; goto out_req_regions;
In other words: put a copy of IO config at start of MMIO BAR1, making sure we don't overlap the MSIX table that's there.
This does not break windows guests, and makes us compliant to the PCI express spec.
Is this small enough to start going immediately, without waiting 3+ months?
I really think it's a good idea to put something like this in the field: we might discover more issues around MMIO and we'll address them in the new config layout.
This way we don't get two changes: new layout and switch to MMIO all at once.
If you like this, I can make the appropriate spec and qemu changes in a couple of days.
Signature · 1 line
MST
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Regards,
Anthony Liguori
Quoted 7 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 11 lines · Anthony Liguori
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...
Quoted 2 lines
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
Yes, absolutely, this wording in spec is not there without reason.
Existing guests allocate io space for PCI express ports in multiples on 4K.
Since each express device is behind such a port, this means at most 15 such devices can use IO ports in a system.
That's why to make a pci express virtio device, we must allow MMIO and/or some other communication mechanism as the spec requires.
That's on x86.
Besides x86, there are achitectures where IO is unavailable or very slow.
Quoted 6 lines
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Regards,
Anthony Liguori
Why do you bring this up? No one advocates any ABI breakage, I only suggest extensions.
Quoted 7 lines
Quoted 7 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 20 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...Quoted 2 lines
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
Yes, absolutely, this wording in spec is not there without reason.
Existing guests allocate io space for PCI express ports in multiples on 4K.
Quoted 6 lines
Since each express device is behind such a port, this means at most 15 such devices can use IO ports in a system.
That's why to make a pci express virtio device, we must allow MMIO and/or some other communication mechanism as the spec requires.
This is precisely why this is an ABI breaker.
If you disable IO bars in the BIOS, than the interface that the OS sees will *not have an IO bar*.
This *breaks existing guests*.
Any time the programming interfaces changes on a PCI device, the revision ID and/or device ID must change. The spec is very clear about this.
We cannot disable the IO BAR without changing revision ID/device ID.
Quoted 7 lines
That's on x86.
Besides x86, there are achitectures where IO is unavailable or very slow.
Quoted 3 lines
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Quoted 6 lines
Quoted 3 lines
Regards,
Anthony Liguori
Why do you bring this up? No one advocates any ABI breakage, I only suggest extensions.
It's an ABI breakage. You're claiming that the guests you tested handle the breakage reasonably but it is unquestionably an ABI breakage.
Regards,
Anthony Liguori
Quoted 13 lines
Quoted 9 lines
Quoted 9 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 34 lines · Anthony Liguori
Quoted 20 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...Quoted 2 lines
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
Yes, absolutely, this wording in spec is not there without reason.
Existing guests allocate io space for PCI express ports in multiples on 4K.
Quoted 6 lines
Since each express device is behind such a port, this means at most 15 such devices can use IO ports in a system.
That's why to make a pci express virtio device, we must allow MMIO and/or some other communication mechanism as the spec requires.
This is precisely why this is an ABI breaker.
If you disable IO bars in the BIOS, than the interface that the OS sees will *not have an IO bar*.
Quoted 7 lines
This *breaks existing guests*.
Any time the programming interfaces changes on a PCI device, the revision ID and/or device ID must change. The spec is very clear about this.We cannot disable the IO BAR without changing revision ID/device ID.
But it's a bios/PC issue. It's not a device issue.
Anyway, let's put express aside.
It's easy to create non-working setups with pci, today:
- create 16 pci bridges
- put one virtio device behind each
boom
Try it.
I want to fix that.
Quoted 14 lines
Quoted 7 lines
That's on x86.
Besides x86, there are achitectures where IO is unavailable or very slow.
Quoted 3 lines
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Quoted 6 lines
Quoted 3 lines
Regards,
Anthony Liguori
Why do you bring this up? No one advocates any ABI breakage, I only suggest extensions.
Quoted 6 lines
It's an ABI breakage. You're claiming that the guests you tested handle the breakage reasonably but it is unquestionably an ABI breakage.
Regards,
Anthony Liguori
Adding BAR is not an ABI breakage, do we agree on that?
Disabling IO would be but I am not proposing disabling IO.
Guests might disable IO.
I propose a way to make virtio still work if they do.
This is *fixing* things. Not breaking.
Quoted 13 lines
Quoted 13 lines
Quoted 9 lines
Quoted 9 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 78 lines · Michael S. Tsirkin
Quoted 34 lines · Anthony Liguori
Quoted 20 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...Quoted 2 lines
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
Yes, absolutely, this wording in spec is not there without reason.
Existing guests allocate io space for PCI express ports in multiples on 4K.
Quoted 6 lines
Since each express device is behind such a port, this means at most 15 such devices can use IO ports in a system.
That's why to make a pci express virtio device, we must allow MMIO and/or some other communication mechanism as the spec requires.
This is precisely why this is an ABI breaker.
If you disable IO bars in the BIOS, than the interface that the OS sees will *not have an IO bar*.
Quoted 7 lines
This *breaks existing guests*.
Any time the programming interfaces changes on a PCI device, the revision ID and/or device ID must change. The spec is very clear about this.We cannot disable the IO BAR without changing revision ID/device ID.
But it's a bios/PC issue. It's not a device issue.
Anyway, let's put express aside.
It's easy to create non-working setups with pci, today:
- create 16 pci bridges
- put one virtio device behind eachboom
Try it.
I want to fix that.
Quoted 19 lines
Quoted 7 lines
That's on x86.
Besides x86, there are achitectures where IO is unavailable or very slow.
Quoted 3 lines
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Quoted 6 lines
Quoted 3 lines
Regards,
Anthony Liguori
Why do you bring this up? No one advocates any ABI breakage, I only suggest extensions.
It's an ABI breakage. You're claiming that the guests you tested handle the breakage reasonably but it is unquestionably an ABI breakage.
Regards,
Quoted 7 lines
Quoted 1 line
Anthony Liguori
Adding BAR is not an ABI breakage, do we agree on that?
Disabling IO would be but I am not proposing disabling IO.
Guests might disable IO.
Look, it's very simple.
If the failure in the guest is that BAR0 mapping fails because the device is enabled but the BAR is disabled, then you've broken the ABI.
And what's worse is that this isn't for an obscure scenario (like having 15 PCI bridges) but for something that would become the standard scenario (using a PCI-e bus).
We need to either bump the revision ID or the device ID if we do this.
Regards,
Anthony Liguori
Quoted 78 lines · Michael S. Tsirkin
Quoted 34 lines · Anthony Liguori
Quoted 20 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...Quoted 2 lines
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
Yes, absolutely, this wording in spec is not there without reason.
Existing guests allocate io space for PCI express ports in multiples on 4K.
Quoted 6 lines
Since each express device is behind such a port, this means at most 15 such devices can use IO ports in a system.
That's why to make a pci express virtio device, we must allow MMIO and/or some other communication mechanism as the spec requires.
This is precisely why this is an ABI breaker.
If you disable IO bars in the BIOS, than the interface that the OS sees will *not have an IO bar*.
Quoted 7 lines
This *breaks existing guests*.
Any time the programming interfaces changes on a PCI device, the revision ID and/or device ID must change. The spec is very clear about this.We cannot disable the IO BAR without changing revision ID/device ID.
But it's a bios/PC issue. It's not a device issue.
Anyway, let's put express aside.
It's easy to create non-working setups with pci, today:
- create 16 pci bridges
- put one virtio device behind eachboom
Try it.
I want to fix that.
Quoted 19 lines
Quoted 7 lines
That's on x86.
Besides x86, there are achitectures where IO is unavailable or very slow.
Quoted 3 lines
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Quoted 6 lines
Quoted 3 lines
Regards,
Anthony Liguori
Why do you bring this up? No one advocates any ABI breakage, I only suggest extensions.
It's an ABI breakage. You're claiming that the guests you tested handle the breakage reasonably but it is unquestionably an ABI breakage.
Regards,
Quoted 7 lines
Quoted 1 line
Anthony Liguori
Adding BAR is not an ABI breakage, do we agree on that?
Disabling IO would be but I am not proposing disabling IO.
Guests might disable IO.
Look, it's very simple.
If the failure in the guest is that BAR0 mapping fails because the device is enabled but the BAR is disabled, then you've broken the ABI.
And what's worse is that this isn't for an obscure scenario (like having 15 PCI bridges) but for something that would become the standard scenario (using a PCI-e bus).
We need to either bump the revision ID or the device ID if we do this.
Regards,
Anthony Liguori
Michael S. Tsirkin
maintainerQuoted 84 lines · Anthony Liguori
Quoted 82 lines · Michael S. Tsirkin
Quoted 34 lines · Anthony Liguori
Quoted 20 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 8 lines · Michael S. Tsirkin
On Tue, Jun 04, 2013 at 03:01:50PM +0930, Rusty Russell wrote:
You mean make BAR0 an MMIO BAR?
Yes, it would break current windows guests.
Further, as long as we use same address to notify all queues, we would also need to decode the instruction on x86 and that's measureably slower than PIO.
We could go back to discussing hypercall use for notifications, but that has its own set of issues...Quoted 2 lines
So... does "violating the PCI-e" spec really matter? Is it preventing any guest from working properly?
Yes, absolutely, this wording in spec is not there without reason.
Existing guests allocate io space for PCI express ports in multiples on 4K.
Quoted 6 lines
Since each express device is behind such a port, this means at most 15 such devices can use IO ports in a system.
That's why to make a pci express virtio device, we must allow MMIO and/or some other communication mechanism as the spec requires.
This is precisely why this is an ABI breaker.
If you disable IO bars in the BIOS, than the interface that the OS sees will *not have an IO bar*.
Quoted 7 lines
This *breaks existing guests*.
Any time the programming interfaces changes on a PCI device, the revision ID and/or device ID must change. The spec is very clear about this.We cannot disable the IO BAR without changing revision ID/device ID.
But it's a bios/PC issue. It's not a device issue.
Anyway, let's put express aside.
It's easy to create non-working setups with pci, today:
- create 16 pci bridges
- put one virtio device behind eachboom
Try it.
I want to fix that.
Quoted 14 lines
Quoted 7 lines
That's on x86.
Besides x86, there are achitectures where IO is unavailable or very slow.
Quoted 3 lines
I don't think we should rush an ABI breakage if the only benefit is claiming spec compliance.
Quoted 6 lines
Quoted 3 lines
Regards,
Anthony Liguori
Why do you bring this up? No one advocates any ABI breakage, I only suggest extensions.
Quoted 6 lines
It's an ABI breakage. You're claiming that the guests you tested handle the breakage reasonably but it is unquestionably an ABI breakage.
Regards,
Anthony Liguori
Adding BAR is not an ABI breakage, do we agree on that?
Quoted 8 lines
Quoted 3 lines
Disabling IO would be but I am not proposing disabling IO.
Guests might disable IO.
Look, it's very simple.
If the failure in the guest is that BAR0 mapping fails because the device is enabled but the BAR is disabled,
There's no such thing as device is enabled and neither BAR is disabled in PCI spec.
Spec says IO and memory can be enabled/disabled, separately. PCI Express spec says devices should work without IO.
So modern guests will assume it's ok to work without IO, and it will become more common in the future.
Quoted 1 line
then you've broken the ABI.
No. It means that the ABI is broken.
Guests can disable IO *today* and when they do things don't work.
Quoted 10 lines
And what's worse is that this isn't for an obscure scenario (like having 15 PCI bridges) but for something that would become the standard scenario (using a PCI-e bus).
We need to either bump the revision ID or the device ID if we do this.
Regards,
Anthony Liguori
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
Signature · 1 line
MST
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
virtio-pci uses a IO bars exclusively today. Existing guest drivers assume that there is an IO bar that contains the virtio-pci registers.
So let's consider the following scenarios:
QEMU of today:
1) qemu -drive file=ubuntu-13.04.img,if=virtio
This works today. Does adding an MMIO bar at BAR1 break this? Certainly not if the device is behind a PCI bus...
But are we going to put devices behind a PCI-e bus by default? Are we going to ask the user to choose whether devices are put behind a legacy bus or the express bus?
What happens if we put the device behind a PCI-e bus by default? Well, it can still work. That is, until we do something like this:
2) qemu -drive file=ubuntu-13.04.img,if=virtio -device virtio-rng -device virtio-balloon..
Such that we have more than a dozen or so devices. This works perfectly fine today. It works fine because we've designed virtio to make sure it works fine. Quoting the spec:
"Configuration space is generally used for rarely-changing or
initialization-time parameters. But it is a limited resource, so it might be better to use a virtqueue to update configuration information (the network device does this for filtering, otherwise the table in the config space could potentially be very large)."
In fact, we can have 100s of PCI devices today without running out of IO space because we're so careful about this.
So if we switch to using PCI-e by default *and* we keep virtio-pci without modifying the device IDs, then very frequently we are going to break existing guests because the drivers they already have no longer work.
A few virtio-serial channels, a few block devices, a couple of network adapters, the balloon and RNG driver, and we hit the IO space limit pretty damn quickly so this is not a contrived scenario at all. I would expect that we frequently run into this if we don't address this problem.
So we have a few options:
1) Punt all of this complexity to libvirt et al and watch people make the wrong decisions about when to use PCI-e. This will become yet another example of KVM being too hard to configure.
2) Enable PCI-e by default and just force people to upgrade their drivers.
3) Don't use PCI-e by default but still add BAR1 to virtio-pci
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely), give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.
I think 1 == 2 == 3 and I view 2 as an ABI breaker. libvirt does like policy so they're going to make a simple decision and always use the same bus by default. I suspect if we made PCI the default, they might just always set the PCI-e flag just because.
There are hundreds of thousands if not millions of guests with existing virtio-pci drivers. Forcing them to upgrade better have an extremely good justification.
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
Regards,
Anthony Liguori
Quoted 8 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 11 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Quoted 8 lines
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
Let me add 3 other issues which I mentioned and you seem to miss:
3) architectures which don't have fast access to IO ports, exist virtio does not work there ATM
4) setups with many PCI bridges exist and have the same issue as PCI express. virtio does not work there ATM
5) On x86, even with nested page tables, firmware only decodes the page address on an invalid PTE, not the data. You need to emulate the guest to get at the data. Without
nested page tables, we have to do page table walk and emulate to get both address and data. Since this is how MMIO
is implemented in kvm on x86, MMIO is much slower than PIO (with nested page tables by a factor of >2, did not test without).
Quoted 49 lines
virtio-pci uses a IO bars exclusively today. Existing guest drivers assume that there is an IO bar that contains the virtio-pci registers. So let's consider the following scenarios:
QEMU of today:
1) qemu -drive file=ubuntu-13.04.img,if=virtio
This works today. Does adding an MMIO bar at BAR1 break this? Certainly not if the device is behind a PCI bus...
But are we going to put devices behind a PCI-e bus by default? Are we going to ask the user to choose whether devices are put behind a legacy bus or the express bus?
What happens if we put the device behind a PCI-e bus by default? Well, it can still work. That is, until we do something like this:
2) qemu -drive file=ubuntu-13.04.img,if=virtio -device virtio-rng -device virtio-balloon..
Such that we have more than a dozen or so devices. This works perfectly fine today. It works fine because we've designed virtio to make sure it works fine. Quoting the spec:
"Configuration space is generally used for rarely-changing or
initialization-time parameters. But it is a limited resource, so it might be better to use a virtqueue to update configuration information (the network device does this for filtering, otherwise the table in the config space could potentially be very large)."In fact, we can have 100s of PCI devices today without running out of IO space because we're so careful about this.
So if we switch to using PCI-e by default *and* we keep virtio-pci without modifying the device IDs, then very frequently we are going to break existing guests because the drivers they already have no longer work.
A few virtio-serial channels, a few block devices, a couple of network adapters, the balloon and RNG driver, and we hit the IO space limit pretty damn quickly so this is not a contrived scenario at all. I would expect that we frequently run into this if we don't address this problem.
So we have a few options:
1) Punt all of this complexity to libvirt et al and watch people make the wrong decisions about when to use PCI-e. This will become yet another example of KVM being too hard to configure.
Quoted 6 lines
2) Enable PCI-e by default and just force people to upgrade their drivers.
3) Don't use PCI-e by default but still add BAR1 to virtio-pci
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely),
We can't do this - it will hurt performance.
Quoted 6 lines
give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.I think 1 == 2 == 3 and I view 2 as an ABI breaker.
Why do you think 2 == 3? 2 changes default behaviour. 3 does not.
Quoted 4 lines
libvirt does like
policy so they're going to make a simple decision and always use the same bus by default. I suspect if we made PCI the default, they might just always set the PCI-e flag just because.
This sounds very strange. But let's assume you are right for the sake of the argument ...
Quoted 8 lines
There are hundreds of thousands if not millions of guests with existing virtio-pci drivers. Forcing them to upgrade better have an extremely good justification.
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
I don't think how this changes the situation. libvirt still need to set policy and decide which device to use.
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Only because the chance it's 100% compatible on the software level is 0. It always has some hardware specific quirks.
No such excuse here.
Quoted 2 lines
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
This last paragraph is wrong, it ignores the issues 3) to 5) I added above.
If you do take them into account:
- there are reasons to add MMIO BAR to PCI, even without PCI express
- we won't be able to drop IO BAR from virtio
Quoted 12 lines
Regards,
Anthony Liguori
Quoted 8 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 30 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Quoted 8 lines
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
Let me add 3 other issues which I mentioned and you seem to miss:
3) architectures which don't have fast access to IO ports, exist virtio does not work there ATM
4) setups with many PCI bridges exist and have the same issue as PCI express. virtio does not work there ATM
Quoted 7 lines
5) On x86, even with nested page tables, firmware only decodes the page address on an invalid PTE, not the data. You need to emulate the guest to get at the data. Without
nested page tables, we have to do page table walk and emulate to get both address and data. Since this is how MMIO
is implemented in kvm on x86, MMIO is much slower than PIO (with nested page tables by a factor of >2, did not test without).
Oh I forgot:
6) access to MMIO BARs is painful in the BIOS environment so BIOS would typically need to enable IO for the boot device.
Quoted 118 lines
Quoted 49 lines
virtio-pci uses a IO bars exclusively today. Existing guest drivers assume that there is an IO bar that contains the virtio-pci registers. So let's consider the following scenarios:
QEMU of today:
1) qemu -drive file=ubuntu-13.04.img,if=virtio
This works today. Does adding an MMIO bar at BAR1 break this? Certainly not if the device is behind a PCI bus...
But are we going to put devices behind a PCI-e bus by default? Are we going to ask the user to choose whether devices are put behind a legacy bus or the express bus?
What happens if we put the device behind a PCI-e bus by default? Well, it can still work. That is, until we do something like this:
2) qemu -drive file=ubuntu-13.04.img,if=virtio -device virtio-rng -device virtio-balloon..
Such that we have more than a dozen or so devices. This works perfectly fine today. It works fine because we've designed virtio to make sure it works fine. Quoting the spec:
"Configuration space is generally used for rarely-changing or
initialization-time parameters. But it is a limited resource, so it might be better to use a virtqueue to update configuration information (the network device does this for filtering, otherwise the table in the config space could potentially be very large)."In fact, we can have 100s of PCI devices today without running out of IO space because we're so careful about this.
So if we switch to using PCI-e by default *and* we keep virtio-pci without modifying the device IDs, then very frequently we are going to break existing guests because the drivers they already have no longer work.
A few virtio-serial channels, a few block devices, a couple of network adapters, the balloon and RNG driver, and we hit the IO space limit pretty damn quickly so this is not a contrived scenario at all. I would expect that we frequently run into this if we don't address this problem.
So we have a few options:
1) Punt all of this complexity to libvirt et al and watch people make the wrong decisions about when to use PCI-e. This will become yet another example of KVM being too hard to configure.Quoted 6 lines
2) Enable PCI-e by default and just force people to upgrade their drivers.
3) Don't use PCI-e by default but still add BAR1 to virtio-pci
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely),
We can't do this - it will hurt performance.
Quoted 6 lines
give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.I think 1 == 2 == 3 and I view 2 as an ABI breaker.
Why do you think 2 == 3? 2 changes default behaviour. 3 does not.
Quoted 4 lines
libvirt does like
policy so they're going to make a simple decision and always use the same bus by default. I suspect if we made PCI the default, they might just always set the PCI-e flag just because.This sounds very strange. But let's assume you are right for the sake of the argument ...
Quoted 8 lines
There are hundreds of thousands if not millions of guests with existing virtio-pci drivers. Forcing them to upgrade better have an extremely good justification.
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
I don't think how this changes the situation. libvirt still need to set policy and decide which device to use.
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Only because the chance it's 100% compatible on the software level is 0. It always has some hardware specific quirks.
No such excuse here.Quoted 2 lines
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
This last paragraph is wrong, it ignores the issues 3) to 5) I added above.
If you do take them into account:
- there are reasons to add MMIO BAR to PCI, even without PCI express
- we won't be able to drop IO BAR from virtioQuoted 12 lines
Regards,
Anthony Liguori
Quoted 8 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Michael S. Tsirkin
maintainerQuoted 74 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
virtio-pci uses a IO bars exclusively today. Existing guest drivers assume that there is an IO bar that contains the virtio-pci registers.
So let's consider the following scenarios:
QEMU of today:
1) qemu -drive file=ubuntu-13.04.img,if=virtio
This works today. Does adding an MMIO bar at BAR1 break this? Certainly not if the device is behind a PCI bus...
But are we going to put devices behind a PCI-e bus by default? Are we going to ask the user to choose whether devices are put behind a legacy bus or the express bus?
What happens if we put the device behind a PCI-e bus by default? Well, it can still work. That is, until we do something like this:
2) qemu -drive file=ubuntu-13.04.img,if=virtio -device virtio-rng -device virtio-balloon..
Such that we have more than a dozen or so devices. This works perfectly fine today. It works fine because we've designed virtio to make sure it works fine. Quoting the spec:
"Configuration space is generally used for rarely-changing or
initialization-time parameters. But it is a limited resource, so it might be better to use a virtqueue to update configuration information (the network device does this for filtering, otherwise the table in the config space could potentially be very large)."In fact, we can have 100s of PCI devices today without running out of IO space because we're so careful about this.
So if we switch to using PCI-e by default *and* we keep virtio-pci without modifying the device IDs, then very frequently we are going to break existing guests because the drivers they already have no longer work.
A few virtio-serial channels, a few block devices, a couple of network adapters, the balloon and RNG driver, and we hit the IO space limit pretty damn quickly so this is not a contrived scenario at all. I would expect that we frequently run into this if we don't address this problem.
So we have a few options:
1) Punt all of this complexity to libvirt et al and watch people make the wrong decisions about when to use PCI-e. This will become yet another example of KVM being too hard to configure.
2) Enable PCI-e by default and just force people to upgrade their drivers.
Quoted 6 lines
3) Don't use PCI-e by default but still add BAR1 to virtio-pci
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely), give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.
For the record, with respect to PCI-e discussion, I have no problem with the idea of changing the device ID or revision id and asking guests to upgrade if they want to use a pcie device.
That's not exactly 4 however.
I see no reason to couple PCI-e with MMIO discussion, that's just one of the reasons to support MMIO.
Quoted 33 lines
I think 1 == 2 == 3 and I view 2 as an ABI breaker. libvirt does like policy so they're going to make a simple decision and always use the same bus by default. I suspect if we made PCI the default, they might just always set the PCI-e flag just because.
There are hundreds of thousands if not millions of guests with existing virtio-pci drivers. Forcing them to upgrade better have an extremely good justification.
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
Regards,
Anthony Liguori
Quoted 8 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 22 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Quoted 8 lines
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
Quoted 4 lines
Let me add 3 other issues which I mentioned and you seem to miss:
3) architectures which don't have fast access to IO ports, exist virtio does not work there ATM
Which architectures have PCI but no IO ports?
Quoted 2 lines
4) setups with many PCI bridges exist and have the same issue as PCI express. virtio does not work there ATM
This is not virtio specific. This is true for all devices that use IO.
Quoted 7 lines
5) On x86, even with nested page tables, firmware only decodes the page address on an invalid PTE, not the data. You need to emulate the guest to get at the data. Without
nested page tables, we have to do page table walk and emulate to get both address and data. Since this is how MMIO
is implemented in kvm on x86, MMIO is much slower than PIO (with nested page tables by a factor of >2, did not test without).
Am well aware of this, this is why we use PIO.
I fully agree with you that when we do MMIO, we should switch the notification mechanism to avoid encoding anything meaningful as data.
Quoted 49 lines
Quoted 49 lines
virtio-pci uses a IO bars exclusively today. Existing guest drivers assume that there is an IO bar that contains the virtio-pci registers. So let's consider the following scenarios:
QEMU of today:
1) qemu -drive file=ubuntu-13.04.img,if=virtio
This works today. Does adding an MMIO bar at BAR1 break this? Certainly not if the device is behind a PCI bus...
But are we going to put devices behind a PCI-e bus by default? Are we going to ask the user to choose whether devices are put behind a legacy bus or the express bus?
What happens if we put the device behind a PCI-e bus by default? Well, it can still work. That is, until we do something like this:
2) qemu -drive file=ubuntu-13.04.img,if=virtio -device virtio-rng -device virtio-balloon..
Such that we have more than a dozen or so devices. This works perfectly fine today. It works fine because we've designed virtio to make sure it works fine. Quoting the spec:
"Configuration space is generally used for rarely-changing or
initialization-time parameters. But it is a limited resource, so it might be better to use a virtqueue to update configuration information (the network device does this for filtering, otherwise the table in the config space could potentially be very large)."In fact, we can have 100s of PCI devices today without running out of IO space because we're so careful about this.
So if we switch to using PCI-e by default *and* we keep virtio-pci without modifying the device IDs, then very frequently we are going to break existing guests because the drivers they already have no longer work.
A few virtio-serial channels, a few block devices, a couple of network adapters, the balloon and RNG driver, and we hit the IO space limit pretty damn quickly so this is not a contrived scenario at all. I would expect that we frequently run into this if we don't address this problem.
So we have a few options:
1) Punt all of this complexity to libvirt et al and watch people make the wrong decisions about when to use PCI-e. This will become yet another example of KVM being too hard to configure.
Quoted 8 lines
Quoted 6 lines
2) Enable PCI-e by default and just force people to upgrade their drivers.
3) Don't use PCI-e by default but still add BAR1 to virtio-pci
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely),
We can't do this - it will hurt performance.
Can you explain? I thought the whole trick with separating out the virtqueue notification register was to regain the performance?
Quoted 8 lines
Quoted 6 lines
give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.I think 1 == 2 == 3 and I view 2 as an ABI breaker.
Why do you think 2 == 3? 2 changes default behaviour. 3 does not.
It doesn't change the default behavior but then we're pushing the decision of when to use pci-e to the user. They have to understand that there can be subtle breakages because the virtio-pci driver may not work if they are using an old guest.
Quoted 12 lines
Quoted 4 lines
libvirt does like
policy so they're going to make a simple decision and always use the same bus by default. I suspect if we made PCI the default, they might just always set the PCI-e flag just because.This sounds very strange. But let's assume you are right for the sake of the argument ...
Quoted 4 lines
There are hundreds of thousands if not millions of guests with existing virtio-pci drivers. Forcing them to upgrade better have an extremely good justification.
Quoted 7 lines
Quoted 4 lines
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
I don't think how this changes the situation. libvirt still need to set policy and decide which device to use.
But virtio-pcie never exhausts the IO configuration space. That's the difference.
And virtio-pcie is a separate driver so presumably libvirt will make that visible in the XML. In fact, it should.
Quoted 11 lines
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Only because the chance it's 100% compatible on the software level is 0. It always has some hardware specific quirks.
No such excuse here.Quoted 2 lines
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
Quoted 6 lines
This last paragraph is wrong, it ignores the issues 3) to 5) I added above.
If you do take them into account:
- there are reasons to add MMIO BAR to PCI, even without PCI express
So far, the only reason you've provided is "it doesn't work on some architectures." Which architectures?
Quoted 1 line
- we won't be able to drop IO BAR from virtio
An IO BAR is useless if it means we can't have more than 12 devices.
Quoted 17 lines
Quoted 13 lines
Regards,
Anthony Liguori
Quoted 8 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 39 lines · Michael S. Tsirkin
Quoted 30 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Quoted 8 lines
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
Let me add 3 other issues which I mentioned and you seem to miss:
3) architectures which don't have fast access to IO ports, exist virtio does not work there ATM
4) setups with many PCI bridges exist and have the same issue as PCI express. virtio does not work there ATM
Quoted 7 lines
5) On x86, even with nested page tables, firmware only decodes the page address on an invalid PTE, not the data. You need to emulate the guest to get at the data. Without
nested page tables, we have to do page table walk and emulate to get both address and data. Since this is how MMIO
is implemented in kvm on x86, MMIO is much slower than PIO (with nested page tables by a factor of >2, did not test without).
Quoted 4 lines
Oh I forgot:
6) access to MMIO BARs is painful in the BIOS environment so BIOS would typically need to enable IO for the boot device.
But if you want to boot from the 16th device, the BIOS needs to solve this problem anyway.
Regards,
Anthony Liguori
Quoted 4 lines · Michael S. Tsirkin
Spec says IO and memory can be enabled/disabled, separately. PCI Express spec says devices should work without IO.
For "native endpoints". Currently virtio would be a "legacy endpoint" which is quite correct -- it is compatible with a legacy interface.
-hpa
Michael S. Tsirkin
maintainerQuoted 29 lines · Anthony Liguori
Quoted 22 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Quoted 8 lines
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
Quoted 4 lines
Let me add 3 other issues which I mentioned and you seem to miss:
3) architectures which don't have fast access to IO ports, exist virtio does not work there ATM
Quoted 6 lines
Which architectures have PCI but no IO ports?
Quoted 2 lines
4) setups with many PCI bridges exist and have the same issue as PCI express. virtio does not work there ATM
This is not virtio specific. This is true for all devices that use IO.
Absolutely. And you will find that modern devices make use of IO ports optional.
Quoted 65 lines
Quoted 7 lines
5) On x86, even with nested page tables, firmware only decodes the page address on an invalid PTE, not the data. You need to emulate the guest to get at the data. Without
nested page tables, we have to do page table walk and emulate to get both address and data. Since this is how MMIO
is implemented in kvm on x86, MMIO is much slower than PIO (with nested page tables by a factor of >2, did not test without).Am well aware of this, this is why we use PIO.
I fully agree with you that when we do MMIO, we should switch the notification mechanism to avoid encoding anything meaningful as data.
Quoted 52 lines
Quoted 52 lines
virtio-pci uses a IO bars exclusively today. Existing guest drivers assume that there is an IO bar that contains the virtio-pci registers. So let's consider the following scenarios:
QEMU of today:
1) qemu -drive file=ubuntu-13.04.img,if=virtio
This works today. Does adding an MMIO bar at BAR1 break this? Certainly not if the device is behind a PCI bus...
But are we going to put devices behind a PCI-e bus by default? Are we going to ask the user to choose whether devices are put behind a legacy bus or the express bus?
What happens if we put the device behind a PCI-e bus by default? Well, it can still work. That is, until we do something like this:
2) qemu -drive file=ubuntu-13.04.img,if=virtio -device virtio-rng -device virtio-balloon..
Such that we have more than a dozen or so devices. This works perfectly fine today. It works fine because we've designed virtio to make sure it works fine. Quoting the spec:
"Configuration space is generally used for rarely-changing or
initialization-time parameters. But it is a limited resource, so it might be better to use a virtqueue to update configuration information (the network device does this for filtering, otherwise the table in the config space could potentially be very large)."In fact, we can have 100s of PCI devices today without running out of IO space because we're so careful about this.
So if we switch to using PCI-e by default *and* we keep virtio-pci without modifying the device IDs, then very frequently we are going to break existing guests because the drivers they already have no longer work.
A few virtio-serial channels, a few block devices, a couple of network adapters, the balloon and RNG driver, and we hit the IO space limit pretty damn quickly so this is not a contrived scenario at all. I would expect that we frequently run into this if we don't address this problem.
So we have a few options:
1) Punt all of this complexity to libvirt et al and watch people make the wrong decisions about when to use PCI-e. This will become yet another example of KVM being too hard to configure.2) Enable PCI-e by default and just force people to upgrade their drivers.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
3) Don't use PCI-e by default but still add BAR1 to virtio-pci
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely),
We can't do this - it will hurt performance.
Can you explain? I thought the whole trick with separating out the virtqueue notification register was to regain the performance?
Yes but this trick only works well with NPT (it's still a bit slower than PIO but not so drastically).
Without NPT you still need a page walk so it will be slow.
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
I think 1 == 2 == 3 and I view 2 as an ABI breaker.
Why do you think 2 == 3? 2 changes default behaviour. 3 does not.
It doesn't change the default behavior but then we're pushing the decision of when to use pci-e to the user. They have to understand that there can be subtle breakages because the virtio-pci driver may not work if they are using an old guest.
pci-e is a separate issue. E.g. we can make pci-e a new device id.
Quoted 17 lines
Quoted 17 lines
Quoted 4 lines
libvirt does like
policy so they're going to make a simple decision and always use the same bus by default. I suspect if we made PCI the default, they might just always set the PCI-e flag just because.This sounds very strange. But let's assume you are right for the sake of the argument ...
Quoted 8 lines
There are hundreds of thousands if not millions of guests with existing virtio-pci drivers. Forcing them to upgrade better have an extremely good justification.
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
Quoted 8 lines
Quoted 2 lines
I don't think how this changes the situation. libvirt still need to set policy and decide which device to use.
But virtio-pcie never exhausts the IO configuration space. That's the difference.
And virtio-pcie is a separate driver so presumably libvirt will make that visible in the XML. In fact, it should.
It should already do that explicit even if device name is the same. In fact, it does: bus name is pcie.0 versus pci.0
Quoted 14 lines
Quoted 14 lines
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Only because the chance it's 100% compatible on the software level is 0. It always has some hardware specific quirks.
No such excuse here.Quoted 2 lines
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
This last paragraph is wrong, it ignores the issues 3) to 5) I added above.
Quoted 6 lines
Quoted 3 lines
If you do take them into account:
- there are reasons to add MMIO BAR to PCI, even without PCI expressSo far, the only reason you've provided is "it doesn't work on some architectures." Which architectures?
PowerPC wants this.
Quoted 3 lines
Quoted 1 line
- we won't be able to drop IO BAR from virtio
An IO BAR is useless if it means we can't have more than 12 devices.
It's not useless. A smart BIOS can enable devices one by one as it tries to boot from them.
Quoted 17 lines
Quoted 17 lines
Quoted 13 lines
Regards,
Anthony Liguori
Quoted 8 lines
Signature · 6 lines
-- MST -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 4 lines · Anthony Liguori
But if you want to boot from the 16th device, the BIOS needs to solve this problem anyway.
No. Just have the BIOS plumb the path to the device it wants to boot from. Problem solved.
-hpa
Michael S. Tsirkin
maintainerQuoted 46 lines · Anthony Liguori
Quoted 39 lines · Michael S. Tsirkin
Quoted 30 lines · Michael S. Tsirkin
Quoted 11 lines · Anthony Liguori
Quoted 5 lines · Michael S. Tsirkin
Quoted 1 line · Anthony Liguori
Look, it's very simple.
We only need to do it if we do a change that breaks guests.
Please find a guest that is broken by the patches. You won't find any.
I think the problem in this whole discussion is that we're talking past each other.
Quoted 8 lines
Here is my understanding:
1) PCI-e says that you must be able to disable IO bars and still have a functioning device.
2) It says (1) because you must size IO bars to 4096 which means that practically speaking, once you enable a dozen or so PIO bars, you run out of PIO space (16 * 4k == 64k and not all that space can be used).
Let me add 3 other issues which I mentioned and you seem to miss:
3) architectures which don't have fast access to IO ports, exist virtio does not work there ATM
4) setups with many PCI bridges exist and have the same issue as PCI express. virtio does not work there ATM
Quoted 7 lines
5) On x86, even with nested page tables, firmware only decodes the page address on an invalid PTE, not the data. You need to emulate the guest to get at the data. Without
nested page tables, we have to do page table walk and emulate to get both address and data. Since this is how MMIO
is implemented in kvm on x86, MMIO is much slower than PIO (with nested page tables by a factor of >2, did not test without).Quoted 4 lines
Oh I forgot:
6) access to MMIO BARs is painful in the BIOS environment so BIOS would typically need to enable IO for the boot device.
Quoted 6 lines
But if you want to boot from the 16th device, the BIOS needs to solve this problem anyway.
Regards,
Anthony Liguori
But it's solvable: just enable devices one by one to boot from them.
Signature · 1 line
MST
Michael S. Tsirkin
maintainerQuoted 10 lines · H. Peter Anvin
Quoted 4 lines · Michael S. Tsirkin
Spec says IO and memory can be enabled/disabled, separately. PCI Express spec says devices should work without IO.
For "native endpoints". Currently virtio would be a "legacy endpoint" which is quite correct -- it is compatible with a legacy interface.
-hpa
Yes. But it's not the spec wording that we care for most. It's supporting setups that have trouble using IO.
Signature · 1 line
MST
Quoted 8 lines · H. Peter Anvin
Quoted 4 lines · Michael S. Tsirkin
Spec says IO and memory can be enabled/disabled, separately. PCI Express spec says devices should work without IO.
For "native endpoints". Currently virtio would be a "legacy endpoint" which is quite correct -- it is compatible with a legacy interface.
Do legacy endpoints also use 4k for BARs?
If not, can't we use a new device id for native endpoints and call it a day? Legacy endpoints would continue using the existing BAR layout.
Regards,
Anthony Liguori
Quoted 7 lines
-hpa
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 9 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
"Michael S. Tsirkin" <[email protected]> writes:
Can you explain? I thought the whole trick with separating out the virtqueue notification register was to regain the performance?
Yes but this trick only works well with NPT (it's still a bit slower than PIO but not so drastically).
Without NPT you still need a page walk so it will be slow.
Do you mean NPT/EPT?
If your concern is shadow paging, then I think you're concerned about hardware that is so slow to start with that it's not worth considering.
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Only because the chance it's 100% compatible on the software level is 0. It always has some hardware specific quirks.
No such excuse here.Quoted 2 lines
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
This last paragraph is wrong, it ignores the issues 3) to 5) I added above.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
If you do take them into account:
- there are reasons to add MMIO BAR to PCI, even without PCI expressSo far, the only reason you've provided is "it doesn't work on some architectures." Which architectures?
PowerPC wants this.
Existing PowerPC remaps PIO to MMAP so it works fine today.
Future platforms may not do this but future platforms can use a different device. They certainly won't be able to use the existing drivers anyway.
Ben, am I wrong here?
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
- we won't be able to drop IO BAR from virtio
An IO BAR is useless if it means we can't have more than 12 devices.
It's not useless. A smart BIOS can enable devices one by one as it tries to boot from them.
A smart BIOS can also use MMIO to program virtio.
Regards,
Anthony Liguori
Quoted 9 lines · Michael S. Tsirkin
Quoted 4 lines · Anthony Liguori
"Michael S. Tsirkin" <[email protected]> writes:
Can you explain? I thought the whole trick with separating out the virtqueue notification register was to regain the performance?
Yes but this trick only works well with NPT (it's still a bit slower than PIO but not so drastically).
Without NPT you still need a page walk so it will be slow.
Do you mean NPT/EPT?
If your concern is shadow paging, then I think you're concerned about hardware that is so slow to start with that it's not worth considering.
Quoted 14 lines
Quoted 14 lines
Quoted 14 lines
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Only because the chance it's 100% compatible on the software level is 0. It always has some hardware specific quirks.
No such excuse here.Quoted 2 lines
It also means we don't need to play games about sometimes enabling IO bars and sometimes not.
This last paragraph is wrong, it ignores the issues 3) to 5) I added above.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
If you do take them into account:
- there are reasons to add MMIO BAR to PCI, even without PCI expressSo far, the only reason you've provided is "it doesn't work on some architectures." Which architectures?
PowerPC wants this.
Existing PowerPC remaps PIO to MMAP so it works fine today.
Future platforms may not do this but future platforms can use a different device. They certainly won't be able to use the existing drivers anyway.
Ben, am I wrong here?
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
- we won't be able to drop IO BAR from virtio
An IO BAR is useless if it means we can't have more than 12 devices.
It's not useless. A smart BIOS can enable devices one by one as it tries to boot from them.
A smart BIOS can also use MMIO to program virtio.
Regards,
Anthony Liguori
Quoted 4 lines · Anthony Liguori
Quoted 2 lines · H. Peter Anvin
Quoted 1 line · Michael S. Tsirkin
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Spec says IO and memory can be enabled/disabled, separately. PCI Express spec says devices should work without IO.
For "native endpoints". Currently virtio would be a "legacy endpoint" which is quite correct -- it is compatible with a legacy interface.
Do legacy endpoints also use 4k for BARs?
There are no 4K BARs. In fact, I/O BARs are restricted by spec (there is no technical enforcement, however) to 256 bytes.
The 4K come from the upstream bridge windows, which are only 4K granular (historic stuff from when bridges were assumed rare.) However, there can be multiple devices, functions, and BARs inside that window.
The issue with PCIe is that each PCIe port is a bridge, so in reality there is only one real device per bus number.
Quoted 2 lines
If not, can't we use a new device id for native endpoints and call it a day? Legacy endpoints would continue using the existing BAR layout.
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
-hpa
Quoted 12 lines · H. Peter Anvin
Quoted 11 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 4 lines · Michael S. Tsirkin
Spec says IO and memory can be enabled/disabled, separately. PCI Express spec says devices should work without IO.
For "native endpoints". Currently virtio would be a "legacy endpoint" which is quite correct -- it is compatible with a legacy interface.
Quoted 8 lines
Quoted 1 line
Do legacy endpoints also use 4k for BARs?
There are no 4K BARs. In fact, I/O BARs are restricted by spec (there is no technical enforcement, however) to 256 bytes.
The 4K come from the upstream bridge windows, which are only 4K granular (historic stuff from when bridges were assumed rare.) However, there can be multiple devices, functions, and BARs inside that window.
Got it.
Quoted 10 lines
The issue with PCIe is that each PCIe port is a bridge, so in reality there is only one real device per bus number.
Quoted 2 lines
If not, can't we use a new device id for native endpoints and call it a day? Legacy endpoints would continue using the existing BAR layout.
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
Regards,
Anthony Liguori
Quoted 7 lines
-hpa
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 7 lines · Anthony Liguori
Existing PowerPC remaps PIO to MMAP so it works fine today.
Future platforms may not do this but future platforms can use a different device. They certainly won't be able to use the existing drivers anyway.
Ben, am I wrong here?
Well, we do remap PIO, though it's ugly and annoying so we'd rather avoid it alltogether :-)
Our latest HW PHBs don't support PIO at all, so while we can still support it in the virtual ones in the guest for ever, it's becoming pretty clear that PIO is on the way out ...
Quoted 9 lines
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
- we won't be able to drop IO BAR from virtio
An IO BAR is useless if it means we can't have more than 12 devices.
It's not useless. A smart BIOS can enable devices one by one as it tries to boot from them.
A smart BIOS can also use MMIO to program virtio.
Indeed :-)
I see no reason why not providing both access path though. Have the PIO BAR there for compatibility/legacy/BIOS/x86 purposes and *also* have the MMIO window which I'd be happy to favor on power.
We could even put somewhere in there a feature bit set by qemu to indicate whether it thinks PIO or MMIO is faster on a given platform if you really think that's worth it (I don't).
Cheers,
Ben.
Quoted 6 lines · Benjamin Herrenschmidt
Quoted 1 line · Anthony Liguori
A smart BIOS can also use MMIO to program virtio.
Indeed :-)
Quoted 7 lines
I see no reason why not providing both access path though. Have the PIO BAR there for compatibility/legacy/BIOS/x86 purposes and *also* have the MMIO window which I'd be happy to favor on power.
We could even put somewhere in there a feature bit set by qemu to indicate whether it thinks PIO or MMIO is faster on a given platform if you really think that's worth it (I don't).
That's okay, but what I'm most concerned about is compatibility.
A virtio PCI device that's a "native endpoint" needs to have a different device ID than one that is a "legacy endpoint". The current drivers have no hope of working (well) with virtio PCI devices exposed as native endpoints.
I don't care if the native PCI endpoint also has a PIO bar. But it seems silly (and confusing) to me to make that layout be the "legacy" layout verses a straight mirror of the new layout if we're already changing the device ID.
In addition, it doesn't seem at all necessary to have an MMIO bar to the legacy device. If the reason you want MMIO is to avoid using PIO, then you break existing drivers because they assume PIO. If you are breaking existing drivers then you should change the device ID.
If strictly speaking it's just that MMIO is a bit faster, I'm not sure that complexity is worth it without seeing performance numbers first.
Regards,
Anthony Liguori
Quoted 9 lines
Cheers,
Ben.--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
-hpa
Quoted 2 lines · Anthony Liguori
If strictly speaking it's just that MMIO is a bit faster, I'm not sure that complexity is worth it without seeing performance numbers first.
You mean PIO. I agree with all your points here. The only thing I saw as an option would be to add a PIO BAR that is an exact mirror of the MMIO BAR if and only if:
- PIO is indeed significantly faster on platforms we care about
and/or
- There is significant simplification in things like BIOSes in using PIO over MMIO
I personally don't care and would chose to use MMIO always including in firmware on ppc.
Cheers,
Ben.
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Regards,
Anthony Liguori
Quoted 8 lines
-hpa
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 4 lines · Anthony Liguori
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely), give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.
Now you have a different compatibility problem; how do you know the guest supports the new virtio-pcie net?
If you put a virtio-pci card behind a PCI-e bridge today, it's not compliant, but AFAICT it will Just Work. (Modulo the 16-dev limit).
I've been assuming we'd avoid a "flag day" change; that devices would look like existing virtio-pci with capabilities indicating the new config layout.
Quoted 4 lines
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
Removing both forward and backward compatibility is easy to explain, but I think it'll be harder to deploy. This is your area though, so perhaps I'm wrong.
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Noone expected the new cards to Just Work with old OSes: a new machine meant a new OS and new drivers. Hardware vendors like that.
Since virtualization often involves legacy, our priorities might be different.
Cheers,
Rusty.
Quoted 14 lines · Anthony Liguori
Quoted 12 lines · H. Peter Anvin
Quoted 10 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
Quoted 9 lines
Quoted 3 lines
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
--
Gleb.
Michael S. Tsirkin
maintainerQuoted 48 lines · Rusty Russell
Quoted 47 lines · Michael S. Tsirkin
Quoted 30 lines · Rusty Russell
Quoted 29 lines · Michael S. Tsirkin
Quoted 14 lines · Anthony Liguori
Quoted 11 lines · Rusty Russell
Quoted 5 lines · Anthony Liguori
Forcing a guest driver change is a really big
deal and I see no reason to do that unless there's a compelling reason to.So we're stuck with the 1.0 config layout for a very long time.
We definitely must not force a guest change. The explicit aim of the standard is that "legacy" and 1.0 be backward compatible. One deliverable is a document detailing how this is done (effectively a summary of changes between what we have and 1.0).
Quoted 7 lines
If 2.0 is fully backwards compatible, great. It seems like such a difference that that would be impossible but I need to investigate further.
Regards,
Anthony Liguori
If you look at my patches you'll see how it works. Basically old guests use BAR0 new ones don't, so it's easy: BAR0 access means legacy guest. Only started testing but things seem to work fine with old guests so far.
Quoted 7 lines
Quoted 3 lines
I think we need a spec, not just driver code.
Rusty what's the plan? Want me to write it?
We need both, of course, but the spec work will happen in the OASIS WG. A draft is good, but let's not commit anything to upstream QEMU until we get the spec finalized. And that is proposed to be late this year.
Well that would be quite sad really.
This means we can't make virtio a spec compliant pci express device, and we can't add any more feature bits, so no
flexible buffer optimizations for virtio net.There are probably more projects that will be blocked.
Quoted 7 lines
Quoted 4 lines
So how about we keep extending legacy layout for a bit longer:
- add a way to access device with MMIO
- use feature bit 31 to signal 64 bit features
(and shift device config accordingly)By my count, net still has 7 feature bits left, so I don't think the feature bits are likely to be a limitation in the next 6 months?
Actually I count 5 net specific ones:
3, 4 and then 25, 26, 27
Unless you count 31 even though it's a generic transport bit?
That's still 6 ...
Quoted 5 lines
MMIO is a bigger problem. Linux guests are happy with it: does it break the Windows drivers?
Thanks,
Rusty.
Quoted 9 lines · Gleb Natapov
Quoted 4 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
In some ways it is even worse for PXE, since PXE is defined to work from 16-bit protected mode... but the mechanism to request a segment mapping for the high memory area doesn't actually work.
-hpa
Hi Rusty,
Quoted 8 lines · Rusty Russell
Quoted 4 lines · Anthony Liguori
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely), give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.Now you have a different compatibility problem; how do you know the guest supports the new virtio-pcie net?
We don't care.
We would still use virtio-pci for existing devices. Only new devices would use virtio-pcie.
Quoted 2 lines
If you put a virtio-pci card behind a PCI-e bridge today, it's not compliant, but AFAICT it will Just Work. (Modulo the 16-dev limit).
I believe you can put it in legacy mode and then there isn't the 16-dev limit. I believe the only advantage of putting it in native mode is that then you can do native hotplug (as opposed to ACPI hotplug).
So sticking with virtio-pci seems reasonable to me.
Quoted 3 lines
I've been assuming we'd avoid a "flag day" change; that devices would look like existing virtio-pci with capabilities indicating the new config layout.
I don't think that's feasible. Maybe 5 or 10 years from now, we switch the default adapter to virtio-pcie.
Quoted 8 lines
Quoted 4 lines
I think 4 is the best path forward. It's better for users (guests continue to work as they always have). There's less confusion about enabling PCI-e support--you must ask for the virtio-pcie variant and you must have a virtio-pcie driver. It's easy to explain.
Removing both forward and backward compatibility is easy to explain, but I think it'll be harder to deploy. This is your area though, so perhaps I'm wrong.
My concern is that it's not real backwards compatibility.
Quoted 6 lines
Quoted 3 lines
It also maps to what regular hardware does. I highly doubt that there are any real PCI cards that made the shift from PCI to PCI-e without bumping at least a revision ID.
Noone expected the new cards to Just Work with old OSes: a new machine meant a new OS and new drivers. Hardware vendors like that.
Yup.
Quoted 2 lines
Since virtualization often involves legacy, our priorities might be different.
So realistically, I think if we introduce virtio-pcie with a different vendor ID, it will be adopted fairly quickly. The drivers will show up in distros quickly and get backported.
New devices can be limited to supporting virtio-pcie and we'll certainly provide a way to use old devices with virtio-pcie too. But for practical reasons, I think we have to continue using virtio-pci by default.
Regards,
Anthony Liguori
Quoted 7 lines
Cheers,
Rusty.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
SeaBIOS is polling for completion anyway.
Regards,
Anthony Liguori
Quoted 7 lines
--
Gleb.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
SeaBIOS is polling for completion anyway.
Regards,
Anthony Liguori
Quoted 7 lines
--
Gleb.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Gleb Natapov
Quoted 6 lines · Anthony Liguori
Oh, you mean in real mode.
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler.
Exactly. It's "only" the initialization code which has ASSERt32FLAT() all over the place. Which actually is the majority of the code in most cases as all the hardware detection and initialization code is there. But kicking I/O requests must work from 16bit mode too.
Quoted 2 lines
The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
For seabios itself this isn't a big issue, see pci_{readl,writel} in src/pci.c. When called in 16bit mode it goes into 32bit mode temporarily, just for accessing the mmio register. ahci driver uses it, xhci driver (wip atm) will use that too, and virtio-{blk,scsi} drivers in seabios can do the same.
But as hpa mentioned it will be more tricky for option roms (aka virtio-net).
cheers,
Gerd
Quoted 17 lines · Gerd Hoffmann
Quoted 10 lines · Gleb Natapov
Quoted 6 lines · Anthony Liguori
Oh, you mean in real mode.
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler.
Exactly. It's "only" the initialization code which has
ASSERt32FLAT()all over the place. Which actually is the majority of the code in most cases as all the hardware detection and initialization code is there. But kicking I/O requests must work from 16bit mode too.
Quoted 9 lines
Quoted 2 lines
The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
For seabios itself this isn't a big issue, see pci_{readl,writel} in
src/pci.c. When called in 16bit mode it goes into 32bit mode temporarily, just for accessing the mmio register. ahci driver uses it, xhci driver (wip atm) will use that too, and virtio-{blk,scsi} drivers in seabios can do the same.
Isn't this approach broken? How can SeaBIOS be sure it restores real mode registers to exactly same state they were before entering 32bit mode?
Quoted 6 lines
But as hpa mentioned it will be more tricky for option roms (aka virtio-net).
cheers,
Gerd
--
Gleb.
Quoted 6 lines · Gleb Natapov
Quoted 1 line · Gerd Hoffmann
Isn't this approach broken? How can SeaBIOS be sure it restores real mode registers to exactly same state they were before entering 32bit mode?
It can't... so yes, it is broken.
-hpa
Hi,
Quoted 9 lines
Quoted 6 lines
For seabios itself this isn't a big issue, see pci_{readl,writel} in
src/pci.c. When called in 16bit mode it goes into 32bit mode temporarily, just for accessing the mmio register. ahci driver uses it, xhci driver (wip atm) will use that too, and virtio-{blk,scsi} drivers in seabios can do the same.Isn't this approach broken? How can SeaBIOS be sure it restores real mode registers to exactly same state they were before entering 32bit mode?
Don't know the details of that magic. Kevin had some concerns on the stability of this, so maybe there is a theoretic hole. So far I havn't seen any issues in practice, but also didn't stress it too much. Basically only used that with all kinds of boot loaders, could very well be it breaks if you try to use that with more esoteric stuff such as dos extenders, then hit unhandled corner cases ...
cheers,
Gerd
Quoted 10 lines · Anthony Liguori
Hi Rusty,
Quoted 6 lines · Rusty Russell
Quoted 4 lines · Anthony Liguori
4) Do virtio-pcie, make it PCI-e friendly (drop the IO BAR completely), give
it a new device/vendor ID. Continue to use virtio-pci for existing
devices potentially adding virtio-{net,blk,...}-pcie variants for people that care to use them.
Quoted 7 lines
Quoted 2 lines
Now you have a different compatibility problem; how do you know the guest supports the new virtio-pcie net?
We don't care.
We would still use virtio-pci for existing devices. Only new devices would use virtio-pcie.
My concern is that new features make the virtio-pcie driver so desirable that libvirt is pressured to use it ASAP. Have we just punted the problem upstream?
(Of course, feature bits were supposed to avoid such a transition issue, but mistakes accumulate).
Cheers,
Rusty.
Quoted 2 lines · Anthony Liguori
We would still use virtio-pci for existing devices. Only new devices would use virtio-pcie.
Surely you'd want to support both for any new device, because
(a) transport is orthogonal to backend functionality
(b) not all existing machine models have pci-e ?
Or am I misunderstanding?
thanks
-- PMM
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 6 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
And when backend stops polling? When guest's driver reconfigure it? But what if this will never happen?
Quoted 13 lines
SeaBIOS is polling for completion anyway.
Regards,
Anthony Liguori
Quoted 7 lines
--
Gleb.
--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
--
Gleb.
Michael S. Tsirkin
maintainerQuoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.
If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
Quoted 2 lines
SeaBIOS is polling for completion anyway.
I think that's different because a disk will normally respond quickly. So it polls a bit, but then it stops as there are no outstanding requests.
Signature · 1 line
MST
Quoted 38 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
Quoted 9 lines
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
I wouldn't call BIOS int13 interface performance critical.
Quoted 9 lines
Quoted 2 lines
SeaBIOS is polling for completion anyway.
I think that's different because a disk will normally respond quickly. So it polls a bit, but then it stops as there are no outstanding requests.
Signature · 1 line
MST
--
Gleb.
Michael S. Tsirkin
maintainerQuoted 43 lines · Gleb Natapov
Quoted 42 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.
Quoted 6 lines
Quoted 5 lines
If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
I wouldn't call BIOS int13 interface performance critical.
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to devices
hpa felt performance does matter there but didn't clarify why ...
Quoted 12 lines
Quoted 9 lines
Quoted 2 lines
SeaBIOS is polling for completion anyway.
I think that's different because a disk will normally respond quickly. So it polls a bit, but then it stops as there are no outstanding requests.
Signature · 1 line
MST--
Gleb.
Quoted 49 lines · Michael S. Tsirkin
Quoted 48 lines · Gleb Natapov
Quoted 47 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
Quoted 8 lines
Quoted 1 line
I wouldn't call BIOS int13 interface performance critical.
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to deviceshpa felt performance does matter there but didn't clarify why ...
You do not what to make it too slow obviously, this is interface that is used to load OS during boot.
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
SeaBIOS is polling for completion anyway.
I think that's different because a disk will normally respond quickly. So it polls a bit, but then it stops as there are no outstanding requests.
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Signature · 1 line
MST--
Gleb.
--
Gleb.
Michael S. Tsirkin
maintainerQuoted 49 lines · Michael S. Tsirkin
Quoted 48 lines · Gleb Natapov
Quoted 47 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
Quoted 7 lines
Quoted 1 line
I wouldn't call BIOS int13 interface performance critical.
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to deviceshpa felt performance does matter there but didn't clarify why ...
Also
<gleb> mst, well the question is if it is safe to call int13 in the middle of pci bus enumeration/configuration
<gleb> mst, and int13 predates PCI, so how knows
Quoted 12 lines
Quoted 12 lines
Quoted 9 lines
Quoted 2 lines
SeaBIOS is polling for completion anyway.
I think that's different because a disk will normally respond quickly. So it polls a bit, but then it stops as there are no outstanding requests.
Signature · 1 line
MST--
Gleb.
Michael S. Tsirkin
maintainerQuoted 52 lines · Gleb Natapov
Quoted 51 lines · Michael S. Tsirkin
Quoted 49 lines · Gleb Natapov
Quoted 38 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
Quoted 9 lines
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
I wouldn't call BIOS int13 interface performance critical.
Quoted 8 lines
Quoted 6 lines
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to deviceshpa felt performance does matter there but didn't clarify why ...
You do not what to make it too slow obviously, this is interface that is used to load OS during boot.
And possibly installation?
Quoted 15 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines
SeaBIOS is polling for completion anyway.
I think that's different because a disk will normally respond quickly. So it polls a bit, but then it stops as there are no outstanding requests.
Quoted 5 lines
Quoted 5 lines
Quoted 2 lines
Signature · 1 line
MST--
Gleb.--
Gleb.
Quoted 57 lines · Michael S. Tsirkin
Quoted 56 lines · Gleb Natapov
Quoted 55 lines · Michael S. Tsirkin
Quoted 43 lines · Gleb Natapov
Quoted 42 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.Quoted 6 lines
Quoted 5 lines
If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
I wouldn't call BIOS int13 interface performance critical.
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to devices
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines
hpa felt performance does matter there but didn't clarify why ...
You do not what to make it too slow obviously, this is interface that is used to load OS during boot.
And possibly installation?
Only the stage that reads files from CDROM. IIRC actual installation runs with native drivers. This is why Windows asks you to provide floppy with a driver at very early stage of installation.
--
Gleb.
Michael S. Tsirkin
maintainerQuoted 60 lines · Gleb Natapov
Quoted 59 lines · Michael S. Tsirkin
Quoted 58 lines · Gleb Natapov
Quoted 57 lines · Michael S. Tsirkin
Quoted 43 lines · Gleb Natapov
Quoted 42 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native devices, too, so having an I/O BAR (which would not be used by the OS driver) should still at the very least be an option.
What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.Quoted 6 lines
Quoted 5 lines
If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
I wouldn't call BIOS int13 interface performance critical.
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to deviceshpa felt performance does matter there but didn't clarify why ...
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
You do not what to make it too slow obviously, this is interface that is used to load OS during boot.
And possibly installation?
Only the stage that reads files from CDROM. IIRC actual installation runs with native drivers. This is why Windows asks you to provide floppy with a driver at very early stage of installation.
Have any numbers to tell us how much time is spent there? E.g. if it's slowed down by a factor of 2, is it a problem? How about a factor of 10?
Quoted 2 lines
--
Gleb.
Quoted 66 lines · Michael S. Tsirkin
Quoted 65 lines · Gleb Natapov
Quoted 64 lines · Michael S. Tsirkin
Quoted 52 lines · Gleb Natapov
Quoted 51 lines · Michael S. Tsirkin
Quoted 49 lines · Gleb Natapov
Quoted 38 lines · Michael S. Tsirkin
Quoted 31 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 20 lines · Anthony Liguori
Quoted 8 lines · H. Peter Anvin
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Definitely an option. However, we want to be able to boot from native
devices, too, so having an I/O BAR (which would not be used by the OS
driver) should still at the very least be an option.What makes it so difficult to work with an MMIO bar for PCI-e?
Quoted 7 lines
Quoted 3 lines
With legacy PCI, tracking allocation of MMIO vs. PIO is pretty straight forward. Is there something special about PCI-e here?
It's not tracking allocation. It is that accessing memory above 1 MiB is incredibly painful in the BIOS environment, which basically means MMIO is inaccessible.
Oh, you mean in real mode.
Quoted 7 lines
Quoted 3 lines
SeaBIOS runs the virtio code in 32-bit mode with a flat memory layout. There are loads of ASSERT32FLAT()s in the code to make sure of this.
Well, not exactly. Initialization is done in 32bit, but disk reads/writes are done in 16bit mode since it should work from int13 interrupt handler. The only way I know to access MMIO bars from 16 bit is to use SMM which we do not have in KVM.
Quoted 5 lines
Ah, if it's just the dataplane operations then there's another solution.
We can introduce a virtqueue flag that asks the backend to poll for new requests. Then SeaBIOS can add the request to the queue and not worry about kicking or reading the ISR.
Quoted 9 lines
This will pin a host CPU.
If we do something timer based it will likely both increase host CPU utilization and slow device down.If we didn't care about performance at all we could do config cycles for signalling, which is much more elegant than polling in host, but I don't think that's the case.
I wouldn't call BIOS int13 interface performance critical.
Quoted 8 lines
Quoted 6 lines
So the plan always was to
- add an MMIO BAR
- add a register for pci-config based access to deviceshpa felt performance does matter there but didn't clarify why ...
You do not what to make it too slow obviously, this is interface that is used to load OS during boot.
And possibly installation?
Quoted 8 lines
Quoted 3 lines
Only the stage that reads files from CDROM. IIRC actual installation runs with native drivers. This is why Windows asks you to provide floppy with a driver at very early stage of installation.
Have any numbers to tell us how much time is spent there? E.g. if it's slowed down by a factor of 2, is it a problem? How about a factor of 10?
No I do not.
--
Gleb.
I only recently saw this email.
Quoted 10 lines · Gleb Natapov
Quoted 6 lines · Gerd Hoffmann
For seabios itself this isn't a big issue, see pci_{readl,writel} in
src/pci.c. When called in 16bit mode it goes into 32bit mode temporarily, just for accessing the mmio register. ahci driver uses it, xhci driver (wip atm) will use that too, and virtio-{blk,scsi} drivers in seabios can do the same.Isn't this approach broken? How can SeaBIOS be sure it restores real mode registers to exactly same state they were before entering 32bit mode?
You are correct - SeaBIOS can't fully restore the "hidden" segment registers. So, in a way it is broken.
In practice, it seems to work on modern bootloaders (eg, ntldr, grub, lilo). It definitely doesn't work with EMM386 (old DOS stuff), but does seem to work okay with FreeDOS as long as one doesn't run EMM386.
The AHCI code uses this 32bit/16bit trampoline because it would not be possible to support AHCI otherwise. I haven't seen any complaints of failures with the AHCI code - probably because people using AHCI are using modern guests.
I explored this a bit some time back and the only way I could think of to reliably restore the 16bit registers would be via SMM. Unfortunately, using SMM introduces a whole host of complexity and problems.
-Kevin