Enable non page boundary BAR device assignment
PATCH 1/1 Enable non page boundary BAR device assignment
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K]
Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256]
Region 4: I/O ports at b100 [size=256]Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size & 0xFFF != 0 sized regions by looping it through userspace.
I verified that it works by passing through an e1000 with this additional patch applied and the card acted the same way it did without this patch:
map_func = assigned_dev_iomem_map;
- if (cur_region->size & 0xFFF) {
+ if (i != PCI_ROM_SLOT){
fprintf(stderr, "PCI region %d at address 0x%llx "--- hw/device-assignment.c | 129 +++++++++++++++++++++++++++++++++++++++++++++-- 1 files changed, 123 insertions(+), 6 deletions(-)
hw/device-assignment.c +123 -6
@@ -148,6 +148,105 @@ static uint32_t assigned_dev_ioport_readl(void *opaque, uint32_t addr)return value; } +static uint32_t slow_bar_readb (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint8_t *in = (uint8_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static uint32_t slow_bar_readw (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static uint32_t slow_bar_readl (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static void slow_bar_writeb (void *opaque,target_phys_addr_t addr,uint32_t val) +{ +AssignedDevRegion *d = opaque; +uint8_t *out = (uint8_t *)(d->u.r_virtbase + addr); + +DEBUG ("slow_bar_writeb addr=0x" TARGET_FMT_plx" val=0x%02x\n" , addr, val); + *out = val; +} + +static void slow_bar_writew (void *opaque,target_phys_addr_t addr,uint32_t val) +{ +AssignedDevRegion *d = opaque; +uint16_t *out = (uint16_t *)(d->u.r_virtbase + addr); + +DEBUG ("slow_bar_writew addr=0x" TARGET_FMT_plx" val=0x%04x\n" , addr, val); + *out = val; +} + +static void slow_bar_writel (void *opaque,target_phys_addr_t addr,uint32_t val) +{ +AssignedDevRegion *d = opaque; +uint32_t *out = (uint32_t *)(d->u.r_virtbase + addr); + +DEBUG ("slow_bar_writel addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, val); + *out = val; +} + +static CPUWriteMemoryFunc *const slow_bar_write[] = { + &slow_bar_writeb, + &slow_bar_writew, + &slow_bar_writel +}; + +static CPUReadMemoryFunc *const slow_bar_read[] = { + &slow_bar_readb, + &slow_bar_readw, + &slow_bar_readl +}; + +static void assigned_dev_iomem_map_slow (PCIDevice *pci_dev,int region_num, +pcibus_t e_phys,pcibus_t e_size, +int type) +{ +AssignedDevice *r_dev =container_of (pci_dev, AssignedDevice, dev); +AssignedDevRegion *region = &r_dev->v_addrs[region_num]; +PCIRegion *real_region = &r_dev->real_device.regions[region_num]; +int m; + +DEBUG ("slow map\n" ); + m =cpu_register_io_memory (slow_bar_read, slow_bar_write, region); +cpu_register_physical_memory (e_phys, e_size, m); + +/* MSI-X MMIO page */ +if ((e_size >0 ) && + real_region->base_addr <= r_dev->msix_table_addr && + real_region->base_addr + real_region->size >= r_dev->msix_table_addr) { +int offset = r_dev->msix_table_addr - real_region->base_addr; + +cpu_register_physical_memory (e_phys + offset, +TARGET_PAGE_SIZE , r_dev->mmio_index); + } +} +static void assigned_dev_iomem_map (PCIDevice *pci_dev,int region_num,pcibus_t e_phys,pcibus_t e_size,int type) {
@@ -381,15 +480,25 @@ static int assigned_dev_register_regions(PCIRegion *io_regions,/* handle memory io regions */ if (cur_region->type &IORESOURCE_MEM ) { +void (*map_func)(PCIDevice *,int ,pcibus_t ,pcibus_t ,int ); +int slow_map =0 ;int t = cur_region->type &IORESOURCE_PREFETCH ?PCI_BASE_ADDRESS_MEM_PREFETCH :PCI_BASE_ADDRESS_SPACE_MEMORY ; + + map_func = assigned_dev_iomem_map;if (cur_region->size &0xFFF ) { -fprintf (stderr,"Unable to assign device: PCI region %d " -"at address 0x%llx has size 0x%x, " -" which is not a multiple of 4K\n" , +fprintf (stderr,"PCI region %d at address 0x%llx " +"has size 0x%x, which is not a multiple of 4K. " +"Using slow map\n" , i, (unsigned long long )cur_region->base_addr, cur_region->size); + map_func = assigned_dev_iomem_map_slow; + slow_map =1 ; + } + +if (slow_map && (i ==PCI_ROM_SLOT )) { +fprintf (stderr,"ROM not aligned - can't continue\n" );return -1 ; }
@@ -402,6 +511,12 @@ static int assigned_dev_register_regions(PCIRegion *io_regions,PROT_WRITE |PROT_READ ,MAP_ANONYMOUS |MAP_PRIVATE , 0, (off_t) 0); + }else if (slow_map) { + pci_dev->v_addrs[i].u.r_virtbase = +mmap (NULL , + cur_region->size, +PROT_WRITE |PROT_READ ,MAP_SHARED , + cur_region->resource_fd, (off_t )0 ); }else { pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL,
@@ -429,12 +544,14 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, pci_dev->v_addrs[i].e_size =0 ;/* add offset */ - pci_dev->v_addrs[i].u.r_virtbase += - (cur_region->base_addr &0xFFF ); +if (!slow_map) { + pci_dev->v_addrs[i].u.r_virtbase += + (cur_region->base_addr &0xFFF ); + }pci_register_bar ((PCIDevice *) pci_dev, i, cur_region->size, t, - assigned_dev_iomem_map); + map_func);continue ; }/* handle port io regions */
Signature · 1 line
1.6.0.2
Quoted 21 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.I verified that it works by passing through an e1000 with this additional patch applied and the card acted the same way it did without this patch:
map_func = assigned_dev_iomem_map;(quoted diff) +1
-1- if (cur_region->size & 0xFFF) { + if (i != PCI_ROM_SLOT){ fprintf(stderr, "PCI region %d at address 0x%llx "
Quoted 1 line
Signed-off-by Alexander Graf <[email protected]>
Good idea.
One thing I am concerned with, is that some users might see performance degradation and not see the error message. Maybe we can have a flag to optionally fail unless passthrough can be fast? I'm not really sure it's worth it to add such a flag though - what do others think?
Minor nits below.
Quoted 115 lines
--- hw/device-assignment.c | 129 +++++++++++++++++++++++++++++++++++++++++++++-- 1 files changed, 123 insertions(+), 6 deletions(-)hw/device-assignment.c +99
-0@@ -148,6 +148,105 @@ static uint32_t assigned_dev_ioport_readl(void *opaque, uint32_t addr)return value; } +static uint32_t slow_bar_readb (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint8_t *in = (uint8_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static uint32_t slow_bar_readw (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static uint32_t slow_bar_readl (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static void slow_bar_writeb (void *opaque,target_phys_addr_t addr,uint32_t val) +{ +AssignedDevRegion *d = opaque; +uint8_t *out = (uint8_t *)(d->u.r_virtbase + addr); + +DEBUG ("slow_bar_writeb addr=0x" TARGET_FMT_plx" val=0x%02x\n" , addr, val); + *out = val; +} + +static void slow_bar_writew (void *opaque,target_phys_addr_t addr,uint32_t val) +{ +AssignedDevRegion *d = opaque; +uint16_t *out = (uint16_t *)(d->u.r_virtbase + addr); + +DEBUG ("slow_bar_writew addr=0x" TARGET_FMT_plx" val=0x%04x\n" , addr, val); + *out = val; +} + +static void slow_bar_writel (void *opaque,target_phys_addr_t addr,uint32_t val) +{ +AssignedDevRegion *d = opaque; +uint32_t *out = (uint32_t *)(d->u.r_virtbase + addr); + +DEBUG ("slow_bar_writel addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, val); + *out = val; +} + +static CPUWriteMemoryFunc *const slow_bar_write[] = { + &slow_bar_writeb, + &slow_bar_writew, + &slow_bar_writel +}; + +static CPUReadMemoryFunc *const slow_bar_read[] = { + &slow_bar_readb, + &slow_bar_readw, + &slow_bar_readl +}; + +static void assigned_dev_iomem_map_slow (PCIDevice *pci_dev,int region_num, +pcibus_t e_phys,pcibus_t e_size, +int type) +{ +AssignedDevice *r_dev =container_of (pci_dev, AssignedDevice, dev); +AssignedDevRegion *region = &r_dev->v_addrs[region_num]; +PCIRegion *real_region = &r_dev->real_device.regions[region_num]; +int m; + +DEBUG ("slow map\n" ); + m =cpu_register_io_memory (slow_bar_read, slow_bar_write, region); +cpu_register_physical_memory (e_phys, e_size, m); + +/* MSI-X MMIO page */ +if ((e_size >0 ) && + real_region->base_addr <= r_dev->msix_table_addr && + real_region->base_addr + real_region->size >= r_dev->msix_table_addr) { +int offset = r_dev->msix_table_addr - real_region->base_addr; + +cpu_register_physical_memory (e_phys + offset, +TARGET_PAGE_SIZE , r_dev->mmio_index); + } +} +static void assigned_dev_iomem_map (PCIDevice *pci_dev,int region_num,pcibus_t e_phys,pcibus_t e_size,int type) {
hw/device-assignment.c static int assigned_dev_register_regions(PCIRegion *io_regions, · line 484
Quoted 5 lines
hw/device-assignment.c +2
-0@@ -381,15 +480,25 @@ static int assigned_dev_register_regions(PCIRegion *io_regions,/* handle memory io regions */ if (cur_region->type &IORESOURCE_MEM ) { +void (*map_func)(PCIDevice *,int ,pcibus_t ,pcibus_t ,int ); +int slow_map =0 ;
This is a bit too clever for my taste. Let's just do below
pci_register_bar((PCIDevice *) pci_dev, i,
cur_region->size, t,
slow_map ? assigned_dev_iomem_map : assigned_dev_iomem_map_slow);(by the way, this cast to (PCIDevice *) is also unnecessary, isn't, it?
hw/device-assignment.c static int assigned_dev_register_regions(PCIRegion *io_regions, · line 493
Quoted 12 lines
hw/device-assignment.c +5
-3int t = cur_region->type &IORESOURCE_PREFETCH ?PCI_BASE_ADDRESS_MEM_PREFETCH :PCI_BASE_ADDRESS_SPACE_MEMORY ; + + map_func = assigned_dev_iomem_map;if (cur_region->size &0xFFF ) { - fprintf(stderr, "Unable to assign device: PCI region %d " - "at address 0x%llx has size 0x%x, " - " which is not a multiple of 4K\n", + fprintf(stderr, "PCI region %d at address 0x%llx " + "has size 0x%x, which is not a multiple of 4K. " + "Using slow map\n",
Maybe replace "Using slow map" with "The device will be slow.".
Quoted 13 lines
hw/device-assignment.c +6
-0i, (unsigned long long )cur_region->base_addr, cur_region->size); + map_func = assigned_dev_iomem_map_slow; + slow_map =1 ; + } + +if (slow_map && (i ==PCI_ROM_SLOT )) { +fprintf (stderr,"ROM not aligned - can't continue\n" );return -1 ; }
hw/device-assignment.c static int assigned_dev_register_regions(PCIRegion *io_regions, · line 519
Quoted 9 lines
hw/device-assignment.c +6
-0@@ -402,6 +511,12 @@ static int assigned_dev_register_regions(PCIRegion *io_regions,PROT_WRITE |PROT_READ ,MAP_ANONYMOUS |MAP_PRIVATE , 0, (off_t) 0); + }else if (slow_map) { + pci_dev->v_addrs[i].u.r_virtbase = +mmap (NULL , + cur_region->size, +PROT_WRITE |PROT_READ ,MAP_SHARED , + cur_region->resource_fd, (off_t )0 );
Isn't this the same as the case below it?
I expect mmap to act identically for slow and fast ... Just replace (cur_region->size + 0xFFF) & 0xFFFFF000, with cur_region->size.
Quoted 23 lines
hw/device-assignment.c +5
-3}else { pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL,@@ -429,12 +544,14 @@ static int assigned_dev_register_regions(PCIRegion *io_regions, pci_dev->v_addrs[i].e_size =0 ;/* add offset */ - pci_dev->v_addrs[i].u.r_virtbase += - (cur_region->base_addr &0xFFF ); +if (!slow_map) { + pci_dev->v_addrs[i].u.r_virtbase += + (cur_region->base_addr &0xFFF ); + }pci_register_bar ((PCIDevice *) pci_dev, i, cur_region->size, t, - assigned_dev_iomem_map); + map_func);continue ; }/* handle port io regions */ Signature · 1 line
1.6.0.2
Am 09.12.2009 um 21:49 schrieb "Michael S. Tsirkin" <[email protected]>:
Quoted 26 lines · Alexander Graf
Quoted 23 lines
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.I verified that it works by passing through an e1000 with this additional patch applied and the card acted the same way it did without this patch:
map_func = assigned_dev_iomem_map;(quoted diff) +1
-1- if (cur_region->size & 0xFFF) { + if (i != PCI_ROM_SLOT){ fprintf(stderr, "PCI region %d at address 0x%llx "Quoted 1 line
Signed-off-by Alexander Graf <[email protected]>
Quoted 7 lines
Good idea.
One thing I am concerned with, is that some users might see performance degradation and not see the error message. Maybe we can have a flag to optionally fail unless passthrough can be fast? I'm not really sure it's worth it to add such a flag though - what do others think?
It only kicks in on small BARs which are usually not _that_ performance critical. We're mostly talking about ack and status bits.
Also not failing > failing IMHO :). Even if it's not as fast as native.
Quoted 142 lines
Minor nits below.
Quoted 139 lines
---
hw/device-assignment.c| 129 ++++++++++++++++++++++++++++++++++++++
+++++++--1 files changed, 123 insertions(+), 6 deletions(-)hw/device-assignment.c +13
-0@@ -148,6 +148,105 @@ static uint32_t assigned_dev_ioport_readl (void *opaque, uint32_t addr)return value; } +static uint32_t slow_bar_readb (void *opaque,target_phys_addr_t addr) +{ +AssignedDevRegion *d = opaque; +uint8_t *in = (uint8_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; +DEBUG ("slow_bar_readl addr=0x" TARGET_FMT_plx" val=0x%08x\n" , addr, r); + +return r; +} + +static uint32_t slow_bar_readw(void *opaque, target_phys_addr_taddr)
hw/device-assignment.c +7
-0+{ +AssignedDevRegion *d = opaque; +uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; + DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n",addr, r);
hw/device-assignment.c +5
-0+ +return r; +} + +static uint32_t slow_bar_readl(void *opaque, target_phys_addr_taddr)
hw/device-assignment.c +7
-0+{ +AssignedDevRegion *d = opaque; +uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr); +uint32_t r =-1 ; + + r = *in; + DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n",addr, r);
hw/device-assignment.c +5
-0+ +return r; +} + +static void slow_bar_writeb(void *opaque, target_phys_addr_t addr,
uint32_tval)hw/device-assignment.c +5
-0+{ + AssignedDevRegion *d = opaque; + uint8_t *out = (uint8_t*)(d->u.r_virtbase + addr); + + DEBUG("slow_bar_writeb addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr, val);
hw/device-assignment.c +4
-0+ *out = val; +} + +static void slow_bar_writew(void *opaque, target_phys_addr_t addr,
uint32_tval)hw/device-assignment.c +5
-0+{ + AssignedDevRegion *d = opaque; + uint16_t *out = (uint16_t*)(d->u.r_virtbase + addr); + + DEBUG("slow_bar_writew addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr, val);
hw/device-assignment.c +4
-0+ *out = val; +} + +static void slow_bar_writel(void *opaque, target_phys_addr_t addr,
uint32_tval)hw/device-assignment.c +5
-0+{ + AssignedDevRegion *d = opaque; + uint32_t *out = (uint32_t*)(d->u.r_virtbase + addr); + + DEBUG("slow_bar_writel addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, val);
hw/device-assignment.c +16
-0+ *out = val; +} + +static CPUWriteMemoryFunc *const slow_bar_write[] = { + &slow_bar_writeb, + &slow_bar_writew, + &slow_bar_writel +}; + +static CPUReadMemoryFunc *const slow_bar_read[] = { + &slow_bar_readb, + &slow_bar_readw, + &slow_bar_readl +}; + +static void assigned_dev_iomem_map_slow(PCIDevice *pci_dev, int
region_num,
+pcibus_te_phys,pcibus_te_size,hw/device-assignment.c +3
-0+ int type) +{ + AssignedDevice *r_dev = container_of(pci_dev, AssignedDevice,dev);
hw/device-assignment.c +2
-0+AssignedDevRegion *region = &r_dev->v_addrs[region_num]; +PCIRegion *real_region = &r_dev->real_device.regions[
region_num];hw/device-assignment.c +4
-0+int m; + +DEBUG ("slow map\n" ); + m = cpu_register_io_memory(slow_bar_read, slow_bar_write,region);
hw/device-assignment.c +6
-0+cpu_register_physical_memory (e_phys, e_size, m); + +/* MSI-X MMIO page */ +if ((e_size >0 ) && + real_region->base_addr <= r_dev->msix_table_addr && + real_region->base_addr + real_region->size >= r_dev-Quoted 1 line
msix_table_addr) {+ int offset =
r_dev->msix_table_addr- real_region-Quoted 1 line
base_addr;hw/device-assignment.c +6
-0+ +cpu_register_physical_memory (e_phys + offset, +TARGET_PAGE_SIZE , r_dev->mmio_index); + } +} +static void assigned_dev_iomem_map(PCIDevice
*pci_dev, intregion_num,
pcibus_te_phys,pcibus_te_size, int type)
{hw/device-assignment.c +0
-0@@ -381,15 +480,25 @@ static int assigned_dev_register_regions (PCIRegion *io_regions,
Quoted 11 lines
hw/device-assignment.c static int assigned_dev_register_regions(PCIRegion *io_regions, · line 484
Quoted 5 lines
hw/device-assignment.c +2
-0/* handle memory io regions */ if (cur_region->type &IORESOURCE_MEM ) { +void (*map_func)(PCIDevice *,int ,pcibus_t ,pcibus_t ,int ); +int slow_map =0 ;This is a bit too clever for my taste. Let's just do below
pci_register_bar((PCIDevice *)pci_dev, i,
cur_region->size, t,
slow_map?assigned_dev_iomem_map:assigned_dev_iomem_map_slow);
We could do that too, yeah. It's a matter of taste I guess. I find function pointers pretty easily readable :).
Quoted 18 lines
(by the way, this cast to (PCIDevice *) is also unnecessary, isn't, it?
Quoted 12 lines
hw/device-assignment.c +2
-1int t = cur_region->type &IORESOURCE_PREFETCH ?PCI_BASE_ADDRESS_MEM_PREFETCH :PCI_BASE_ADDRESS_SPACE_MEMORY ; + + map_func = assigned_dev_iomem_map;if (cur_region->size &0xFFF ) { - fprintf(stderr, "Unable to assign device: PCIregion %d "
hw/device-assignment.c +2
-2-"at address 0x%llx has size 0x%x, " - " which is not a multiple of 4K\n", +fprintf (stderr,"PCI region %d at address 0x%llx " + "has size 0x%x, which is not a multiple ofQuoted 2 lines
4K. "
+ "Using slow map\n",
Quoted 1 line
Maybe replace "Using slow map" with "The device will be slow.".
I doubt it would. So far I only found the emulex and an ehci controller that have BARs that are no multiple of page_size. I didn't test the ehci controller and the emulex is still broken despite this patch, probably interrupt related.
Quoted 20 lines
Quoted 20 lines
i, (unsigned long long)cur_region->base_addr, cur_region->size);hw/device-assignment.c +6
-0+ map_func = assigned_dev_iomem_map_slow; + slow_map =1 ; + } + +if (slow_map && (i ==PCI_ROM_SLOT )) { + fprintf(stderr, "ROM not aligned - can't continue\n"); return -1; }hw/device-assignment.c +0
-0@@ -402,6 +511,12 @@ static int assigned_dev_register_regions (PCIRegion *io_regions,PROT_WRITE |PROT_READ ,MAP_ANONYMOUS |MAP_PRIVATE ,0 , (off_t )0 );
Quoted 11 lines
hw/device-assignment.c static int assigned_dev_register_regions(PCIRegion *io_regions, · line 519
Quoted 6 lines
hw/device-assignment.c +6
-0+ }else if (slow_map) { + pci_dev->v_addrs[i].u.r_virtbase = +mmap (NULL , + cur_region->size, +PROT_WRITE |PROT_READ ,MAP_SHARED , + cur_region->resource_fd, (off_t )0 );Isn't this the same as the case below it?
I expect mmap to act identically for slow and fast ... Just replace (cur_region->size+0xFFF) &0xFFFFF000, withcur_region->size.
You mean remove the alignment for the fast case? Hum, we shouldn't hit unaligned bars with fast mapping anymore, so I guess you're right.
Quoted 29 lines
Quoted 22 lines
hw/device-assignment.c +0
-0}else {pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL,hw/device-assignment.c +5
-3@@ -429,12 +544,14 @@ static int assigned_dev_register_regions (PCIRegion *io_regions, pci_dev->v_addrs[i].e_size = 0;/* add offset */ - pci_dev->v_addrs[i].u.r_virtbase += - (cur_region->base_addr &0xFFF ); +if (!slow_map) { + pci_dev->v_addrs[i].u.r_virtbase += + (cur_region->base_addr &0xFFF ); + }pci_register_bar ((PCIDevice *) pci_dev, i, cur_region->size, t, - assigned_dev_iomem_map); + map_func);continue ; }Quoted 3 lines
/* handle port io regions */Signature · 1 line
1.6.0.2--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 31 lines · Alexander Graf
Am 09.12.2009 um 21:49 schrieb "Michael S. Tsirkin" <[email protected]>:
Quoted 28 lines · Alexander Graf
Quoted 23 lines
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.I verified that it works by passing through an e1000 with this additional patch applied and the card acted the same way it did without this patch:
map_func = assigned_dev_iomem_map;(quoted diff) +1
-1- if (cur_region->size & 0xFFF) { + if (i != PCI_ROM_SLOT){ fprintf(stderr, "PCI region %d at address 0x%llx "Quoted 1 line
Signed-off-by Alexander Graf <[email protected]>Good idea.
Quoted 8 lines
Quoted 5 lines
One thing I am concerned with, is that some users might see performance degradation and not see the error message. Maybe we can have a flag to optionally fail unless passthrough can be fast? I'm not really sure it's worth it to add such a flag though - what do others think?
It only kicks in on small BARs which are usually not _that_ performance critical. We're mostly talking about ack and status bits.
Well, ack and status are pretty important if accessed every interrupt.
Quoted 1 line
Also not failing > failing IMHO :). Even if it's not as fast as native.
Yes.
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]
Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.
That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
Cheers,
Muli
Signature · 5 lines
Muli Ben-Yehuda | [email protected] | +972-4-8281080 Manager, Virtualization and Systems Architecture Master Inventor, IBM Research -- Haifa Second Workshop on I/O Virtualization (WIOV '10): http://sysrun.haifa.il.ibm.com/hrl/wiov2010/
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
That's exactly what this patch does. It only allows access to the region defined in the BAR.
Alex
Quoted 2 lines · Alexander Graf
That's exactly what this patch does. It only allows access to the region defined in the BAR.
Sorry, missed it!
Cheers,
Muli
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
This patch seems to implement range checks correctly, let me know if I am missing something.
One also notes that we currently link qemu with libpci which I think requires admin cap to work. However, in the future we might extend this to also support getting device fds over a unix socket from a higher priviledged process.
If or when this is done, we will have to be
extra careful when passing
device file descriptor to an unpriveledged qemu process if the BARs are less than full page in size: mapping such BAR will allow qemu access outside this BAR.
A possible solution to this problem
if/when it arises would be adding yet another sysfs file for each resource, which would allow read/write but not mmap access, and perform range checks in the kernel.
Quoted 8 lines
Cheers,
MuliSignature · 5 lines
Muli Ben-Yehuda | [email protected] | +972-4-8281080 Manager, Virtualization and Systems Architecture Master Inventor, IBM Research -- Haifa Second Workshop on I/O Virtualization (WIOV '10): http://sysrun.haifa.il.ibm.com/hrl/wiov2010/
Quoted 46 lines · Michael S. Tsirkin
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
This patch seems to implement range checks correctly, let me know if I am missing something.
One also notes that we currently link qemu with libpci which I think requires admin cap to work. However, in the future we might extend this to also support getting device fds over a unix socket from a higher priviledged process.
If or when this is done, we will have to be
extra careful when passing
device file descriptor to an unpriveledged qemu process if the BARs are less than full page in size: mapping such BAR will allow qemu access outside this BAR.
Quoted 4 lines
A possible solution to this problem
if/when it arises would be adding yet another sysfs file for each resource, which would allow read/write but not mmap access, and perform range checks in the kernel.
Sounds like the best solution to this problem, yeah. Though we'd only need those for non-page-boundary BARs. So I guess the best would be to always export them from the kernel, but only use them when BAR & (PAGE_SIZE-1).
Either way, FWIW the device assignment stuff needs to be completely rewritten for qemu upstream anyways. So while it's good to collect ideas for now, let's not too put too much effort code-wise into the current code (unless it doesn't work).
Alex
Quoted 49 lines · Alexander Graf
Quoted 46 lines · Michael S. Tsirkin
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
This patch seems to implement range checks correctly, let me know if I am missing something.
One also notes that we currently link qemu with libpci which I think requires admin cap to work. However, in the future we might extend this to also support getting device fds over a unix socket from a higher priviledged process.
If or when this is done, we will have to be
extra careful when passing
device file descriptor to an unpriveledged qemu process if the BARs are less than full page in size: mapping such BAR will allow qemu access outside this BAR.
Quoted 6 lines
Quoted 4 lines
A possible solution to this problem
if/when it arises would be adding yet another sysfs file for each resource, which would allow read/write but not mmap access, and perform range checks in the kernel.Sounds like the best solution to this problem, yeah. Though we'd only need those for non-page-boundary BARs. So I guess the best would be to always export them from the kernel, but only use them when BAR & (PAGE_SIZE-1).
Hm, or add read/write fd functions that always do boundary checks to the existing interface and only allow mmap on size & PAGE_SIZE. Or only allow non-aligned mmap when the admin cap is present.
Alex
Quoted 57 lines · Alexander Graf
Quoted 54 lines · Alexander Graf
Quoted 50 lines · Michael S. Tsirkin
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
This patch seems to implement range checks correctly, let me know if I am missing something.
One also notes that we currently link qemu with libpci which I think requires admin cap to work. However, in the future we might extend this to also support getting device fds over a unix socket from a higher priviledged process.
If or when this is done, we will have to be
extra careful when passing
device file descriptor to an unpriveledged qemu process if the BARs are less than full page in size: mapping such BAR will allow qemu access outside this BAR.A possible solution to this problem
if/when it arises would be adding yet another sysfs file for each resource, which would allow read/write but not mmap access, and perform range checks in the kernel.
Quoted 5 lines
Quoted 1 line
Sounds like the best solution to this problem, yeah. Though we'd only need those for non-page-boundary BARs. So I guess the best would be to always export them from the kernel, but only use them when BAR & (PAGE_SIZE-1).
Hm, or add read/write fd functions that always do boundary checks to the existing interface and only allow mmap on size &
PAGE_SIZE. Or only allow non-aligned mmap when the admin cap is present.Alex
This might break existing applications.
We don't want that.
Quoted 58 lines · Michael S. Tsirkin
Quoted 57 lines · Alexander Graf
Quoted 54 lines · Alexander Graf
Quoted 50 lines · Michael S. Tsirkin
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
This patch seems to implement range checks correctly, let me know if I am missing something.
One also notes that we currently link qemu with libpci which I think requires admin cap to work. However, in the future we might extend this to also support getting device fds over a unix socket from a higher priviledged process.
If or when this is done, we will have to be
extra careful when passing
device file descriptor to an unpriveledged qemu process if the BARs are less than full page in size: mapping such BAR will allow qemu access outside this BAR.A possible solution to this problem
if/when it arises would be adding yet another sysfs file for each resource, which would allow read/write but not mmap access, and perform range checks in the kernel.
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
Sounds like the best solution to this problem, yeah. Though we'd only need those for non-page-boundary BARs. So I guess the best would be to always export them from the kernel, but only use them when BAR & (PAGE_SIZE-1).
Hm, or add read/write fd functions that always do boundary checks to the existing interface and only allow mmap on size &
PAGE_SIZE. Or only allow non-aligned mmap when the admin cap is present.Alex
This might break existing applications.
We don't want that.
Well currently you can't mmap the resource at all without at least r/w rights on the file, right? But yeah, we'd probably change behavior that could break someone - sigh.
Alex
Quoted 63 lines · Alexander Graf
Quoted 60 lines · Michael S. Tsirkin
Quoted 59 lines · Alexander Graf
Quoted 55 lines · Alexander Graf
Quoted 46 lines · Michael S. Tsirkin
Quoted 26 lines · Muli Ben-Yehuda
Quoted 7 lines · Alexander Graf
While trying to get device passthrough working with an emulex hba, kvm refused to pass it through because it has a BAR of 256 bytes:
Region 0: Memory at d2100000 (64-bit, non-prefetchable) [size=4K] Region 2: Memory at d2101000 (64-bit, non-prefetchable) [size=256] Region 4: I/O ports at b100 [size=256]Quoted 6 lines
Since the page boundary is an arbitrary optimization to allow 1:1 mapping of physical to virtual addresses, we can still take the old MMIO callback route.
So let's add a second code path that allows for size &
0xFFF!= 0 sized regions by looping it through userspace.That makes sense in general *but* the 4K-aligned check isn't just an optimization, it also has a security implication. Consider the theoretical case where has a multi-function device has BARs for two functions on the same page (within a 4K boundary), and each function is assigned to a different guest. With your current patch both guests will be able to write to each other's BARs. Another case is where a device has a bug and you must not write beyond the BAR or Bad Things Happen. With this patch an *unprivileged* guest could exploit that bug and make bad things happen.
Quoted 3 lines
This can be fixed if the slow userspace mmio path checks that all MMIO accesses by a guest fall within the portion of the page that is assigned to it.
This patch seems to implement range checks correctly, let me know if I am missing something.
One also notes that we currently link qemu with libpci which I think requires admin cap to work. However, in the future we might extend this to also support getting device fds over a unix socket from a higher priviledged process.
If or when this is done, we will have to be
extra careful when passing
device file descriptor to an unpriveledged qemu process if the BARs are less than full page in size: mapping such BAR will allow qemu access outside this BAR.Quoted 4 lines
A possible solution to this problem
if/when it arises would be adding yet another sysfs file for each resource, which would allow read/write but not mmap access, and perform range checks in the kernel.Sounds like the best solution to this problem, yeah. Though we'd only need those for non-page-boundary BARs. So I guess the best would be to always export them from the kernel, but only use them when BAR & (PAGE_SIZE-1).
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Hm, or add read/write fd functions that always do boundary checks to the existing interface and only allow mmap on size &
PAGE_SIZE. Or only allow non-aligned mmap when the admin cap is present.Alex
This might break existing applications.
We don't want that.Well currently you can't mmap the resource at all without at least r/w rights on the file, right?
You could have dropped the cap or got the fd from another process.
Quoted 3 lines
But yeah, we'd probably change behavior that could break someone - sigh.
Alex
Quoted 4 lines · Alexander Graf
Either way, FWIW the device assignment stuff needs to be completely rewritten for qemu upstream anyways. So while it's good to collect ideas for now, let's not too put too much effort code-wise into the current code (unless it doesn't work).
What do you have in mind for such a rewrite?
Cheers,
Muli
Quoted 8 lines · Muli Ben-Yehuda
Quoted 4 lines · Alexander Graf
Either way, FWIW the device assignment stuff needs to be completely rewritten for qemu upstream anyways. So while it's good to collect ideas for now, let's not too put too much effort code-wise into the current code (unless it doesn't work).
What do you have in mind for such a rewrite?
I'd like to see it more well-abstracted and versatile. I don't see an obvious reason why we shouldn't be able to use a physical device in a TCG target :-).
Alex
Quoted 5 lines · Alexander Graf
Quoted 1 line
What do you have in mind for such a rewrite?
I'd like to see it more well-abstracted and versatile. I don't see an obvious reason why we shouldn't be able to use a physical device in a TCG target :-).
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation, and interrupts you probably get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)
Cheers,
Muli
Quoted 10 lines · Muli Ben-Yehuda
Quoted 5 lines · Alexander Graf
Quoted 1 line
What do you have in mind for such a rewrite?
I'd like to see it more well-abstracted and versatile. I don't see an obvious reason why we shouldn't be able to use a physical device in a TCG target :-).
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
Quoted 6 lines
and interrupts you probably
get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)Cheers,
Muli
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
For example, I think a read memory barrier is currently ignored in translation, and host CPU will reorder reads. Some drivers might also rely on ordering guarantees that depend on CPU cacheline sizes. Atomics is another bag of tricks but I expect atomics on a DMA memory are not widely used.
I am not sure this problem is solvable unless host and guest architectures are very similar.
Signature · 1 line
MST
Quoted 21 lines · Michael S. Tsirkin
Quoted 10 lines · Muli Ben-Yehuda
Quoted 5 lines · Alexander Graf
Quoted 1 line
What do you have in mind for such a rewrite?
I'd like to see it more well-abstracted and versatile. I don't see an obvious reason why we shouldn't be able to use a physical device in a TCG target :-).
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
Quoted 6 lines
and interrupts you probably
get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)Cheers,
Muli
Quoted 8 lines
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
For example, I think a read memory barrier is currently ignored in translation, and host CPU will reorder reads. Some drivers might also rely on ordering guarantees that depend on CPU cacheline sizes. Atomics is another bag of tricks but I expect atomics on a DMA memory are not widely used.
Since we'd use the mmio callbacks for MMIO we'd be strictly ordered, no?
Alex
Quoted 33 lines · Alexander Graf
Quoted 21 lines · Michael S. Tsirkin
Quoted 10 lines · Muli Ben-Yehuda
Quoted 5 lines · Alexander Graf
Quoted 1 line
What do you have in mind for such a rewrite?
I'd like to see it more well-abstracted and versatile. I don't see an obvious reason why we shouldn't be able to use a physical device in a TCG target :-).
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
Quoted 6 lines
and interrupts you probably
get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)Cheers,
MuliQuoted 8 lines
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
For example, I think a read memory barrier is currently ignored in translation, and host CPU will reorder reads. Some drivers might also rely on ordering guarantees that depend on CPU cacheline sizes. Atomics is another bag of tricks but I expect atomics on a DMA memory are not widely used.
Quoted 3 lines
Since we'd use the mmio callbacks for MMIO we'd be strictly ordered, no?
Alex
Not unless you issue appropriate host memory barriers on mmio callbacks (kvm currently uses a lock for this, which has an implicit barrier, but I do not
think TCG does this).
But even then, it depends on the device, for some devices DMA memory reads/writes might depend on each other. Look at virtio as an example, a real device might have the same semantics. As a simpler example, some devices DMA the following into ring in host memory to signal data available:
- valid tag
- data length
host will read tag, and when it's valid use data length,
but e.g. on intel this only works well if these share a cache line, otherwise data read might bypass tag read and you get invalid data.
Signature · 1 line
MST
Quoted 42 lines · Michael S. Tsirkin
Quoted 33 lines · Alexander Graf
Quoted 21 lines · Michael S. Tsirkin
Quoted 10 lines · Muli Ben-Yehuda
Quoted 5 lines · Alexander Graf
Quoted 1 line
What do you have in mind for such a rewrite?
I'd like to see it more well-abstracted and versatile. I don't see an obvious reason why we shouldn't be able to use a physical device in a TCG target :-).
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
Quoted 6 lines
and interrupts you probably
get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)Cheers,
MuliQuoted 8 lines
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
For example, I think a read memory barrier is currently ignored in translation, and host CPU will reorder reads. Some drivers might also rely on ordering guarantees that depend on CPU cacheline sizes. Atomics is another bag of tricks but I expect atomics on a DMA memory are not widely used.
Quoted 3 lines
Since we'd use the mmio callbacks for MMIO we'd be strictly ordered, no?
Alex
Not unless you issue appropriate host memory barriers on mmio callbacks (kvm currently uses a lock for this, which has an implicit barrier, but I do not
think TCG does this).
Quoted 8 lines
But even then, it depends on the device, for some devices DMA memory reads/writes might depend on each other. Look at virtio as an example, a real device might have the same semantics. As a simpler example, some devices DMA the following into ring in host memory to signal data available:
- valid tag
- data length
host will read tag, and when it's valid use data length,
May be:
- data length
- valid tag
Otherwise how can you guaranty that at the time tag is valid data length is already up-to-date and not in process to be written? An on arch such as Altix DMA from device to memory can arrive out of order if memory is not mapped in a special way, but then DMA is slow.
--
Gleb.
Quoted 4 lines · Michael S. Tsirkin
Quoted 2 lines
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
With or without an IOMMU?
Quoted 5 lines
Quoted 2 lines
and interrupts you probably get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
Yep, I know nothing about TCG, only looking at this from the device interaction side.
Quoted 2 lines
I am not sure this problem is solvable unless host and guest architectures are very similar.
Now you are ignoring the most interesting issue, namely, why would you want to solve it? What is the value of device assignment for TCG targets?
Cheers,
Muli
Signature · 5 lines
Muli Ben-Yehuda | [email protected] | +972-4-8281080 Manager, Virtualization and Systems Architecture Master Inventor, IBM Research -- Haifa Second Workshop on I/O Virtualization (WIOV '10): http://sysrun.haifa.il.ibm.com/hrl/wiov2010/
Quoted 18 lines · Muli Ben-Yehuda
Quoted 4 lines · Michael S. Tsirkin
Quoted 2 lines
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
With or without an IOMMU?
Quoted 5 lines
Quoted 2 lines
and interrupts you probably get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
Yep, I know nothing about TCG, only looking at this from the device interaction side.
Quoted 6 lines
Quoted 2 lines
I am not sure this problem is solvable unless host and guest architectures are very similar.
Now you are ignoring the most interesting issue, namely, why would you want to solve it? What is the value of device assignment for TCG targets?
Why would you want to emulate a machine at all? ;-)
Imagine you were a hardware manufacturer who develops some self-made hardware. Now that manufacturer of course has x86 boxes. Chances are pretty low he has PPC ones and imagine we'd ever get MMIO/PCI on S390, he definitely does not have those.
Still, while developing his driver it'd be really valuable to know that it works on other architectures as well. Especially since he can claim support for architectures he doesn't have himself. At least until the first customer arrives :-).
While this is a scenario I just made up and don't really have myself, its goal is to illustrate why we shouldn't close doors that don't have to be closed. If TCG doesn't deal with it properly, that doesn't mean we shouldn't work on making the other end compatible.
Alex
Quoted 28 lines · Alexander Graf
Quoted 18 lines · Muli Ben-Yehuda
Quoted 4 lines · Michael S. Tsirkin
Quoted 2 lines
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
With or without an IOMMU?
Quoted 5 lines
Quoted 2 lines
and interrupts you probably get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
Yep, I know nothing about TCG, only looking at this from the device interaction side.
Quoted 6 lines
Quoted 2 lines
I am not sure this problem is solvable unless host and guest architectures are very similar.
Now you are ignoring the most interesting issue, namely, why would you want to solve it? What is the value of device assignment for TCG targets?
Quoted 8 lines
Why would you want to emulate a machine at all? ;-)
Imagine you were a hardware manufacturer who develops some self-made hardware. Now that manufacturer of course has x86 boxes. Chances are pretty low he has PPC ones and imagine we'd ever get MMIO/PCI on S390, he definitely does not have those.
Still, while developing his driver it'd be really valuable to know that it works on other architectures as well. Especially since he can claim support for architectures he doesn't have himself. At least until the first customer arrives :-).
While this is a scenario I just made up and don't really have myself, its goal is to illustrate why we shouldn't close doors that don't have to be closed. If TCG doesn't deal with it properly, that doesn't mean we shouldn't work on making the other end compatible.
Well, it does IMO mean that such a project should not be a blocker for passthrough in upstream qemu, after fixing endian-ness issues and generally cleaning up the code.
Quoted 1 line
Alex
Quoted 8 lines · Muli Ben-Yehuda
Quoted 4 lines · Michael S. Tsirkin
Quoted 2 lines
mmio and pio are easy, DMA you'd need an IOMMU for security, or whatever uio does just for translation,
uio currently does not support DMA, but I plan to fix this
With or without an IOMMU?
With an IOMMU.
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
and interrupts you probably get for free from uio. Seems eminently doable to me. Why you'd want to is another matter :-)
The list above ignores the biggest issue: you would have to change TCG code generation to make this work.
Yep, I know nothing about TCG, only looking at this from the device interaction side.
Quoted 6 lines
Quoted 2 lines
I am not sure this problem is solvable unless host and guest architectures are very similar.
Now you are ignoring the most interesting issue, namely, why would you want to solve it? What is the value of device assignment for TCG targets?
No idea.
Quoted 8 lines
Cheers,
MuliSignature · 5 lines
Muli Ben-Yehuda | [email protected] | +972-4-8281080 Manager, Virtualization and Systems Architecture Master Inventor, IBM Research -- Haifa Second Workshop on I/O Virtualization (WIOV '10): http://sysrun.haifa.il.ibm.com/hrl/wiov2010/