mark nic as trusted
PATCH 1/1 mark nic as trusted
This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.
hw/e1000.c +3 -0
@@ -1045,6 +1045,9 @@ pci_e1000_init(PCIBus *bus, NICInfo *nd, int devfn) pci_conf[0x3d ] =1 ;// interrupt pin 0 +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + d->mmio_index =cpu_register_io_memory (0 , e1000_mmio_read, e1000_mmio_write, d);
hw/eepro100.c +3 -0
@@ -1754,6 +1754,9 @@ static void nic_init(PCIBus * bus, NICInfo * nd,pci_reset (s); +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); +/* Add 64 * 2 EEPROM. i82557 and i82558 support a 64 word EEPROM, * i82559 and later support 64 or 256 word EEPROM. */ s->eeprom =eeprom93xx_new (EEPROM_SIZE );
hw/ne2000.c +4 -0
@@ -806,6 +806,10 @@ void pci_ne2000_init(PCIBus *bus, NICInfo *nd, int devfn)pci_register_io_region (&d->dev,0 ,0x100 ,PCI_ADDRESS_SPACE_IO , ne2000_map); + +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + s = &d->ne2000; s->irq = d->dev.irq[0 ]; s->pci_dev = (PCIDevice *)d;
hw/pci.c +17 -0
@@ -536,6 +536,23 @@ static void pci_set_irq(void *opaque, int irq_num, int level) bus->set_irq (bus->irq_opaque, irq_num, bus->irq_count[irq_num] !=0 ); } +int pci_add_capability (PCIDevice *d,uint8_t cap_id,uint8_t off,uint8_t *buf, +int len) +{ +uint16_t status =le16_to_cpu (*(uint16_t *)(d->config +0x06 )); + +if (off + len +2 >0x100 || off <0x40 ) +return -1 ; + + d->config[0x06 ] =cpu_to_le16 (status |0x10 ); + d->config[off] = cap_id; + d->config[off +1 ] = d->config[0x34 ]; + d->config[0x34 ] = off; +memcpy (d->config + off +2 , buf, len); + +return 0 ; +} +/***********************************************************/ /* monitor info on PCI */
hw/pci.h +2 -0
@@ -117,6 +117,8 @@ typedef void (*pci_set_irq_fn)(qemu_irq *pic, int irq_num, int level);typedef int (*pci_map_irq_fn )(PCIDevice *pci_dev,int irq_num);PCIBus *pci_register_bus (pci_set_irq_fn set_irq,pci_map_irq_fn map_irq,qemu_irq *pic,int devfn_min,int nirq); +int pci_add_capability (PCIDevice *d,uint8_t cap_id,uint8_t off,uint8_t *buf, +int len);void pci_nic_init (PCIBus *bus,NICInfo *nd,int devfn);void pci_data_write (void *opaque,uint32_t addr,uint32_t val,int len);
hw/pcnet.c +3 -0
@@ -2024,6 +2024,9 @@ void pci_pcnet_init(PCIBus *bus, NICInfo *nd, int devfn) pci_conf[0x3e ] =0x06 ; pci_conf[0x3f ] =0xff ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); +/* Handler for memory-mapped I/O */ d->mmio_index =cpu_register_io_memory (0 , pcnet_mmio_read, pcnet_mmio_write, d);
hw/rtl8139.c +3 -0
@@ -3423,6 +3423,9 @@ void pci_rtl8139_init(PCIBus *bus, NICInfo *nd, int devfn) pci_conf[0x3d ] =1 ;/* interrupt pin 0 */ pci_conf[0x34 ] =0xdc ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + s = &d->rtl8139;/* I/O handler for memory-mapped I/O */
hw/virtio-net.c +3 -0
@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)if (!n)return NULL ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&n->vdev.pci_dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + n->vdev.get_config = virtio_net_update_config; n->vdev.get_features = virtio_net_get_features; n->vdev.set_features = virtio_net_set_features;
net.c +5 -0
@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)if (get_param_value (buf,sizeof (buf),"model" , p)) { nd->model =strdup (buf); } +if (get_param_value (buf,sizeof (buf),"trusted" , p)) { +strncpy (nd->secure_cookie, buf,sizeof (nd->secure_cookie)); + }else { + nd->secure_cookie[0 ] =NULL ; + } nd->vlan = vlan; nb_nics++; vlan->nb_guest_devs++;
net.h +1 -0
@@ -50,6 +50,7 @@ struct NICInfo {uint8_t macaddr[6 ];const char *model;VLANState *vlan; +uint8_t secure_cookie[14 ]; };extern int nb_nics;
--
Gleb.
Hi Gleb,
Quoted 4 lines · Gleb Natapov
This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.
I'm not sure I follow - is this cookie a shared secret that only the host and guest knows, or do literally mean that the cookie will contain the string "Trusted" as a indicator that the guest can trust the NIC?
Quoted 1 line
Signed-off-by Gleb Natapov <[email protected]>
...
hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313
Quoted 10 lines
hw/virtio-net.c +2
-0@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)if (!n)return NULL ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&n->vdev.pci_dev,0x0f ,0xf0 , nd->secure_cookie,14 );
How was the Capability ID 0x0f chosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:
#define PCI_CAP_ID_AGP3 0x0E /* AGP Target PCI-PCI bridge */ #define PCI_CAP_ID_EXP 0x10 /* PCI Express */
Also, to reduce magic numbers it would be nice to define the CAP_ID (0xf) and offset (0xf0) as a macro somewhere and use sizeof(nd->secure_cookie) instead of 14.
net.c int net_client_init(const char *device, const char *p) · line 1480
Quoted 12 lines
net.c +4
-0@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)if (get_param_value (buf,sizeof (buf),"model" , p)) { nd->model =strdup (buf); } +if (get_param_value (buf,sizeof (buf),"trusted" , p)) { +strncpy (nd->secure_cookie, buf,sizeof (nd->secure_cookie)); + }else { + nd->secure_cookie[0 ] =NULL ;
NULL isn't a uint8_t, use '\0' instead I guess. Or maybe just memset() the NICInfo struct before starting to assign to it.
Cheers,
Mark.
Quoted 4 lines · Mark McLoughlin
Hi Gleb,
On Wed, 2009-01-07 at 16:26 +0200, Gleb Natapov wrote:
Quoted 9 lines
Quoted 4 lines
This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.
I'm not sure I follow - is this cookie a shared secret that only the host and guest knows, or do literally mean that the cookie will contain the string "Trusted" as a indicator that the guest can trust the NIC?
The presence of the capability indicates that the nic is trusted, but I added a possibility to pass 14 byte string from a host too. The string can be used as shared secret.
Quoted 10 lines
Quoted 1 line
Signed-off-by Gleb Natapov <[email protected]>...
Quoted 8 lines
hw/virtio-net.c +0
-0@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)if (!n)return NULL ;
Quoted 9 lines
hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313
Quoted 2 lines
hw/virtio-net.c +2
-0+if (nd->secure_cookie[0 ]) +pci_add_capability (&n->vdev.pci_dev,0x0f ,0xf0 , nd->secure_cookie,14 );How was the Capability ID
0x0fchosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:#define PCI_CAP_ID_AGP3 0x0E /* AGP Target PCI-PCI bridge */ #define PCI_CAP_ID_EXP 0x10 /* PCI Express */
It is "secure device capability", so I used it based on the name.
Quoted 3 lines
Also, to reduce magic numbers it would be nice to define the
CAP_ID(0xf) and offset (0xf0) as a macro somewhere and usesizeof(nd->secure_cookie)instead of 14.
OK. Good point.
Quoted 17 lines
net.c int net_client_init(const char *device, const char *p) · line 1480
Quoted 13 lines
net.c +4
-0@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)if (get_param_value (buf,sizeof (buf),"model" , p)) { nd->model =strdup (buf); } +if (get_param_value (buf,sizeof (buf),"trusted" , p)) { +strncpy (nd->secure_cookie, buf,sizeof (nd->secure_cookie)); + }else { + nd->secure_cookie[0 ] =NULL ;NULL isn't a
uint8_t, use '\0' instead I guess. Or maybe justmemset()the NICInfo struct before starting to assign to it.
OK. I wounder why I've used NULL here in the first place.
--
Gleb.
Quoted 10 lines · Gleb Natapov
Quoted 9 lines
hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313
Quoted 2 lines
hw/virtio-net.c +2
-0+if (nd->secure_cookie[0 ]) +pci_add_capability (&n->vdev.pci_dev,0x0f ,0xf0 , nd->secure_cookie,14 );How was the Capability ID
0x0fchosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:#define PCI_CAP_ID_AGP3 0x0E /* AGP Target PCI-PCI bridge */ #define PCI_CAP_ID_EXP 0x10 /* PCI Express */It is "secure device capability", so I used it based on the name.
Ah, I see this in pciutils now:
#define PCI_CAP_ID_SECURE 0x0F /* Secure device (?) */
Where is the capability format published? I can't seem to find any specification for it ...
Cheers,
Mark.
Quoted 11 lines · Mark McLoughlin
Quoted 9 lines · Gleb Natapov
Quoted 9 lines
hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313
Quoted 2 lines
hw/virtio-net.c +2
-0+if (nd->secure_cookie[0 ]) +pci_add_capability (&n->vdev.pci_dev,0x0f ,0xf0 , nd->secure_cookie,14 );How was the Capability ID
0x0fchosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:#define PCI_CAP_ID_AGP3 0x0E /* AGP Target PCI-PCI bridge */ #define PCI_CAP_ID_EXP 0x10 /* PCI Express */
Quoted 9 lines
Quoted 1 line
It is "secure device capability", so I used it based on the name.
Ah, I see this in pciutils now:
#define PCI_CAP_ID_SECURE 0x0F /* Secure device (?) */Where is the capability format published? I can't seem to find any specification for it ...
Me too. The only thing I found is that AMD barcelona chipset uses it. Look at page 260 of this doc for details how:
http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/31116_PUB_BKDG-3-18_11-13-08.pdf
I just hijacked the capability for our needs :)
--
Gleb.
Quoted 7 lines · Gleb Natapov
This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.
Signed-off-by Gleb Natapov <[email protected]>
What utility does this have? Does this make Windows happy in some special way?
Regards,
Anthony Liguori
Quoted 159 lines
hw/e1000.c +3
-0@@ -1045,6 +1045,9 @@ pci_e1000_init(PCIBus *bus, NICInfo *nd, int devfn) pci_conf[0x3d ] =1 ;// interrupt pin 0 +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + d->mmio_index =cpu_register_io_memory (0 , e1000_mmio_read, e1000_mmio_write, d);hw/eepro100.c +3
-0@@ -1754,6 +1754,9 @@ static void nic_init(PCIBus * bus, NICInfo * nd,pci_reset (s); +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); +/* Add 64 * 2 EEPROM. i82557 and i82558 support a 64 word EEPROM, * i82559 and later support 64 or 256 word EEPROM. */ s->eeprom =eeprom93xx_new (EEPROM_SIZE );hw/ne2000.c +4
-0@@ -806,6 +806,10 @@ void pci_ne2000_init(PCIBus *bus, NICInfo *nd, int devfn)pci_register_io_region (&d->dev,0 ,0x100 ,PCI_ADDRESS_SPACE_IO , ne2000_map); + +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + s = &d->ne2000; s->irq = d->dev.irq[0 ]; s->pci_dev = (PCIDevice *)d;hw/pci.c +17
-0@@ -536,6 +536,23 @@ static void pci_set_irq(void *opaque, int irq_num, int level) bus->set_irq (bus->irq_opaque, irq_num, bus->irq_count[irq_num] !=0 ); } +int pci_add_capability (PCIDevice *d,uint8_t cap_id,uint8_t off,uint8_t *buf, +int len) +{ +uint16_t status =le16_to_cpu (*(uint16_t *)(d->config +0x06 )); + +if (off + len +2 >0x100 || off <0x40 ) +return -1 ; + + d->config[0x06 ] =cpu_to_le16 (status |0x10 ); + d->config[off] = cap_id; + d->config[off +1 ] = d->config[0x34 ]; + d->config[0x34 ] = off; +memcpy (d->config + off +2 , buf, len); + +return 0 ; +} +/***********************************************************/ /* monitor info on PCI */ hw/pci.h +2
-0@@ -117,6 +117,8 @@ typedef void (*pci_set_irq_fn)(qemu_irq *pic, int irq_num, int level);typedef int (*pci_map_irq_fn )(PCIDevice *pci_dev,int irq_num);PCIBus *pci_register_bus (pci_set_irq_fn set_irq,pci_map_irq_fn map_irq,qemu_irq *pic,int devfn_min,int nirq); +int pci_add_capability (PCIDevice *d,uint8_t cap_id,uint8_t off,uint8_t *buf, +int len);void pci_nic_init (PCIBus *bus,NICInfo *nd,int devfn);void pci_data_write (void *opaque,uint32_t addr,uint32_t val,int len);hw/pcnet.c +3
-0@@ -2024,6 +2024,9 @@ void pci_pcnet_init(PCIBus *bus, NICInfo *nd, int devfn) pci_conf[0x3e ] =0x06 ; pci_conf[0x3f ] =0xff ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); +/* Handler for memory-mapped I/O */ d->mmio_index =cpu_register_io_memory (0 , pcnet_mmio_read, pcnet_mmio_write, d);hw/rtl8139.c +3
-0@@ -3423,6 +3423,9 @@ void pci_rtl8139_init(PCIBus *bus, NICInfo *nd, int devfn) pci_conf[0x3d ] =1 ;/* interrupt pin 0 */ pci_conf[0x34 ] =0xdc ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&d->dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + s = &d->rtl8139;/* I/O handler for memory-mapped I/O */ hw/virtio-net.c +3
-0@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)if (!n)return NULL ; +if (nd->secure_cookie[0 ]) +pci_add_capability (&n->vdev.pci_dev,0x0f ,0xf0 , nd->secure_cookie,14 ); + n->vdev.get_config = virtio_net_update_config; n->vdev.get_features = virtio_net_get_features; n->vdev.set_features = virtio_net_set_features;net.c +5
-0@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)if (get_param_value (buf,sizeof (buf),"model" , p)) { nd->model =strdup (buf); } +if (get_param_value (buf,sizeof (buf),"trusted" , p)) { +strncpy (nd->secure_cookie, buf,sizeof (nd->secure_cookie)); + }else { + nd->secure_cookie[0 ] =NULL ; + } nd->vlan = vlan; nb_nics++; vlan->nb_guest_devs++;net.h +1
-1@@ -50,6 +50,7 @@ struct NICInfo {uint8_t macaddr[6 ];const char *model;VLANState *vlan; +uint8_t secure_cookie[14 ]; };extern int nb_nics; --Gleb.
Quoted 6 lines · Anthony Liguori
Quoted 5 lines · Gleb Natapov
This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.
Quoted 6 lines
Quoted 2 lines
Signed-off-by Gleb Natapov <[email protected]>What utility does this have? Does this make Windows happy in some special way?
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.
--
Gleb.
Quoted 12 lines · Gleb Natapov
Quoted 10 lines · Anthony Liguori
Quoted 8 lines · Gleb Natapov
This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.
Signed-off-by Gleb Natapov <[email protected]>
Quoted 8 lines
Quoted 3 lines
What utility does this have? Does this make Windows happy in some special way?
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.
So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Regards,
Anthony Liguori
Quoted 5 lines
--
Gleb.
Quoted 6 lines · Anthony Liguori
Quoted 4 lines
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.
So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Like if you were doing PCI passthrough of an e1000...
Regards,
Anthony Liguori
Quoted 10 lines
Regards,
Anthony Liguori
Quoted 5 lines
Signature · 1 line
Gleb.
Quoted 7 lines · Anthony Liguori
Quoted 6 lines · Anthony Liguori
Quoted 5 lines
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real
host and "own" a guest.
Quoted 5 lines
Quoted 2 lines
So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Like if you were doing PCI passthrough of an e1000...
It's not just one bit. It is 14 byte string. We can put something unique there.
--
Gleb.
Quoted 9 lines · Gleb Natapov
Quoted 7 lines · Anthony Liguori
Quoted 5 lines · Anthony Liguori
Quoted 5 lines
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Like if you were doing PCI passthrough of an e1000...
It's not just one bit. It is 14 byte string. We can put something unique there.
This is for vmchannel? Why not add a feature to virtio-net?
Regards,
Anthony Liguori
Quoted 13 lines · Anthony Liguori
Quoted 12 lines · Gleb Natapov
Quoted 10 lines · Anthony Liguori
Quoted 8 lines · Anthony Liguori
Quoted 5 lines
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real
host and "own" a guest.So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Like if you were doing PCI passthrough of an e1000...
It's not just one bit. It is 14 byte string. We can put something unique there.
This is for vmchannel? Why not add a feature to virtio-net?
Yes. This is for vmchannel. Or any other management solution that work over network. It has to know what network it can trust. The alternative is much more complex (security certificates, etc). Why do it virtio-net specific? What's wrong with more general solution?
--
Gleb.
Quoted 25 lines · Gleb Natapov
Quoted 23 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 8 lines · Anthony Liguori
Quoted 5 lines · Anthony Liguori
Quoted 5 lines
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.
Quoted 7 lines
Quoted 3 lines
So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Like if you were doing PCI passthrough of an e1000...
It's not just one bit. It is 14 byte string. We can put something unique there.
Quoted 8 lines
Quoted 3 lines
This is for vmchannel? Why not add a feature to virtio-net?
Yes. This is for vmchannel. Or any other management solution that work over network. It has to know what network it can trust. The alternative is much more complex (security certificates, etc). Why do it virtio-net specific? What's wrong with more general solution?
Does Windows provide an API for determining "trustedness"? How is this exposed to userspace in Linux?
The thinking behind virtio-net is that you may want to expose a different userspace interface in Windows than networking (maybe something more direct) since it may be impossible to get around the Windows firewalling nonsense. With virtio-net, you could have the Windows driver check the special "vmchannel" flag and present a different userspace interface than the traditional network driver.
Although that's just a thought.
Regards,
Anthony Liguori
Quoted 5 lines
--
Gleb.
Quoted 25 lines · Anthony Liguori
Quoted 24 lines · Gleb Natapov
Quoted 22 lines · Anthony Liguori
Quoted 9 lines · Gleb Natapov
Quoted 7 lines · Anthony Liguori
Quoted 5 lines · Anthony Liguori
Quoted 5 lines
That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real
host and "own" a guest.Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?
Like if you were doing PCI passthrough of an e1000...
It's not just one bit. It is 14 byte string. We can put something unique there.
This is for vmchannel? Why not add a feature to virtio-net?
Quoted 9 lines
Quoted 5 lines
Yes. This is for vmchannel. Or any other management solution that work over network. It has to know what network it can trust. The alternative is much more complex (security certificates, etc). Why do it virtio-net specific? What's wrong with more general solution?
Does Windows provide an API for determining "trustedness"? How is this exposed to userspace in Linux?
Windows and Linux allows access to PCI space, so it is easy to check "trustedness". AFAIK we already have Windows code to do that. In Linux it can be done even by shell script by examining files in /sys.
Quoted 7 lines
The thinking behind virtio-net is that you may want to expose a different userspace interface in Windows than networking (maybe something more direct) since it may be impossible to get around the Windows firewalling nonsense. With virtio-net, you could have the Windows driver check the special "vmchannel" flag and present a different userspace interface than the traditional network driver.
Then we will have to implement some kind of reliable protocol on top of that. It can be done, but it make sense to reuse TCP for that. We will have to request proper firewall config from VM admin though. It is possible to dial with build in Windows firewall automatically, but third party firewalls may require admin intervention.
--
Gleb.
Quoted 6 lines · Gleb Natapov
Quoted 1 line · Anthony Liguori
Windows and Linux allows access to PCI space, so it is easy to check "trustedness". AFAIK we already have Windows code to do that. In Linux it can be done even by shell script by examining files in /sys.
Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?
Regards,
Anthony Liguori
Quoted 3 lines · Anthony Liguori
Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?
That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.
Is there some way to mark a PCI device so it will be ignored at boot time generically? Changing the PCI ID will do that for all guests, but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?
Suppose you start a guest with two "trusted" nics, because you want to run two unrelated vmchannel-using admin apps. How does each app know which nic to use - or do they share it?
As the guest OS's TCP is being used, what do you do about IP address space conflicts?
I.e. if NIC #1 is the guest's LAN, and NIC #2 is the vmchannel, how is the vmchannel NIC going to be configured in a way that's guaranteed to avoid breaking the LAN networking, which could be assigned any legal subnet (especially when bridging is used), and on some networks changes from time to time?
Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?
-- Jamie
Quoted 10 lines · Jamie Lokier
Quoted 2 lines · Anthony Liguori
Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?
That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.
We can make fedora, rhel and libvirt support it. It might be a bit painful but since a network device was chosen for this propose then that's the right way to go. Others can either use 3rd agents to cancel firewalls like we plan to do for windows, or as was suggested, change the pci device id and load a bit different driver.
You suggestion is good also since it will be good for personal usages where the guest can easily reach the host network and the user can easily cancel firewalls.
Quoted 5 lines
Is there some way to mark a PCI device so it will be ignored at boot time generically? Changing the PCI ID will do that for all guests, but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?
Alternatively one can hot plug the vmchanneled nic, right after boot. IMHO I rather stick with guest mgmt agent take care of the accesses to the nic.
Quoted 5 lines
Suppose you start a guest with two "trusted" nics, because you want to run two unrelated vmchannel-using admin apps. How does each app know which nic to use - or do they share it?
Each vmchannel is bond on the host to a separate pair of qemu_chr_device and a matching ip:port listening. So if there are n agents in the guest, each should connect to his ip:port without being aware to the others.
Quoted 13 lines
As the guest OS's TCP is being used, what do you do about IP address space conflicts?
I.e. if NIC #1 is the guest's LAN, and NIC #2 is the vmchannel, how is the vmchannel NIC going to be configured in a way that's guaranteed to avoid breaking the LAN networking, which could be assigned any legal subnet (especially when bridging is used), and on some networks changes from time to time?
Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?
We plan to pick link local subnets for ipv4.
It solved all the above questions. It should be connected to slirp without passing through the host stack/bridges (although it can be open too).
Quoted 4 lines
-- Jamie
w.r.t the option of using virtio nic, there is advantage of using any other nic since this way there is no requirement to install virtio driver on windows or on other older Linux/other OSs.
Quoted 13 lines · Dor Laor
Quoted 7 lines · Jamie Lokier
Quoted 4 lines · Anthony Liguori
Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?
Quoted 5 lines
That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.
Quoted 4 lines
We can make fedora, rhel and libvirt support it. It might be a bit painful but since
a network device was chosen for this propose then that's the right way to go.
For new Fedora / RHEL yes, but a large number of people using virt are doing so with old OS versions, and the chances of anyone retro-fitting all old distros is near zero. You might get it done if they were back porting the VirtIO NIC devices to old distros, but almost certainly not for arbitrary NIC devices like rtl8139/e1000/etc because its a huge QA testing headache to avoid regressions.
Daniel
Signature · 4 lines
|: Red Hat, Engineering, London -o- http://people.redhat.com/berrange/ :| |: http://libvirt.org -o- http://virt-manager.org -o- http://ovirt.org :| |: http://autobuild.org -o- http://search.cpan.org/~danberr/ :| |: GnuPG: 7D3B9505 -o- F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
Quoted 19 lines · Daniel P. Berrange
Quoted 13 lines · Dor Laor
Quoted 7 lines · Jamie Lokier
Quoted 4 lines · Anthony Liguori
Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?
Quoted 5 lines
That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.
Quoted 4 lines
We can make fedora, rhel and libvirt support it. It might be a bit painful but since
a network device was chosen for this propose then that's the right way to go.
Quoted 6 lines
For new Fedora / RHEL yes, but a large number of people using virt are doing so with old OS versions, and the chances of anyone retro-fitting all old distros is near zero. You might get it done if they were back porting the VirtIO NIC devices to old distros, but almost certainly not for arbitrary NIC devices like rtl8139/e1000/etc because its a huge QA testing headache to avoid regressions.
In some circles, a major purpose of virtualisation is to run old OS versions, either copied from machines where the hardware is aging to keep a working system still working, or for compatibility testing.
Changing a working guest setup isn't cool. Adding a small monitoring/reporting app (e.g. that shows the machine's load average, process table, network connections or whatever over a vmchannel) may be cool, provided it has negligable impact, even if changing its system configuration is not.
-- Jamie
Quoted 10 lines · Jamie Lokier
Quoted 7 lines · Daniel P. Berrange
For new Fedora / RHEL yes, but a large number of people using virt are doing so with old OS versions, and the chances of anyone retro-fitting all old distros is near zero. You might get it done if they were back porting the VirtIO NIC devices to old distros, but almost certainly not for arbitrary NIC devices like rtl8139/e1000/etc because its a huge QA testing headache to avoid regressions.
Quoted 4 lines
In some circles, a major purpose of virtualisation is to run old OS versions, either copied from machines where the hardware is aging to keep a working system still working, or for compatibility testing.
When dealing with older OSes, there really no limit in terms of what we can do. For instance, there's no reason that the backported version of the virtio-net driver cannot expose a chardev interface if the distro doesn't have the appropriate configuration.
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
You cannot get around having a kernel module to support something like vmchannel so that's always going to be a requirement. A completely userspace approach ala the VMware backdoor hackery is just too hideous.
Regards,
Anthony Liguori
Quoted 10 lines
Changing a working guest setup isn't cool. Adding a small monitoring/reporting app (e.g. that shows the machine's load average, process table, network connections or whatever over a vmchannel) may be cool, provided it has negligable impact, even if changing its system configuration is not.
-- Jamie
Quoted 3 lines · Anthony Liguori
You cannot get around having a kernel module to support something like vmchannel so that's always going to be a requirement. A completely userspace approach ala the VMware backdoor hackery is just too hideous.
Eh? I thought the point of vmchannel is to use the guest network stack, so guest processes - including user ones - can potentially speak over it to the host via a marked, designated channel.
If you need a kernel module, i.e. you're doing special kernely things, there's no need for the vmchannel device to be a normal NIC device, no need for it to interfere with the guest networking (e.g. conflicting IPv4 subnets), and it can use virtio, I agree in that case.
Now that it's clear I'm imagining guest userspace admin processes and you're imagining a kernel module on the guest, the rest of this exchange makes sense :-)
Quoted 8 lines
Quoted 3 lines
In some circles, a major purpose of virtualisation is to run old OS versions, either copied from machines where the hardware is aging to keep a working system still working, or for compatibility testing.
When dealing with older OSes, there really no limit in terms of what we can do. For instance, there's no reason that the backported version of the virtio-net driver cannot expose a chardev interface if the distro doesn't have the appropriate configuration.
Realistically, the set of (older) OSes where you'd like to run simple guest-host communicating apps for VM management, especially simple monitoring apps (load avg, memory use, number of processes), is much larger than those older OSes which virtio-net can be loaded onto.
E.g. Linux and Windows apps to do that could run on very old/obscure versions and diverse distributions, if statically linked.
Sometimes there'll be no backport of kernel drivers, and if there is one, that doesn't mean you can load it into an existing guest image or even compile it for that exact guest kernel.
I think it's quite reasonable to want to run simple monitoring apps on such guests.
Quoted 5 lines
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.
-- Jamie
Quoted 6 lines · Jamie Lokier
Quoted 5 lines
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
Quoted 6 lines
Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.
I think a serial port device should be universally supported by any OS and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.
About the overall idea of host marking something as trusted, here are some alternative ideas:
- guest could ask for a trusted channel and then the host would create one
- guest could ask the host to list the trusted devices
- on guest's request, host can cut an existing device from outside world and start using that as the trusted one
Quoted 16 lines · Blue Swirl
Quoted 6 lines · Jamie Lokier
Quoted 5 lines
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
Quoted 7 lines
Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.
Quoted 5 lines
I think a serial port device should be universally supported by any OS and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.
I remember the old days of DOS networking where the Kirschbaum-Netz software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.
Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.
Regards,
Carl-Daniel
Signature · 1 line
http://www.hailfinger.org/
Quoted 31 lines · Carl-Daniel Hailfinger
Quoted 17 lines · Blue Swirl
Quoted 6 lines · Jamie Lokier
Quoted 5 lines
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
Quoted 8 lines
Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.
Quoted 6 lines
I think a serial port device should be universally supported by any OS and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.
I remember the old days of DOS networking where the Kirschbaum-Netz software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.
Quoted 8 lines
Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.
Regards,
Carl-Daniel
Actually vmchannel started as a pv serial implementation. Standard serial is a bit low performing and demand an vmexit per byte (maybe it's not that bad for us). Moreover, various guest do not support more than 4 serial channels. Since there should be several channels and we like to preserve some for console/debug, it is not enough.
Originally, vmchannel was a virtio interface with netlink interface to the kernel. Then, Anthony asked to change it to a socket interface with new address family. It was indeed a logical step. Then, David Miller was reluctant to add such interface to the kernel. Instead, he offered the network device solution.
Are we close to begin this loop again? :)
Let's try to stick to the nic solution. It has some advantages over pv serial:
- Reliable communication if tcp is used
- Migration support for slirp
- No new driver in the guest.
- Might even work for older guests
The disadvantages are:
- Need to 'teach' guest daemons/firewalls not to handle/block the
new nic
- Link local addresses for ipv4 are problematic when using on other
nics in parallel
- We should either 1. not use link local on other links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.
So additional nic can do the job and we have several flavours to choose from.
Regards,
Dor
Quoted 44 lines · Dor Laor
Carl-Daniel Hailfinger wrote:
On 11.01.2009 08:10, Blue Swirl wrote:Quoted 5 lines · Jamie Lokier
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by
designing
something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.I think a serial port device should be universally supported by any OS
and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.I remember the old days of DOS networking where the Kirschbaum-Netz
software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.
Regards,
Carl-Daniel
Quoted 9 lines
Actually vmchannel started as a pv serial implementation. Standard serial
is
a bit low performing and demand an vmexit per byte (maybe it's not that bad
for us).
Moreover, various guest do not support more than 4 serial channels. Since
there
should be several channels and we like to preserve some for console/debug,
it is
not enough.
There could be similar OS limits for number of nics in the system.
Quoted 8 lines
Originally, vmchannel was a virtio interface with netlink interface to the
kernel.
Then, Anthony asked to change it to a socket interface with new address
family. It
was indeed a logical step. Then, David Miller was reluctant to add such
interface to the
kernel. Instead, he offered the network device solution.
Are we close to begin this loop again? :)
I propose to make the vmchannel connect to any capable device (serial, nic, usb, IO port, whatever) by adding some indirection. Moreover, at start no device should be "vmchannel-enabled" but the connection could be made dynamically at guest's request, then some of the disadvantages you listed are gone.
Quoted 9 lines
Let's try to stick to the nic solution. It has some advantages over pv
serial:
- Reliable communication if tcp is used
- Migration support for slirp
- No new driver in the guest.
- Might even work for older guests
The disadvantages are:
- Need to 'teach' guest daemons/firewalls not to handle/block the new
nic
The guest could request a vmchannel only after ensuring that the firewall is fixed..
Quoted 2 lines
- Link local addresses for ipv4 are problematic when using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
Quoted 6 lines
- We should either 1. not use link local on other links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.So additional nic can do the job and we have several flavours to choose
from.
The solution should be generic enough so that any nic can be connected to vmchannel.
Quoted 56 lines · Blue Swirl
Quoted 43 lines · Dor Laor
Carl-Daniel Hailfinger wrote:
On 11.01.2009 08:10, Blue Swirl wrote:Quoted 5 lines · Jamie Lokier
But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.
Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by
designing
something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.I think a serial port device should be universally supported by any OS
and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.I remember the old days of DOS networking where the Kirschbaum-Netz
software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.
Regards,
Carl-DanielQuoted 10 lines
Actually vmchannel started as a pv serial implementation. Standard serial
is
a bit low performing and demand an vmexit per byte (maybe it's not that bad
for us).
Moreover, various guest do not support more than 4 serial channels. Since
there
should be several channels and we like to preserve some for console/debug,
it is
not enough.
Quoted 2 lines
There could be similar OS limits for number of nics in the system.
But the number is much higher. It's the limitation of the #pci devices.
Quoted 11 lines
Quoted 10 lines
Originally, vmchannel was a virtio interface with netlink interface to the
kernel.
Then, Anthony asked to change it to a socket interface with new address
family. It
was indeed a logical step. Then, David Miller was reluctant to add such
interface to the
kernel. Instead, he offered the network device solution.
Are we close to begin this loop again? :)
Quoted 7 lines
I propose to make the vmchannel connect to any capable device (serial, nic, usb, IO port, whatever) by adding some indirection. Moreover, at start no device should be "vmchannel-enabled" but the connection could be made dynamically at guest's request, then some of the disadvantages you listed are gone.
It's logical, mainly for the serial.
My only fear is that too many options will confuse the users/developers.
Quoted 11 lines
Quoted 10 lines
Let's try to stick to the nic solution. It has some advantages over pv
serial:
- Reliable communication if tcp is used
- Migration support for slirp
- No new driver in the guest.
- Might even work for older guests
The disadvantages are:
- Need to 'teach' guest daemons/firewalls not to handle/block the new
nic
Quoted 3 lines
The guest could request a vmchannel only after ensuring that the firewall is fixed..
The installer of the guest agent is responsible for punching a hole in the firewall.
Quoted 7 lines
Quoted 4 lines
- Link local addresses for ipv4 are problematic when using on other
nics in parallelLikewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.
Quoted 5 lines
Quoted 5 lines
- We should either 1. not use link local on other links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.
Quoted 9 lines
Quoted 3 lines
So additional nic can do the job and we have several flavours to choose
from.The solution should be generic enough so that any nic can be connected to vmchannel.
Agreed.
Quoted 2 lines · Dor Laor
The installer of the guest agent is responsible for punching a hole in the firewall.
That's asking a lot from a generic installer. Guests differ enormously in how you do that - including different Linux guests.
Something else you have to do is disable forwarding between the vmchannel NIC and other NICs - even if the other NICs are forwarding enabled to each other. How do you do that on Linux? /proc/sys/net/ipv4/ip_forward is global, not per NIC... How do you do it on other guests?
It's easy to imagine a few simple guest agents written in C that compile easily on any guest unix you might want to run on... except this vmchannel setup would be the only non-portable part, and highly non-portable at that.
Quoted 8 lines
- Link local addresses for ipv4 are problematic when
using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.
No, that's not enough. Even when you have globally unique link-local addresses, you have the problem that NICs configured for link-local IP always have the same subnet, so routing doesn't work.
You could workaround this by using non-standard link-local IP on the vmchannel NIC. Now you're playing more games...
Quoted 11 lines
- We should either 1. not use link local on other
links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.So additional nic can do the job and we have several
flavours to choose
from.
The solution should be generic enough so that any nic can be connected
to vmchannel.
It sounds "generic" in the sense that you need a custom configuration which depends on the rest of the guest's configuration. Not really "drop in guest vmchannel app and it just works", is it?
If the guest vmchannel app installer looks at other NICs, and picks an IP subnet that the others aren't using, or uses link-local when that's not used on the others... That will work most of the time. But sometimes it will break a working guest some hours after it's installed. What happens if the guests's LAN NIC is using DHCP, so the vmchannel app picks link-local - and then the guests's LAN NIC changes to link-local itself after some hours running? That's not uncommon behaviour nowadays on some networks.
Handling all the cases _reliably_, adapting reactively to network config _changes_ on the other NICs while running, and doing so across many guest types (even just Linux distros and Windows) without having to have custom code for each guest type, is harder than it looks.
On the other hand, using packet sockets and not IP over the vmchannel NIC... (just pick another ethernet type) that would work reliably, but without the convenience of TCP/IP. It would need more support in the guest vmchannel app, and guest root access, but both sound plausible to implement.
-- Jamie
Quoted 7 lines · Jamie Lokier
Quoted 2 lines · Dor Laor
The installer of the guest agent is responsible for punching a hole in the firewall.
That's asking a lot from a generic installer. Guests differ enormously in how you do that - including different Linux guests.
Using network for vmchannel has its downsides. This is one of them. Every networking daemon needs some king of configuration and this is well understood by admins. BTW vmchannel will use only outgoing connection and they are usually allowed by firewalls.
Quoted 5 lines
Something else you have to do is disable forwarding between the vmchannel NIC and other NICs - even if the other NICs are forwarding enabled to each other. How do you do that on Linux?
/proc/sys/net/ipv4/ip_forwardis global, not per NIC... How do you do it on other guests?
No need to do that. Slirp will drop any packet forwarded to it while running in restricted mode.
Quoted 6 lines
It's easy to imagine a few simple guest agents written in C that compile easily on any guest unix you might want to run on... except this vmchannel setup would be the only non-portable part, and highly non-portable at that.
Actually the only nonportable part I see is finding vmchannel network device. After vmchannel device is determined getting IP from a network device is portable between Unixes.
Quoted 9 lines
Quoted 8 lines
- Link local addresses for ipv4 are problematic when
using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.
Quoted 4 lines
No, that's not enough. Even when you have globally unique link-local addresses, you have the problem that NICs configured for link-local IP always have the same subnet, so routing doesn't work.
Most Unixes have SO_BINDTODEVICE to solve this problem. Windows and others will probably have to add host route. But I prefer to use privet subnet outside of link local addresses. One less RFC to worry about.
Quoted 15 lines
You could workaround this by using non-standard link-local IP on the vmchannel NIC. Now you're playing more games...
Quoted 11 lines
- We should either 1. not use link local on other
links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.So additional nic can do the job and we have several
flavours to choose
from.
The solution should be generic enough so that any nic can be connected
to vmchannel.
Quoted 4 lines
It sounds "generic" in the sense that you need a custom configuration which depends on the rest of the guest's configuration. Not really "drop in guest vmchannel app and it just works", is it?
We all wanted to use something else for vmchannel, but unfortunately we were pushed to a networking solution. I still have PF_VMCHANNEL socket family code, so if you can convince David Miller that network is not good fit for vmchannel go for it :) There are certain restriction applies when you talk to him though: you can't mention virtualization as justification for vmchannel!
Quoted 8 lines
If the guest vmchannel app installer looks at other NICs, and picks an IP subnet that the others aren't using, or uses link-local when that's not used on the others... That will work most of the time. But sometimes it will break a working guest some hours after it's installed. What happens if the guests's LAN NIC is using DHCP, so the vmchannel app picks link-local - and then the guests's LAN NIC changes to link-local itself after some hours running? That's not uncommon behaviour nowadays on some networks.
In my opinion we shouldn't be too smart about choosing of a subnet and leave that to admin (using some reasonable default of cause).
Quoted 6 lines
Handling all the cases _reliably_, adapting reactively to network config _changes_ on the other NICs while running, and doing so across many guest types (even just Linux distros and Windows) without having to have custom code for each guest type, is harder than it looks.
Quoted 6 lines
On the other hand, using packet sockets and not IP over the vmchannel NIC... (just pick another ethernet type) that would work reliably, but without the convenience of TCP/IP. It would need more support in the guest vmchannel app, and guest root access, but both sound plausible to implement.
Firewalls can still filter your packets though.
--
Gleb.
Quoted 96 lines · Gleb Natapov
Quoted 9 lines · Jamie Lokier
Quoted 3 lines · Dor Laor
The installer of the guest agent is responsible for punching a hole in the firewall.
That's asking a lot from a generic installer. Guests differ enormously in how you do that - including different Linux guests.
Using network for vmchannel has its downsides. This is one of them. Every networking daemon needs some king of configuration and this is well understood by admins. BTW vmchannel will use only outgoing connection and they are usually allowed by firewalls.
Quoted 6 lines
Something else you have to do is disable forwarding between the vmchannel NIC and other NICs - even if the other NICs are forwarding enabled to each other. How do you do that on Linux?
/proc/sys/net/ipv4/ip_forwardis global, not per NIC... How do you do it on other guests?No need to do that. Slirp will drop any packet forwarded to it while running in restricted mode.
Quoted 6 lines
It's easy to imagine a few simple guest agents written in C that compile easily on any guest unix you might want to run on... except this vmchannel setup would be the only non-portable part, and highly non-portable at that.
Actually the only nonportable part I see is finding vmchannel network device. After vmchannel device is determined getting IP from a network device is portable between Unixes.
Quoted 9 lines
Quoted 9 lines
- Link local addresses for ipv4 are problematic when
using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.Quoted 5 lines
No, that's not enough. Even when you have globally unique link-local addresses, you have the problem that NICs configured for link-local IP always have the same subnet, so routing doesn't work.
Most Unixes have
SO_BINDTODEVICEto solve this problem. Windows and others will probably have to add host route. But I prefer to use privet subnet outside of link local addresses. One less RFC to worry about.Quoted 16 lines
You could workaround this by using non-standard link-local IP on the vmchannel NIC. Now you're playing more games...
Quoted 5 lines
- We should either 1. not use link local on other
links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.Quoted 7 lines
So additional nic can do the job and we have several
flavours to choose
from.
The solution should be generic enough so that any nic can be connected
to vmchannel.Quoted 5 lines
It sounds "generic" in the sense that you need a custom configuration which depends on the rest of the guest's configuration. Not really "drop in guest vmchannel app and it just works", is it?
We all wanted to use something else for vmchannel, but unfortunately we were pushed to a networking solution. I still have
PF_VMCHANNELsocket family code, so if you can convince David Miller that network is not good fit for vmchannel go for it :) There are certain restriction applies when you talk to him though: you can't mention virtualization as justification for vmchannel!Quoted 9 lines
If the guest vmchannel app installer looks at other NICs, and picks an IP subnet that the others aren't using, or uses link-local when that's not used on the others... That will work most of the time. But sometimes it will break a working guest some hours after it's installed. What happens if the guests's LAN NIC is using DHCP, so the vmchannel app picks link-local - and then the guests's LAN NIC changes to link-local itself after some hours running? That's not uncommon behaviour nowadays on some networks.
Quoted 16 lines
In my opinion we shouldn't be too smart about choosing of a subnet and leave that to admin (using some reasonable default of cause).
Quoted 12 lines
Handling all the cases _reliably_, adapting reactively to network config _changes_ on the other NICs while running, and doing so across many guest types (even just Linux distros and Windows) without having to have custom code for each guest type, is harder than it looks.
On the other hand, using packet sockets and not IP over the vmchannel NIC... (just pick another ethernet type) that would work reliably, but without the convenience of TCP/IP. It would need more support in the guest vmchannel app, and guest root access, but both sound plausible to implement.
Right. We have 3 options with their pros and cons:
1. Use link local addresses for the vmchannel link ONLY. Do not allow other nics to use it. The upside is there is no new subnet to manage.
btw: anybody know how physical host with multiple nic using link local behaves?
2. Use standard ip range using slirp dhcp for the vmchannel. There is no link local addresses for this nic. The down side is that the admin needs to provide/manage another subnet mask. Also slirp has to be changed in order to allow dynamic replacement of the 'host' IP (although shutdown+boot go around it.)
3. Using packet socket.
The upside - no IP addressing.
The downside - no IP addressing (kidding, mainly tcp reliability). The guest agent/host need to get synchronous acks for every
message.
Quoted 7 lines
Firewalls can still filter your packets though.
--
Gleb.
Quoted 11 lines · Dor Laor
As the guest OS's TCP is being used, what do you do about IP address space conflicts?
I.e. if NIC #1 is the guest's LAN, and NIC #2 is the vmchannel, how is
the vmchannel NIC going to be configured in a way that's guaranteed to
avoid breaking the LAN networking, which could be assigned any legal subnet (especially when bridging is used), and on some networks changes from time to time?
Quoted 5 lines
Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?
We plan to pick link local subnets for ipv4.
It solved all the above questions.
Using an ipv4 link local subnet for the vmchannel may break many guests. The guest's LAN may also be configured with a link local subnet, so routing will get messed up.
When bridged to the host LAN, any Windows guest on a LAN without DHCP will break, for example; so will current Linux distros. They use a link local subnet for the LAN interface, when DHCP is not detected.
(They might do something else when there's a second NIC, though. That would just be a further complication - you want the vmchannel NIC to have no visible effect other than the vmchannel apps working).
In fact, the guest's LAN may regularly _change_ between a link local subnet, a public IP subnet, and a private scope IP subnet (192.168..), while the guest is running.
This can happen if the guest is bridged to the host's LAN, and the host is on a network where DHCP is working sometimes, or where the host is being moved between networks such as a laptop host.
Quoted 3 lines
w.r.t the option of using virtio nic, there is advantage of using any other nic since this way there is no requirement to install virtio driver on windows or on other older Linux/other OSs.
I agree. Simple vmchannel monitoring apps may port easily to OSes which don't have a virtio driver, or even run without any changes if they're simple enough and statically linked.
-- Jamie
Quoted 10 lines · Jamie Lokier
Quoted 2 lines · Anthony Liguori
Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?
That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.
That's probably going to be difficult.
Quoted 2 lines
Is there some way to mark a PCI device so it will be ignored at boot time generically?
No.
Quoted 4 lines
Changing the PCI ID will do that for all guests,
but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?
I don't see how changing the PCI ID (I presume you mean vendor/device ID?) would help.
Quoted 4 lines
Suppose you start a guest with two "trusted" nics, because you want to run two unrelated vmchannel-using admin apps. How does each app know which nic to use - or do they share it?
Unrelated vmchannel apps will use different ports on the same nic. There will only ever be one trusted nic.
Quoted 3 lines
As the guest OS's TCP is being used, what do you do about IP address space conflicts?
The user can choose what address is used for the host.
Quoted 3 lines
Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?
slirp only supports IPv4.
Regards,
Anthony Liguori
Quoted 4 lines
-- Jamie
Quoted 7 lines · Anthony Liguori
Quoted 4 lines
Changing the PCI ID will do that for all guests,
but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?I don't see how changing the PCI ID (I presume you mean vendor/device ID?) would help.
It would stop the OS from loading a NIC driver for it.
Then later, then vmchannel-using guest app would force the OS to load a NIC driver for the alternative PCI ID. E.g. by loading a module on Linux with an alternative name and some module arguments. It's probably not feasible without changes to OSes.
Quoted 4 lines
Quoted 2 lines
As the guest OS's TCP is being used, what do you do about IP address space conflicts?
The user can choose what address is used for the host.
So the vmchannel will need some guest-specific configuration before the guest-host apps work? (And I don't mean OS-version-specific, but specific to individual guest configs?)
-- Jamie
Quoted 10 lines · Jamie Lokier
Quoted 7 lines · Anthony Liguori
Quoted 5 lines
Changing the PCI ID will do that for all guests,
but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?I don't see how changing the PCI ID (I presume you mean vendor/device ID?) would help.
Quoted 7 lines
It would stop the OS from loading a NIC driver for it.
Then later, then vmchannel-using guest app would force the OS to load a NIC driver for the alternative PCI ID. E.g. by loading a module on Linux with an alternative name and some module arguments. It's probably not feasible without changes to OSes.
It isn't. Drivers don't work that way.
Regards,
Anthony Liguori
Quoted 14 lines
Quoted 5 lines
Quoted 3 lines
As the guest OS's TCP is being used, what do you do about IP address space conflicts?
The user can choose what address is used for the host.
So the vmchannel will need some guest-specific configuration before the guest-host apps work? (And I don't mean OS-version-specific, but specific to individual guest configs?)
-- Jamie
Quoted 9 lines · Anthony Liguori
Quoted 7 lines
It would stop the OS from loading a NIC driver for it.
Then later, then vmchannel-using guest app would force the OS to load a NIC driver for the alternative PCI ID. E.g. by loading a module on Linux with an alternative name and some module arguments. It's probably not feasible without changes to OSes.
It isn't. Drivers don't work that way.
A few Linux drivers have done it, maybe not any more, that's why I thought of it.
-- Jamie