Skip to contentSkip to conversation

mark nic as trusted

PATCH Superseded Gleb Natapov 8 participants 34 messages Jan 7, 2009 – Jan 12, 2009

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 mark nic as trusted

This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.

hw/e1000.c +3 -0
index c326671..ea4d824 100644
@@ -1045,6 +1045,9 @@ pci_e1000_init(PCIBus *bus, NICInfo *nd, int devfn)
 
     pci_conf[0x3d] = 1; // interrupt pin 0
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     d->mmio_index = cpu_register_io_memory(0, e1000_mmio_read,
             e1000_mmio_write, d);
 
hw/eepro100.c +3 -0
index cb3ca09..654b389 100644
@@ -1754,6 +1754,9 @@ static void nic_init(PCIBus * bus, NICInfo * nd,
 
     pci_reset(s);
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     /* Add 64 * 2 EEPROM. i82557 and i82558 support a 64 word EEPROM,
      * i82559 and later support 64 or 256 word EEPROM. */
     s->eeprom = eeprom93xx_new(EEPROM_SIZE);
hw/ne2000.c +4 -0
index 3f0ccf5..c3f4e22 100644
@@ -806,6 +806,10 @@ void pci_ne2000_init(PCIBus *bus, NICInfo *nd, int devfn)
 
     pci_register_io_region(&d->dev, 0, 0x100,
                            PCI_ADDRESS_SPACE_IO, ne2000_map);
+
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     s = &d->ne2000;
     s->irq = d->dev.irq[0];
     s->pci_dev = (PCIDevice *)d;
hw/pci.c +17 -0
index 8252d21..05bfa4b 100644
@@ -536,6 +536,23 @@ static void pci_set_irq(void *opaque, int irq_num, int level)
     bus->set_irq(bus->irq_opaque, irq_num, bus->irq_count[irq_num] != 0);
 }
 
+int pci_add_capability(PCIDevice *d, uint8_t cap_id, uint8_t off, uint8_t *buf,
+        int len)
+{
+    uint16_t status = le16_to_cpu(*(uint16_t*)(d->config + 0x06));
+
+    if (off + len + 2 > 0x100 || off < 0x40)
+        return -1;
+
+    d->config[0x06] = cpu_to_le16(status | 0x10);
+    d->config[off] = cap_id;
+    d->config[off + 1] = d->config[0x34];
+    d->config[0x34] = off;
+    memcpy(d->config + off + 2, buf, len);
+
+    return 0;
+}
+
 /***********************************************************/
 /* monitor info on PCI */
 
hw/pci.h +2 -0
index 3b1caf5..87132ca 100644
@@ -117,6 +117,8 @@ typedef void (*pci_set_irq_fn)(qemu_irq *pic, int irq_num, int level);
 typedef int (*pci_map_irq_fn)(PCIDevice *pci_dev, int irq_num);
 PCIBus *pci_register_bus(pci_set_irq_fn set_irq, pci_map_irq_fn map_irq,
                          qemu_irq *pic, int devfn_min, int nirq);
+int pci_add_capability(PCIDevice *d, uint8_t cap_id, uint8_t off, uint8_t *buf,
+        int len);
 
 void pci_nic_init(PCIBus *bus, NICInfo *nd, int devfn);
 void pci_data_write(void *opaque, uint32_t addr, uint32_t val, int len);
hw/pcnet.c +3 -0
index 30c453c..990afb2 100644
@@ -2024,6 +2024,9 @@ void pci_pcnet_init(PCIBus *bus, NICInfo *nd, int devfn)
     pci_conf[0x3e] = 0x06;
     pci_conf[0x3f] = 0xff;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     /* Handler for memory-mapped I/O */
     d->mmio_index =
       cpu_register_io_memory(0, pcnet_mmio_read, pcnet_mmio_write, d);
hw/rtl8139.c +3 -0
index c3ab854..6e4c44f 100644
@@ -3423,6 +3423,9 @@ void pci_rtl8139_init(PCIBus *bus, NICInfo *nd, int devfn)
     pci_conf[0x3d] = 1;    /* interrupt pin 0 */
     pci_conf[0x34] = 0xdc;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     s = &d->rtl8139;
 
     /* I/O handler for memory-mapped I/O */
hw/virtio-net.c +3 -0
index 1f45b2d..186c6bd 100644
@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     if (!n)
         return NULL;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&n->vdev.pci_dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     n->vdev.get_config = virtio_net_update_config;
     n->vdev.get_features = virtio_net_get_features;
     n->vdev.set_features = virtio_net_set_features;
net.c +5 -0
index 6af4255..000768f 100644
@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)
         if (get_param_value(buf, sizeof(buf), "model", p)) {
             nd->model = strdup(buf);
         }
+        if (get_param_value(buf, sizeof(buf), "trusted", p)) {
+            strncpy(nd->secure_cookie, buf, sizeof(nd->secure_cookie));
+        } else {
+            nd->secure_cookie[0] = NULL;
+        }
         nd->vlan = vlan;
         nb_nics++;
         vlan->nb_guest_devs++;
net.h +1 -0
index 31c7a30..e57e6e5 100644
@@ -50,6 +50,7 @@ struct NICInfo {
     uint8_t macaddr[6];
     const char *model;
     VLANState *vlan;
+    uint8_t secure_cookie[14];
 };
 
 extern int nb_nics;

--
Gleb.

#1 lore raw

Hi Gleb,

Quoted 4 lines · Gleb Natapov

This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.

I'm not sure I follow - is this cookie a shared secret that only the host and guest knows, or do literally mean that the cookie will contain the string "Trusted" as a indicator that the guest can trust the NIC?

Quoted 1 line

...

hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313

Quoted 10 lines
hw/virtio-net.c +2 -0
index 1f45b2d..186c6bd 100644
@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     if (!n)
         return NULL;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&n->vdev.pci_dev, 0x0f, 0xf0, nd->secure_cookie, 14);

How was the Capability ID 0x0f chosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:

#define  PCI_CAP_ID_AGP3        0x0E    /* AGP Target PCI-PCI bridge */
#define  PCI_CAP_ID_EXP         0x10    /* PCI Express */

Also, to reduce magic numbers it would be nice to define the CAP_ID (0xf) and offset (0xf0) as a macro somewhere and use sizeof(nd->secure_cookie) instead of 14.

net.c int net_client_init(const char *device, const char *p) · line 1480

Quoted 12 lines
net.c +4 -0
index 6af4255..000768f 100644
@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)
         if (get_param_value(buf, sizeof(buf), "model", p)) {
             nd->model = strdup(buf);
         }
+        if (get_param_value(buf, sizeof(buf), "trusted", p)) {
+            strncpy(nd->secure_cookie, buf, sizeof(nd->secure_cookie));
+        } else {
+            nd->secure_cookie[0] = NULL;

NULL isn't a uint8_t, use '\0' instead I guess. Or maybe just memset() the NICInfo struct before starting to assign to it.

Cheers,
Mark.

#2 lore raw
Quoted 4 lines · Mark McLoughlin

Hi Gleb,

On Wed, 2009-01-07 at 16:26 +0200, Gleb Natapov wrote:

Quoted 9 lines
Quoted 4 lines

This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.

I'm not sure I follow - is this cookie a shared secret that only the host and guest knows, or do literally mean that the cookie will contain the string "Trusted" as a indicator that the guest can trust the NIC?

The presence of the capability indicates that the nic is trusted, but I added a possibility to pass 14 byte string from a host too. The string can be used as shared secret.

Quoted 10 lines
Quoted 1 line

...

Quoted 8 lines
hw/virtio-net.c +0 -0
index 1f45b2d..186c6bd 100644
@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     if (!n)
         return NULL;
 
Quoted 9 lines

hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313

Quoted 2 lines
hw/virtio-net.c +2 -0
+    if (nd->secure_cookie[0])
+        pci_add_capability(&n->vdev.pci_dev, 0x0f, 0xf0, nd->secure_cookie, 14);

How was the Capability ID 0x0f chosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:

#define  PCI_CAP_ID_AGP3        0x0E    /* AGP Target PCI-PCI bridge */
#define  PCI_CAP_ID_EXP         0x10    /* PCI Express */

It is "secure device capability", so I used it based on the name.

Quoted 3 lines

Also, to reduce magic numbers it would be nice to define the CAP_ID (0xf) and offset (0xf0) as a macro somewhere and use sizeof(nd->secure_cookie) instead of 14.

OK. Good point.

Quoted 17 lines

net.c int net_client_init(const char *device, const char *p) · line 1480

Quoted 13 lines
net.c +4 -0
index 6af4255..000768f 100644
@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)
         if (get_param_value(buf, sizeof(buf), "model", p)) {
             nd->model = strdup(buf);
         }
+        if (get_param_value(buf, sizeof(buf), "trusted", p)) {
+            strncpy(nd->secure_cookie, buf, sizeof(nd->secure_cookie));
+        } else {
+            nd->secure_cookie[0] = NULL;

NULL isn't a uint8_t, use '\0' instead I guess. Or maybe just memset() the NICInfo struct before starting to assign to it.

OK. I wounder why I've used NULL here in the first place.

--
Gleb.

#3 lore raw
Quoted 10 lines · Gleb Natapov
Quoted 9 lines

hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313

Quoted 2 lines
hw/virtio-net.c +2 -0
+    if (nd->secure_cookie[0])
+        pci_add_capability(&n->vdev.pci_dev, 0x0f, 0xf0, nd->secure_cookie, 14);

How was the Capability ID 0x0f chosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:

#define  PCI_CAP_ID_AGP3        0x0E    /* AGP Target PCI-PCI bridge */
#define  PCI_CAP_ID_EXP         0x10    /* PCI Express */

It is "secure device capability", so I used it based on the name.

Ah, I see this in pciutils now:

  #define  PCI_CAP_ID_SECURE      0x0F    /* Secure device (?) */

Where is the capability format published? I can't seem to find any specification for it ...

Cheers,
Mark.

#4 lore raw
Quoted 11 lines · Mark McLoughlin
Quoted 9 lines · Gleb Natapov
Quoted 9 lines

hw/virtio-net.c PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn) · line 313

Quoted 2 lines
hw/virtio-net.c +2 -0
+    if (nd->secure_cookie[0])
+        pci_add_capability(&n->vdev.pci_dev, 0x0f, 0xf0, nd->secure_cookie, 14);

How was the Capability ID 0x0f chosen? It it unallocated by the PCI SIG allocated it or ...? I see it's not defined in the kernel sources:

#define  PCI_CAP_ID_AGP3        0x0E    /* AGP Target PCI-PCI bridge */
#define  PCI_CAP_ID_EXP         0x10    /* PCI Express */
Quoted 9 lines
Quoted 1 line

It is "secure device capability", so I used it based on the name.

Ah, I see this in pciutils now:

  #define  PCI_CAP_ID_SECURE      0x0F    /* Secure device (?) */

Where is the capability format published? I can't seem to find any specification for it ...

Me too. The only thing I found is that AMD barcelona chipset uses it. Look at page 260 of this doc for details how:
http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/31116_PUB_BKDG-3-18_11-13-08.pdf

I just hijacked the capability for our needs :)

--
Gleb.

#5 lore raw
In reply to Gleb Natapov
Quoted 7 lines · Gleb Natapov

This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.

What utility does this have? Does this make Windows happy in some special way?

Regards,

Anthony Liguori

Quoted 159 lines
hw/e1000.c +3 -0
index c326671..ea4d824 100644
@@ -1045,6 +1045,9 @@ pci_e1000_init(PCIBus *bus, NICInfo *nd, int devfn)
 
     pci_conf[0x3d] = 1; // interrupt pin 0
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     d->mmio_index = cpu_register_io_memory(0, e1000_mmio_read,
             e1000_mmio_write, d);
 
hw/eepro100.c +3 -0
index cb3ca09..654b389 100644
@@ -1754,6 +1754,9 @@ static void nic_init(PCIBus * bus, NICInfo * nd,
 
     pci_reset(s);
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     /* Add 64 * 2 EEPROM. i82557 and i82558 support a 64 word EEPROM,
      * i82559 and later support 64 or 256 word EEPROM. */
     s->eeprom = eeprom93xx_new(EEPROM_SIZE);
hw/ne2000.c +4 -0
index 3f0ccf5..c3f4e22 100644
@@ -806,6 +806,10 @@ void pci_ne2000_init(PCIBus *bus, NICInfo *nd, int devfn)
 
     pci_register_io_region(&d->dev, 0, 0x100,
                            PCI_ADDRESS_SPACE_IO, ne2000_map);
+
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     s = &d->ne2000;
     s->irq = d->dev.irq[0];
     s->pci_dev = (PCIDevice *)d;
hw/pci.c +17 -0
index 8252d21..05bfa4b 100644
@@ -536,6 +536,23 @@ static void pci_set_irq(void *opaque, int irq_num, int level)
     bus->set_irq(bus->irq_opaque, irq_num, bus->irq_count[irq_num] != 0);
 }
 
+int pci_add_capability(PCIDevice *d, uint8_t cap_id, uint8_t off, uint8_t *buf,
+        int len)
+{
+    uint16_t status = le16_to_cpu(*(uint16_t*)(d->config + 0x06));
+
+    if (off + len + 2 > 0x100 || off < 0x40)
+        return -1;
+
+    d->config[0x06] = cpu_to_le16(status | 0x10);
+    d->config[off] = cap_id;
+    d->config[off + 1] = d->config[0x34];
+    d->config[0x34] = off;
+    memcpy(d->config + off + 2, buf, len);
+
+    return 0;
+}
+
 /***********************************************************/
 /* monitor info on PCI */
 
hw/pci.h +2 -0
index 3b1caf5..87132ca 100644
@@ -117,6 +117,8 @@ typedef void (*pci_set_irq_fn)(qemu_irq *pic, int irq_num, int level);
 typedef int (*pci_map_irq_fn)(PCIDevice *pci_dev, int irq_num);
 PCIBus *pci_register_bus(pci_set_irq_fn set_irq, pci_map_irq_fn map_irq,
                          qemu_irq *pic, int devfn_min, int nirq);
+int pci_add_capability(PCIDevice *d, uint8_t cap_id, uint8_t off, uint8_t *buf,
+        int len);
 
 void pci_nic_init(PCIBus *bus, NICInfo *nd, int devfn);
 void pci_data_write(void *opaque, uint32_t addr, uint32_t val, int len);
hw/pcnet.c +3 -0
index 30c453c..990afb2 100644
@@ -2024,6 +2024,9 @@ void pci_pcnet_init(PCIBus *bus, NICInfo *nd, int devfn)
     pci_conf[0x3e] = 0x06;
     pci_conf[0x3f] = 0xff;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     /* Handler for memory-mapped I/O */
     d->mmio_index =
       cpu_register_io_memory(0, pcnet_mmio_read, pcnet_mmio_write, d);
hw/rtl8139.c +3 -0
index c3ab854..6e4c44f 100644
@@ -3423,6 +3423,9 @@ void pci_rtl8139_init(PCIBus *bus, NICInfo *nd, int devfn)
     pci_conf[0x3d] = 1;    /* interrupt pin 0 */
     pci_conf[0x34] = 0xdc;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&d->dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     s = &d->rtl8139;
 
     /* I/O handler for memory-mapped I/O */
hw/virtio-net.c +3 -0
index 1f45b2d..186c6bd 100644
@@ -309,6 +309,9 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
     if (!n)
         return NULL;
 
+    if (nd->secure_cookie[0])
+        pci_add_capability(&n->vdev.pci_dev, 0x0f, 0xf0, nd->secure_cookie, 14);
+
     n->vdev.get_config = virtio_net_update_config;
     n->vdev.get_features = virtio_net_get_features;
     n->vdev.set_features = virtio_net_set_features;
net.c +5 -0
index 6af4255..000768f 100644
@@ -1474,6 +1474,11 @@ int net_client_init(const char *device, const char *p)
         if (get_param_value(buf, sizeof(buf), "model", p)) {
             nd->model = strdup(buf);
         }
+        if (get_param_value(buf, sizeof(buf), "trusted", p)) {
+            strncpy(nd->secure_cookie, buf, sizeof(nd->secure_cookie));
+        } else {
+            nd->secure_cookie[0] = NULL;
+        }
         nd->vlan = vlan;
         nb_nics++;
         vlan->nb_guest_devs++;
net.h +1 -1
index 31c7a30..e57e6e5 100644
@@ -50,6 +50,7 @@ struct NICInfo {
     uint8_t macaddr[6];
     const char *model;
     VLANState *vlan;
+    uint8_t secure_cookie[14];
 };
 
 extern int nb_nics;
--

Gleb.

#6 lore raw
Quoted 6 lines · Anthony Liguori
Quoted 5 lines · Gleb Natapov

This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.

Quoted 6 lines
Quoted 2 lines

What utility does this have? Does this make Windows happy in some special way?

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.

--
Gleb.

#7 lore raw
Quoted 12 lines · Gleb Natapov
Quoted 10 lines · Anthony Liguori
Quoted 8 lines · Gleb Natapov

This patch allows to mark specific nic as trusted by adding special PCI capability. "Trusted" means that it is used for communication between host and guest and no malicious entity can inject traffic to the nic.

Quoted 8 lines
Quoted 3 lines

What utility does this have? Does this make Windows happy in some special way?

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Regards,

Anthony Liguori

Quoted 5 lines

--
Gleb.

#8 lore raw
Quoted 6 lines · Anthony Liguori
Quoted 4 lines

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Like if you were doing PCI passthrough of an e1000...

Regards,

Anthony Liguori

Quoted 10 lines

Regards,

Anthony Liguori

Quoted 5 lines
Signature · 1 line
            Gleb.
#9 lore raw
Quoted 7 lines · Anthony Liguori
Quoted 6 lines · Anthony Liguori
Quoted 5 lines

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real
host and "own" a guest.

Quoted 5 lines
Quoted 2 lines

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Like if you were doing PCI passthrough of an e1000...

It's not just one bit. It is 14 byte string. We can put something unique there.

--
Gleb.

#10 lore raw
Quoted 9 lines · Gleb Natapov
Quoted 7 lines · Anthony Liguori
Quoted 5 lines · Anthony Liguori
Quoted 5 lines

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.

Quoted 7 lines
Quoted 5 lines
Quoted 2 lines

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Like if you were doing PCI passthrough of an e1000...

It's not just one bit. It is 14 byte string. We can put something unique there.

This is for vmchannel? Why not add a feature to virtio-net?

Regards,

Anthony Liguori

#11 lore raw
Quoted 13 lines · Anthony Liguori
Quoted 12 lines · Gleb Natapov
Quoted 10 lines · Anthony Liguori
Quoted 8 lines · Anthony Liguori
Quoted 5 lines

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real
host and "own" a guest.

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

Like if you were doing PCI passthrough of an e1000...

It's not just one bit. It is 14 byte string. We can put something unique there.

This is for vmchannel? Why not add a feature to virtio-net?

Yes. This is for vmchannel. Or any other management solution that work over network. It has to know what network it can trust. The alternative is much more complex (security certificates, etc). Why do it virtio-net specific? What's wrong with more general solution?

--
Gleb.

#12 lore raw
Quoted 25 lines · Gleb Natapov
Quoted 23 lines · Anthony Liguori
Quoted 21 lines · Gleb Natapov
Quoted 8 lines · Anthony Liguori
Quoted 5 lines · Anthony Liguori
Quoted 5 lines

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real host and "own" a guest.

Quoted 7 lines
Quoted 3 lines

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Like if you were doing PCI passthrough of an e1000...

It's not just one bit. It is 14 byte string. We can put something unique there.

Quoted 8 lines
Quoted 3 lines

This is for vmchannel? Why not add a feature to virtio-net?

Yes. This is for vmchannel. Or any other management solution that work over network. It has to know what network it can trust. The alternative is much more complex (security certificates, etc). Why do it virtio-net specific? What's wrong with more general solution?

Does Windows provide an API for determining "trustedness"? How is this exposed to userspace in Linux?

The thinking behind virtio-net is that you may want to expose a different userspace interface in Windows than networking (maybe something more direct) since it may be impossible to get around the Windows firewalling nonsense. With virtio-net, you could have the Windows driver check the special "vmchannel" flag and present a different userspace interface than the traditional network driver.

Although that's just a thought.

Regards,

Anthony Liguori

Quoted 5 lines

--
Gleb.

#13 lore raw
Quoted 25 lines · Anthony Liguori
Quoted 24 lines · Gleb Natapov
Quoted 22 lines · Anthony Liguori
Quoted 9 lines · Gleb Natapov
Quoted 7 lines · Anthony Liguori
Quoted 5 lines · Anthony Liguori
Quoted 5 lines

That is for secure guest<->host communication over network. Guest has to know somehow which link host uses for communication. If guest has no way to know this, another computer on untrusted network can pretend it is real
host and "own" a guest.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

So this is for vmchannel? How do you differentiate a real device with that bit set compared to the vmchannel device?

Like if you were doing PCI passthrough of an e1000...

It's not just one bit. It is 14 byte string. We can put something unique there.

This is for vmchannel? Why not add a feature to virtio-net?

Quoted 9 lines
Quoted 5 lines

Yes. This is for vmchannel. Or any other management solution that work over network. It has to know what network it can trust. The alternative is much more complex (security certificates, etc). Why do it virtio-net specific? What's wrong with more general solution?

Does Windows provide an API for determining "trustedness"? How is this exposed to userspace in Linux?

Windows and Linux allows access to PCI space, so it is easy to check "trustedness". AFAIK we already have Windows code to do that. In Linux it can be done even by shell script by examining files in /sys.

Quoted 7 lines

The thinking behind virtio-net is that you may want to expose a different userspace interface in Windows than networking (maybe something more direct) since it may be impossible to get around the Windows firewalling nonsense. With virtio-net, you could have the Windows driver check the special "vmchannel" flag and present a different userspace interface than the traditional network driver.

Then we will have to implement some kind of reliable protocol on top of that. It can be done, but it make sense to reuse TCP for that. We will have to request proper firewall config from VM admin though. It is possible to dial with build in Windows firewall automatically, but third party firewalls may require admin intervention.

--
Gleb.

#14 lore raw
Quoted 6 lines · Gleb Natapov
Quoted 1 line · Anthony Liguori

Windows and Linux allows access to PCI space, so it is easy to check "trustedness". AFAIK we already have Windows code to do that. In Linux it can be done even by shell script by examining files in /sys.

Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?

Regards,

Anthony Liguori

#15 lore raw
Quoted 3 lines · Anthony Liguori

Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?

That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.

Is there some way to mark a PCI device so it will be ignored at boot time generically? Changing the PCI ID will do that for all guests, but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?

Suppose you start a guest with two "trusted" nics, because you want to run two unrelated vmchannel-using admin apps. How does each app know which nic to use - or do they share it?

As the guest OS's TCP is being used, what do you do about IP address space conflicts?

I.e. if NIC #1 is the guest's LAN, and NIC #2 is the vmchannel, how is the vmchannel NIC going to be configured in a way that's guaranteed to avoid breaking the LAN networking, which could be assigned any legal subnet (especially when bridging is used), and on some networks changes from time to time?

Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?

-- Jamie

#16 lore raw
Quoted 10 lines · Jamie Lokier
Quoted 2 lines · Anthony Liguori

Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?

That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.

We can make fedora, rhel and libvirt support it. It might be a bit painful but since a network device was chosen for this propose then that's the right way to go. Others can either use 3rd agents to cancel firewalls like we plan to do for windows, or as was suggested, change the pci device id and load a bit different driver.

You suggestion is good also since it will be good for personal usages where the guest can easily reach the host network and the user can easily cancel firewalls.

Quoted 5 lines

Is there some way to mark a PCI device so it will be ignored at boot time generically? Changing the PCI ID will do that for all guests, but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?

Alternatively one can hot plug the vmchanneled nic, right after boot. IMHO I rather stick with guest mgmt agent take care of the accesses to the nic.

Quoted 5 lines

Suppose you start a guest with two "trusted" nics, because you want to run two unrelated vmchannel-using admin apps. How does each app know which nic to use - or do they share it?

Each vmchannel is bond on the host to a separate pair of qemu_chr_device and a matching ip:port listening. So if there are n agents in the guest, each should connect to his ip:port without being aware to the others.

Quoted 13 lines

As the guest OS's TCP is being used, what do you do about IP address space conflicts?

I.e. if NIC #1 is the guest's LAN, and NIC #2 is the vmchannel, how is the vmchannel NIC going to be configured in a way that's guaranteed to avoid breaking the LAN networking, which could be assigned any legal subnet (especially when bridging is used), and on some networks changes from time to time?

Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?

We plan to pick link local subnets for ipv4.
It solved all the above questions. It should be connected to slirp without passing through the host stack/bridges (although it can be open too).

Quoted 4 lines

-- Jamie

w.r.t the option of using virtio nic, there is advantage of using any other nic since this way there is no requirement to install virtio driver on windows or on other older Linux/other OSs.

#30 lore raw
In reply to Jamie Lokier
Quoted 10 lines · Jamie Lokier
Quoted 2 lines · Anthony Liguori

Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?

That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.

That's probably going to be difficult.

Quoted 2 lines

Is there some way to mark a PCI device so it will be ignored at boot time generically?

No.

Quoted 4 lines

Changing the PCI ID will do that for all guests,
but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?

I don't see how changing the PCI ID (I presume you mean vendor/device ID?) would help.

Quoted 4 lines

Suppose you start a guest with two "trusted" nics, because you want to run two unrelated vmchannel-using admin apps. How does each app know which nic to use - or do they share it?

Unrelated vmchannel apps will use different ports on the same nic. There will only ever be one trusted nic.

Quoted 3 lines

As the guest OS's TCP is being used, what do you do about IP address space conflicts?

The user can choose what address is used for the host.

Quoted 3 lines

Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?

slirp only supports IPv4.

Regards,

Anthony Liguori

Quoted 4 lines

-- Jamie

#17 lore raw
In reply to Dor Laor
Quoted 13 lines · Dor Laor
Quoted 7 lines · Jamie Lokier
Quoted 4 lines · Anthony Liguori

Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?

Quoted 5 lines

That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.

Quoted 4 lines

We can make fedora, rhel and libvirt support it. It might be a bit painful but since
a network device was chosen for this propose then that's the right way to go.

For new Fedora / RHEL yes, but a large number of people using virt are doing so with old OS versions, and the chances of anyone retro-fitting all old distros is near zero. You might get it done if they were back porting the VirtIO NIC devices to old distros, but almost certainly not for arbitrary NIC devices like rtl8139/e1000/etc because its a huge QA testing headache to avoid regressions.

Daniel

Signature · 4 lines
|: Red Hat, Engineering, London   -o-   http://people.redhat.com/berrange/ :|
|: http://libvirt.org  -o-  http://virt-manager.org  -o-  http://ovirt.org :|
|: http://autobuild.org       -o-         http://search.cpan.org/~danberr/ :|
|: GnuPG: 7D3B9505  -o-  F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|
#18 lore raw
Quoted 19 lines · Daniel P. Berrange
Quoted 13 lines · Dor Laor
Quoted 7 lines · Jamie Lokier
Quoted 4 lines · Anthony Liguori

Are we going to have a standard way of doing this in Linux distros such that these nics are treated differently from other nics? Have we gotten the appropriate distro folks to agree to this?

Quoted 5 lines

That wouldn't work for older distros and Windows anyway. But you might reasonably want to run apps doing guest-host communication on older guest distros too, simply as an app, not requiring guest customisation.

Quoted 4 lines

We can make fedora, rhel and libvirt support it. It might be a bit painful but since
a network device was chosen for this propose then that's the right way to go.

Quoted 6 lines

For new Fedora / RHEL yes, but a large number of people using virt are doing so with old OS versions, and the chances of anyone retro-fitting all old distros is near zero. You might get it done if they were back porting the VirtIO NIC devices to old distros, but almost certainly not for arbitrary NIC devices like rtl8139/e1000/etc because its a huge QA testing headache to avoid regressions.

In some circles, a major purpose of virtualisation is to run old OS versions, either copied from machines where the hardware is aging to keep a working system still working, or for compatibility testing.

Changing a working guest setup isn't cool. Adding a small monitoring/reporting app (e.g. that shows the machine's load average, process table, network connections or whatever over a vmchannel) may be cool, provided it has negligable impact, even if changing its system configuration is not.

-- Jamie

#29 lore raw
In reply to Dor Laor
Quoted 11 lines · Dor Laor

As the guest OS's TCP is being used, what do you do about IP address space conflicts?

I.e. if NIC #1 is the guest's LAN, and NIC #2 is the vmchannel, how is
the vmchannel NIC going to be configured in a way that's guaranteed to
avoid breaking the LAN networking, which could be assigned any legal subnet (especially when bridging is used), and on some networks changes from time to time?

Quoted 5 lines

Perhaps vmchannel will only use IPv6, so it can confidently pick a unique link-local address?

We plan to pick link local subnets for ipv4.
It solved all the above questions.

Using an ipv4 link local subnet for the vmchannel may break many guests. The guest's LAN may also be configured with a link local subnet, so routing will get messed up.

When bridged to the host LAN, any Windows guest on a LAN without DHCP will break, for example; so will current Linux distros. They use a link local subnet for the LAN interface, when DHCP is not detected.

(They might do something else when there's a second NIC, though. That would just be a further complication - you want the vmchannel NIC to have no visible effect other than the vmchannel apps working).

In fact, the guest's LAN may regularly _change_ between a link local subnet, a public IP subnet, and a private scope IP subnet (192.168..), while the guest is running.

This can happen if the guest is bridged to the host's LAN, and the host is on a network where DHCP is working sometimes, or where the host is being moved between networks such as a laptop host.

Quoted 3 lines

w.r.t the option of using virtio nic, there is advantage of using any other nic since this way there is no requirement to install virtio driver on windows or on other older Linux/other OSs.

I agree. Simple vmchannel monitoring apps may port easily to OSes which don't have a virtio driver, or even run without any changes if they're simple enough and statically linked.

-- Jamie

#31 lore raw
In reply to Anthony Liguori
Quoted 7 lines · Anthony Liguori
Quoted 4 lines

Changing the PCI ID will do that for all guests,
but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?

I don't see how changing the PCI ID (I presume you mean vendor/device ID?) would help.

It would stop the OS from loading a NIC driver for it.

Then later, then vmchannel-using guest app would force the OS to load a NIC driver for the alternative PCI ID. E.g. by loading a module on Linux with an alternative name and some module arguments. It's probably not feasible without changes to OSes.

Quoted 4 lines
Quoted 2 lines

As the guest OS's TCP is being used, what do you do about IP address space conflicts?

The user can choose what address is used for the host.

So the vmchannel will need some guest-specific configuration before the guest-host apps work? (And I don't mean OS-version-specific, but specific to individual guest configs?)

-- Jamie

#19 lore raw
In reply to Jamie Lokier
Quoted 10 lines · Jamie Lokier
Quoted 7 lines · Daniel P. Berrange

For new Fedora / RHEL yes, but a large number of people using virt are doing so with old OS versions, and the chances of anyone retro-fitting all old distros is near zero. You might get it done if they were back porting the VirtIO NIC devices to old distros, but almost certainly not for arbitrary NIC devices like rtl8139/e1000/etc because its a huge QA testing headache to avoid regressions.

Quoted 4 lines

In some circles, a major purpose of virtualisation is to run old OS versions, either copied from machines where the hardware is aging to keep a working system still working, or for compatibility testing.

When dealing with older OSes, there really no limit in terms of what we can do. For instance, there's no reason that the backported version of the virtio-net driver cannot expose a chardev interface if the distro doesn't have the appropriate configuration.

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

You cannot get around having a kernel module to support something like vmchannel so that's always going to be a requirement. A completely userspace approach ala the VMware backdoor hackery is just too hideous.

Regards,

Anthony Liguori

Quoted 10 lines

Changing a working guest setup isn't cool. Adding a small monitoring/reporting app (e.g. that shows the machine's load average, process table, network connections or whatever over a vmchannel) may be cool, provided it has negligable impact, even if changing its system configuration is not.

-- Jamie

#32 lore raw
In reply to Jamie Lokier
Quoted 10 lines · Jamie Lokier
Quoted 7 lines · Anthony Liguori
Quoted 5 lines

Changing the PCI ID will do that for all guests,
but is it then feasible for the vmchannel guest admin software to bind a NIC driver to a non-standard PCI ID, on the major OSes?

I don't see how changing the PCI ID (I presume you mean vendor/device ID?) would help.

Quoted 7 lines

It would stop the OS from loading a NIC driver for it.

Then later, then vmchannel-using guest app would force the OS to load a NIC driver for the alternative PCI ID. E.g. by loading a module on Linux with an alternative name and some module arguments. It's probably not feasible without changes to OSes.

It isn't. Drivers don't work that way.

Regards,

Anthony Liguori

Quoted 14 lines
Quoted 5 lines
Quoted 3 lines

As the guest OS's TCP is being used, what do you do about IP address space conflicts?

The user can choose what address is used for the host.

So the vmchannel will need some guest-specific configuration before the guest-host apps work? (And I don't mean OS-version-specific, but specific to individual guest configs?)

-- Jamie

#33 lore raw
Quoted 9 lines · Anthony Liguori
Quoted 7 lines

It would stop the OS from loading a NIC driver for it.

Then later, then vmchannel-using guest app would force the OS to load a NIC driver for the alternative PCI ID. E.g. by loading a module on Linux with an alternative name and some module arguments. It's probably not feasible without changes to OSes.

It isn't. Drivers don't work that way.

A few Linux drivers have done it, maybe not any more, that's why I thought of it.

-- Jamie

#20 lore raw
In reply to Anthony Liguori
Quoted 3 lines · Anthony Liguori

You cannot get around having a kernel module to support something like vmchannel so that's always going to be a requirement. A completely userspace approach ala the VMware backdoor hackery is just too hideous.

Eh? I thought the point of vmchannel is to use the guest network stack, so guest processes - including user ones - can potentially speak over it to the host via a marked, designated channel.

If you need a kernel module, i.e. you're doing special kernely things, there's no need for the vmchannel device to be a normal NIC device, no need for it to interfere with the guest networking (e.g. conflicting IPv4 subnets), and it can use virtio, I agree in that case.

Now that it's clear I'm imagining guest userspace admin processes and you're imagining a kernel module on the guest, the rest of this exchange makes sense :-)

Quoted 8 lines
Quoted 3 lines

In some circles, a major purpose of virtualisation is to run old OS versions, either copied from machines where the hardware is aging to keep a working system still working, or for compatibility testing.

When dealing with older OSes, there really no limit in terms of what we can do. For instance, there's no reason that the backported version of the virtio-net driver cannot expose a chardev interface if the distro doesn't have the appropriate configuration.

Realistically, the set of (older) OSes where you'd like to run simple guest-host communicating apps for VM management, especially simple monitoring apps (load avg, memory use, number of processes), is much larger than those older OSes which virtio-net can be loaded onto.

E.g. Linux and Windows apps to do that could run on very old/obscure versions and diverse distributions, if statically linked.

Sometimes there'll be no backport of kernel drivers, and if there is one, that doesn't mean you can load it into an existing guest image or even compile it for that exact guest kernel.

I think it's quite reasonable to want to run simple monitoring apps on such guests.

Quoted 5 lines

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.

-- Jamie

#21 lore raw
Quoted 6 lines · Jamie Lokier
Quoted 5 lines

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

Quoted 6 lines

Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.

I think a serial port device should be universally supported by any OS and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.

About the overall idea of host marking something as trusted, here are some alternative ideas:
- guest could ask for a trusted channel and then the host would create one
- guest could ask the host to list the trusted devices
- on guest's request, host can cut an existing device from outside world and start using that as the trusted one

#22 lore raw
Quoted 16 lines · Blue Swirl
Quoted 6 lines · Jamie Lokier
Quoted 5 lines

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

Quoted 7 lines

Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.

Quoted 5 lines

I think a serial port device should be universally supported by any OS and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.

I remember the old days of DOS networking where the Kirschbaum-Netz software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.

Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.

Regards,
Carl-Daniel

Signature · 1 line
http://www.hailfinger.org/
#23 lore raw
Quoted 31 lines · Carl-Daniel Hailfinger
Quoted 17 lines · Blue Swirl
Quoted 6 lines · Jamie Lokier
Quoted 5 lines

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

Quoted 8 lines

Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by designing something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.

Quoted 6 lines

I think a serial port device should be universally supported by any OS and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.

I remember the old days of DOS networking where the Kirschbaum-Netz software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.

Quoted 8 lines

Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.

Regards,
Carl-Daniel

Actually vmchannel started as a pv serial implementation. Standard serial is a bit low performing and demand an vmexit per byte (maybe it's not that bad for us). Moreover, various guest do not support more than 4 serial channels. Since there should be several channels and we like to preserve some for console/debug, it is not enough.
Originally, vmchannel was a virtio interface with netlink interface to the kernel. Then, Anthony asked to change it to a socket interface with new address family. It was indeed a logical step. Then, David Miller was reluctant to add such interface to the kernel. Instead, he offered the network device solution.
Are we close to begin this loop again? :)

Let's try to stick to the nic solution. It has some advantages over pv serial:
- Reliable communication if tcp is used
- Migration support for slirp
- No new driver in the guest.
- Might even work for older guests
The disadvantages are:
- Need to 'teach' guest daemons/firewalls not to handle/block the
new nic
- Link local addresses for ipv4 are problematic when using on other
nics in parallel
- We should either 1. not use link local on other links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.

So additional nic can do the job and we have several flavours to choose from.

Regards,
Dor

#24 lore raw
Quoted 44 lines · Dor Laor

Carl-Daniel Hailfinger wrote:
On 11.01.2009 08:10, Blue Swirl wrote:

Quoted 5 lines · Jamie Lokier

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by
designing
something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.

I think a serial port device should be universally supported by any OS
and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.

I remember the old days of DOS networking where the Kirschbaum-Netz
software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.

Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.

Regards,
Carl-Daniel

Quoted 9 lines

Actually vmchannel started as a pv serial implementation. Standard serial
is
a bit low performing and demand an vmexit per byte (maybe it's not that bad
for us).
Moreover, various guest do not support more than 4 serial channels. Since
there
should be several channels and we like to preserve some for console/debug,
it is
not enough.

There could be similar OS limits for number of nics in the system.

Quoted 8 lines

Originally, vmchannel was a virtio interface with netlink interface to the
kernel.
Then, Anthony asked to change it to a socket interface with new address
family. It
was indeed a logical step. Then, David Miller was reluctant to add such
interface to the
kernel. Instead, he offered the network device solution.
Are we close to begin this loop again? :)

I propose to make the vmchannel connect to any capable device (serial, nic, usb, IO port, whatever) by adding some indirection. Moreover, at start no device should be "vmchannel-enabled" but the connection could be made dynamically at guest's request, then some of the disadvantages you listed are gone.

Quoted 9 lines

Let's try to stick to the nic solution. It has some advantages over pv
serial:
- Reliable communication if tcp is used
- Migration support for slirp
- No new driver in the guest.
- Might even work for older guests
The disadvantages are:
- Need to 'teach' guest daemons/firewalls not to handle/block the new
nic

The guest could request a vmchannel only after ensuring that the firewall is fixed..

Quoted 2 lines

- Link local addresses for ipv4 are problematic when using on other
nics in parallel

Likewise, the guest could check the address situation beforehand.

Quoted 6 lines

- We should either 1. not use link local on other links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.

So additional nic can do the job and we have several flavours to choose
from.

The solution should be generic enough so that any nic can be connected to vmchannel.

#25 lore raw
Quoted 56 lines · Blue Swirl
Quoted 43 lines · Dor Laor

Carl-Daniel Hailfinger wrote:
On 11.01.2009 08:10, Blue Swirl wrote:

Quoted 5 lines · Jamie Lokier

But we also have to think about how to support newer platforms and newer kernels and this will often mean that we have to make intrusive changes so that the integration makes everyone happy. This does not mean that we cannot support older platforms though, we just have to do it a little differently on the older platforms.

Sure, but don't make it _deliberately_ hard to support older/obscure/can't-compile-a-kernel-module guests by
designing
something that's obviously going to require a totally different mechanism on those other guests. It would make it unnecessarily hard to integrate diverse guests with management apps from different authors if they do adopt the vmchannel mechanism.

I think a serial port device should be universally supported by any OS
and it's portable to many systems. Older OS may accidentally enable forwarding between the trusted nic and other nics, this doesn't happen with serial lines.

I remember the old days of DOS networking where the Kirschbaum-Netz
software provided some sort of routed/forwarded networking between PCs over serial ports. It was a default on choice in many corporate and private "LANs" in Germany at the beginning of the last decade.

Except for machines with that software (which is really hard to get nowadays), my concern should be a non-issue, especially for virtual machines which are unlikely to have such software installed.

Regards,
Carl-Daniel

Quoted 10 lines

Actually vmchannel started as a pv serial implementation. Standard serial
is
a bit low performing and demand an vmexit per byte (maybe it's not that bad
for us).
Moreover, various guest do not support more than 4 serial channels. Since
there
should be several channels and we like to preserve some for console/debug,
it is
not enough.

Quoted 2 lines

There could be similar OS limits for number of nics in the system.

But the number is much higher. It's the limitation of the #pci devices.

Quoted 11 lines
Quoted 10 lines

Originally, vmchannel was a virtio interface with netlink interface to the
kernel.
Then, Anthony asked to change it to a socket interface with new address
family. It
was indeed a logical step. Then, David Miller was reluctant to add such
interface to the
kernel. Instead, he offered the network device solution.
Are we close to begin this loop again? :)

Quoted 7 lines

I propose to make the vmchannel connect to any capable device (serial, nic, usb, IO port, whatever) by adding some indirection. Moreover, at start no device should be "vmchannel-enabled" but the connection could be made dynamically at guest's request, then some of the disadvantages you listed are gone.

It's logical, mainly for the serial.
My only fear is that too many options will confuse the users/developers.

Quoted 11 lines
Quoted 10 lines

Let's try to stick to the nic solution. It has some advantages over pv
serial:
- Reliable communication if tcp is used
- Migration support for slirp
- No new driver in the guest.
- Might even work for older guests
The disadvantages are:
- Need to 'teach' guest daemons/firewalls not to handle/block the new
nic

Quoted 3 lines

The guest could request a vmchannel only after ensuring that the firewall is fixed..

The installer of the guest agent is responsible for punching a hole in the firewall.

Quoted 7 lines
Quoted 4 lines

- Link local addresses for ipv4 are problematic when using on other
nics in parallel

Likewise, the guest could check the address situation beforehand.

It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.

Quoted 5 lines
Quoted 5 lines

- We should either 1. not use link local on other links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.

Quoted 9 lines
Quoted 3 lines

So additional nic can do the job and we have several flavours to choose
from.

The solution should be generic enough so that any nic can be connected to vmchannel.

Agreed.

#26 lore raw
Quoted 2 lines · Dor Laor

The installer of the guest agent is responsible for punching a hole in the firewall.

That's asking a lot from a generic installer. Guests differ enormously in how you do that - including different Linux guests.

Something else you have to do is disable forwarding between the vmchannel NIC and other NICs - even if the other NICs are forwarding enabled to each other. How do you do that on Linux? /proc/sys/net/ipv4/ip_forward is global, not per NIC... How do you do it on other guests?

It's easy to imagine a few simple guest agents written in C that compile easily on any guest unix you might want to run on... except this vmchannel setup would be the only non-portable part, and highly non-portable at that.

Quoted 8 lines

- Link local addresses for ipv4 are problematic when
using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.

No, that's not enough. Even when you have globally unique link-local addresses, you have the problem that NICs configured for link-local IP always have the same subnet, so routing doesn't work.

You could workaround this by using non-standard link-local IP on the vmchannel NIC. Now you're playing more games...

Quoted 11 lines

- We should either 1. not use link local on other
links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.

So additional nic can do the job and we have several
flavours to choose
from.
The solution should be generic enough so that any nic can be connected
to vmchannel.

It sounds "generic" in the sense that you need a custom configuration which depends on the rest of the guest's configuration. Not really "drop in guest vmchannel app and it just works", is it?

If the guest vmchannel app installer looks at other NICs, and picks an IP subnet that the others aren't using, or uses link-local when that's not used on the others... That will work most of the time. But sometimes it will break a working guest some hours after it's installed. What happens if the guests's LAN NIC is using DHCP, so the vmchannel app picks link-local - and then the guests's LAN NIC changes to link-local itself after some hours running? That's not uncommon behaviour nowadays on some networks.

Handling all the cases _reliably_, adapting reactively to network config _changes_ on the other NICs while running, and doing so across many guest types (even just Linux distros and Windows) without having to have custom code for each guest type, is harder than it looks.

On the other hand, using packet sockets and not IP over the vmchannel NIC... (just pick another ethernet type) that would work reliably, but without the convenience of TCP/IP. It would need more support in the guest vmchannel app, and guest root access, but both sound plausible to implement.

-- Jamie

#27 lore raw
Quoted 7 lines · Jamie Lokier
Quoted 2 lines · Dor Laor

The installer of the guest agent is responsible for punching a hole in the firewall.

That's asking a lot from a generic installer. Guests differ enormously in how you do that - including different Linux guests.

Using network for vmchannel has its downsides. This is one of them. Every networking daemon needs some king of configuration and this is well understood by admins. BTW vmchannel will use only outgoing connection and they are usually allowed by firewalls.

Quoted 5 lines

Something else you have to do is disable forwarding between the vmchannel NIC and other NICs - even if the other NICs are forwarding enabled to each other. How do you do that on Linux? /proc/sys/net/ipv4/ip_forward is global, not per NIC... How do you do it on other guests?

No need to do that. Slirp will drop any packet forwarded to it while running in restricted mode.

Quoted 6 lines

It's easy to imagine a few simple guest agents written in C that compile easily on any guest unix you might want to run on... except this vmchannel setup would be the only non-portable part, and highly non-portable at that.

Actually the only nonportable part I see is finding vmchannel network device. After vmchannel device is determined getting IP from a network device is portable between Unixes.

Quoted 9 lines
Quoted 8 lines

- Link local addresses for ipv4 are problematic when
using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.

Quoted 4 lines

No, that's not enough. Even when you have globally unique link-local addresses, you have the problem that NICs configured for link-local IP always have the same subnet, so routing doesn't work.

Most Unixes have SO_BINDTODEVICE to solve this problem. Windows and others will probably have to add host route. But I prefer to use privet subnet outside of link local addresses. One less RFC to worry about.

Quoted 15 lines

You could workaround this by using non-standard link-local IP on the vmchannel NIC. Now you're playing more games...

Quoted 11 lines

- We should either 1. not use link local on other
links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.

So additional nic can do the job and we have several
flavours to choose
from.
The solution should be generic enough so that any nic can be connected
to vmchannel.

Quoted 4 lines

It sounds "generic" in the sense that you need a custom configuration which depends on the rest of the guest's configuration. Not really "drop in guest vmchannel app and it just works", is it?

We all wanted to use something else for vmchannel, but unfortunately we were pushed to a networking solution. I still have PF_VMCHANNEL socket family code, so if you can convince David Miller that network is not good fit for vmchannel go for it :) There are certain restriction applies when you talk to him though: you can't mention virtualization as justification for vmchannel!

Quoted 8 lines

If the guest vmchannel app installer looks at other NICs, and picks an IP subnet that the others aren't using, or uses link-local when that's not used on the others... That will work most of the time. But sometimes it will break a working guest some hours after it's installed. What happens if the guests's LAN NIC is using DHCP, so the vmchannel app picks link-local - and then the guests's LAN NIC changes to link-local itself after some hours running? That's not uncommon behaviour nowadays on some networks.

In my opinion we shouldn't be too smart about choosing of a subnet and leave that to admin (using some reasonable default of cause).

Quoted 6 lines

Handling all the cases _reliably_, adapting reactively to network config _changes_ on the other NICs while running, and doing so across many guest types (even just Linux distros and Windows) without having to have custom code for each guest type, is harder than it looks.

Quoted 6 lines

On the other hand, using packet sockets and not IP over the vmchannel NIC... (just pick another ethernet type) that would work reliably, but without the convenience of TCP/IP. It would need more support in the guest vmchannel app, and guest root access, but both sound plausible to implement.

Firewalls can still filter your packets though.

--
Gleb.

#28 lore raw
Quoted 96 lines · Gleb Natapov
Quoted 9 lines · Jamie Lokier
Quoted 3 lines · Dor Laor

The installer of the guest agent is responsible for punching a hole in the firewall.

That's asking a lot from a generic installer. Guests differ enormously in how you do that - including different Linux guests.

Using network for vmchannel has its downsides. This is one of them. Every networking daemon needs some king of configuration and this is well understood by admins. BTW vmchannel will use only outgoing connection and they are usually allowed by firewalls.

Quoted 6 lines

Something else you have to do is disable forwarding between the vmchannel NIC and other NICs - even if the other NICs are forwarding enabled to each other. How do you do that on Linux? /proc/sys/net/ipv4/ip_forward is global, not per NIC... How do you do it on other guests?

No need to do that. Slirp will drop any packet forwarded to it while running in restricted mode.

Quoted 6 lines

It's easy to imagine a few simple guest agents written in C that compile easily on any guest unix you might want to run on... except this vmchannel setup would be the only non-portable part, and highly non-portable at that.

Actually the only nonportable part I see is finding vmchannel network device. After vmchannel device is determined getting IP from a network device is portable between Unixes.

Quoted 9 lines
Quoted 9 lines

- Link local addresses for ipv4 are problematic when
using on other
nics in parallel
Likewise, the guest could check the address situation beforehand.
It does check (meaning we need to fully implement the link local rfc). The problem is that even if we check that no one is using this guest local link address, another nic can use link local addresses. So a remote host on the LAN of the other nic might chose the same address we are using.

Quoted 5 lines

No, that's not enough. Even when you have globally unique link-local addresses, you have the problem that NICs configured for link-local IP always have the same subnet, so routing doesn't work.

Most Unixes have SO_BINDTODEVICE to solve this problem. Windows and others will probably have to add host route. But I prefer to use privet subnet outside of link local addresses. One less RFC to worry about.

Quoted 16 lines

You could workaround this by using non-standard link-local IP on the vmchannel NIC. Now you're playing more games...

Quoted 5 lines

- We should either 1. not use link local on other
links 2. Use
standard dhcp addresses 3. do
not use tcp/ip for vmchannel communication.

Quoted 7 lines

So additional nic can do the job and we have several
flavours to choose
from.
The solution should be generic enough so that any nic can be connected
to vmchannel.

Quoted 5 lines

It sounds "generic" in the sense that you need a custom configuration which depends on the rest of the guest's configuration. Not really "drop in guest vmchannel app and it just works", is it?

We all wanted to use something else for vmchannel, but unfortunately we were pushed to a networking solution. I still have PF_VMCHANNEL socket family code, so if you can convince David Miller that network is not good fit for vmchannel go for it :) There are certain restriction applies when you talk to him though: you can't mention virtualization as justification for vmchannel!

Quoted 9 lines

If the guest vmchannel app installer looks at other NICs, and picks an IP subnet that the others aren't using, or uses link-local when that's not used on the others... That will work most of the time. But sometimes it will break a working guest some hours after it's installed. What happens if the guests's LAN NIC is using DHCP, so the vmchannel app picks link-local - and then the guests's LAN NIC changes to link-local itself after some hours running? That's not uncommon behaviour nowadays on some networks.

Quoted 16 lines

In my opinion we shouldn't be too smart about choosing of a subnet and leave that to admin (using some reasonable default of cause).

Quoted 12 lines

Handling all the cases _reliably_, adapting reactively to network config _changes_ on the other NICs while running, and doing so across many guest types (even just Linux distros and Windows) without having to have custom code for each guest type, is harder than it looks.

On the other hand, using packet sockets and not IP over the vmchannel NIC... (just pick another ethernet type) that would work reliably, but without the convenience of TCP/IP. It would need more support in the guest vmchannel app, and guest root access, but both sound plausible to implement.

Right. We have 3 options with their pros and cons:
1. Use link local addresses for the vmchannel link ONLY. Do not allow other nics to use it. The upside is there is no new subnet to manage.
btw: anybody know how physical host with multiple nic using link local behaves?
2. Use standard ip range using slirp dhcp for the vmchannel. There is no link local addresses for this nic. The down side is that the admin needs to provide/manage another subnet mask. Also slirp has to be changed in order to allow dynamic replacement of the 'host' IP (although shutdown+boot go around it.)
3. Using packet socket.
The upside - no IP addressing.
The downside - no IP addressing (kidding, mainly tcp reliability). The guest agent/host need to get synchronous acks for every
message.

Quoted 7 lines

Firewalls can still filter your packets though.

--
Gleb.