Skip to contentSkip to conversation

net: delay freeing peer host device

PATCH In review Michael S. Tsirkin 3 participants 6 messages Oct 6, 2010

lore
Share & tools
lore
#0 lore raw

PATCH v4 1/1 net: delay freeing peer host device

With -netdev, virtio devices present offload features to guest, depending on the backend used. Thus, removing host netdev peer while guest is active leads to guest-visible inconsistency and/or crashes.

As a solution, while guest (NIC) peer device exists, we prevent the host peer from being deleted.
This patch does this by adding peer_deleted flag in nic state:
if host device is going away while guest device
is around, set this flag and keep a shell of
the host device around for as long as guest device exists.

The link is put down so all packets will get discarded.

At the moment, management can detect that device deletion is delayed by doing info net. As a next step, we shall add commands that control hotplug/unplug without removing the device, and an event to report that guest has responded to the hotplug event.

---

Changes from v3:
make sure ->cleanup is called for NICs
Changes from v2:
fix crash on repeated netdev_del

 net.c |   49 ++++++++++++++++++++++++++++++++++++++++++-------
 net.h |    1 +
 2 files changed, 43 insertions(+), 7 deletions(-)
net.c +42 -7
index 3d0fde7..ed74c7f 100644
@@ -281,29 +281,64 @@ NICState *qemu_new_nic(NetClientInfo *info,
     return nic;
 }
 
-void qemu_del_vlan_client(VLANClientState *vc)
+static void qemu_cleanup_vlan_client(VLANClientState *vc)
 {
     if (vc->vlan) {
         QTAILQ_REMOVE(&vc->vlan->clients, vc, next);
     } else {
-        if (vc->send_queue) {
-            qemu_del_net_queue(vc->send_queue);
-        }
         QTAILQ_REMOVE(&non_vlan_clients, vc, next);
-        if (vc->peer) {
-            vc->peer->peer = NULL;
-        }
     }
 
     if (vc->info->cleanup) {
         vc->info->cleanup(vc);
     }
+}
 
+static void qemu_free_vlan_client(VLANClientState *vc)
+{
+    if (!vc->vlan) {
+        if (vc->send_queue) {
+            qemu_del_net_queue(vc->send_queue);
+        }
+        if (vc->peer) {
+            vc->peer->peer = NULL;
+        }
+    }
     qemu_free(vc->name);
     qemu_free(vc->model);
     qemu_free(vc);
 }
 
+void qemu_del_vlan_client(VLANClientState *vc)
+{
+    /* If there is a peer NIC, delete and cleanup client, but do not free. */
+    if (!vc->vlan && vc->peer && vc->peer->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc->peer);
+        if (nic->peer_deleted) {
+            return;
+        }
+        nic->peer_deleted = true;
+        /* Let NIC know peer is gone. */
+        vc->peer->link_down = true;
+        if (vc->peer->info->link_status_changed) {
+            vc->peer->info->link_status_changed(vc->peer);
+        }
+        qemu_cleanup_vlan_client(vc);
+        return;
+    }
+
+    /* If this is a peer NIC and peer has already been deleted, free it now. */
+    if (!vc->vlan && vc->peer && vc->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc);
+        if (nic->peer_deleted) {
+            qemu_free_vlan_client(vc->peer);
+        }
+    }
+
+    qemu_cleanup_vlan_client(vc);
+    qemu_free_vlan_client(vc);
+}
+
 VLANClientState *
 qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                               const char *client_str)
net.h +1 -0
index 518cf9c..44c31a9 100644
@@ -72,6 +72,7 @@ typedef struct NICState {
     VLANClientState nc;
     NICConf *conf;
     void *opaque;
+    bool peer_deleted;
 } NICState;
 
 struct VLANState {
Signature · 1 line
1.7.3-rc1
Acked-by on v4 1/1
#1 lore raw
Quoted 14 lines · Michael S. Tsirkin

With -netdev, virtio devices present offload features to guest, depending on the backend used. Thus, removing host netdev peer while guest is active leads to guest-visible inconsistency and/or crashes.

As a solution, while guest (NIC) peer device exists, we prevent the host peer from being deleted.
This patch does this by adding peer_deleted flag in nic state:
if host device is going away while guest device
is around, set this flag and keep a shell of
the host device around for as long as guest device exists.

The link is put down so all packets will get discarded.

Quoted 7 lines

At the moment, management can detect that device deletion is delayed by doing info net. As a next step, we shall add commands that control hotplug/unplug without removing the device, and an event to report that guest has responded to the hotplug event.

Quoted 99 lines

---

Changes from v3:
make sure ->cleanup is called for NICs
Changes from v2:
fix crash on repeated netdev_del

 net.c |   49 ++++++++++++++++++++++++++++++++++++++++++-------
 net.h |    1 +
 2 files changed, 43 insertions(+), 7 deletions(-)
net.c +42 -7
index 3d0fde7..ed74c7f 100644
@@ -281,29 +281,64 @@ NICState *qemu_new_nic(NetClientInfo *info,
     return nic;
 }
 
-void qemu_del_vlan_client(VLANClientState *vc)
+static void qemu_cleanup_vlan_client(VLANClientState *vc)
 {
     if (vc->vlan) {
         QTAILQ_REMOVE(&vc->vlan->clients, vc, next);
     } else {
-        if (vc->send_queue) {
-            qemu_del_net_queue(vc->send_queue);
-        }
         QTAILQ_REMOVE(&non_vlan_clients, vc, next);
-        if (vc->peer) {
-            vc->peer->peer = NULL;
-        }
     }
 
     if (vc->info->cleanup) {
         vc->info->cleanup(vc);
     }
+}
 
+static void qemu_free_vlan_client(VLANClientState *vc)
+{
+    if (!vc->vlan) {
+        if (vc->send_queue) {
+            qemu_del_net_queue(vc->send_queue);
+        }
+        if (vc->peer) {
+            vc->peer->peer = NULL;
+        }
+    }
     qemu_free(vc->name);
     qemu_free(vc->model);
     qemu_free(vc);
 }
 
+void qemu_del_vlan_client(VLANClientState *vc)
+{
+    /* If there is a peer NIC, delete and cleanup client, but do not free. */
+    if (!vc->vlan && vc->peer && vc->peer->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc->peer);
+        if (nic->peer_deleted) {
+            return;
+        }
+        nic->peer_deleted = true;
+        /* Let NIC know peer is gone. */
+        vc->peer->link_down = true;
+        if (vc->peer->info->link_status_changed) {
+            vc->peer->info->link_status_changed(vc->peer);
+        }
+        qemu_cleanup_vlan_client(vc);
+        return;
+    }
+
+    /* If this is a peer NIC and peer has already been deleted, free it now. */
+    if (!vc->vlan && vc->peer && vc->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc);
+        if (nic->peer_deleted) {
+            qemu_free_vlan_client(vc->peer);
+        }
+    }
+
+    qemu_cleanup_vlan_client(vc);
+    qemu_free_vlan_client(vc);
+}
+
 VLANClientState *
 qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                               const char *client_str)
net.h +1 -0
index 518cf9c..44c31a9 100644
@@ -72,6 +72,7 @@ typedef struct NICState {
     VLANClientState nc;
     NICConf *conf;
     void *opaque;
+    bool peer_deleted;
 } NICState;
 
 struct VLANState {
#2 lore raw
In reply to Michael S. Tsirkin
Quoted 14 lines · Michael S. Tsirkin

With -netdev, virtio devices present offload features to guest, depending on the backend used. Thus, removing host netdev peer while guest is active leads to guest-visible inconsistency and/or crashes.

As a solution, while guest (NIC) peer device exists, we prevent the host peer from being deleted.
This patch does this by adding peer_deleted flag in nic state:
if host device is going away while guest device
is around, set this flag and keep a shell of
the host device around for as long as guest device exists.

The link is put down so all packets will get discarded.

Quoted 8 lines

At the moment, management can detect that device deletion is delayed by doing info net. As a next step, we shall add commands that control hotplug/unplug without removing the device, and an event to report that guest has responded to the hotplug event.

I don't think this is a very good idea. Making netdev_del conditionally succeed is going to break management tools and is very non-intuitive.

Regards,

Anthony Liguori

Quoted 100 lines

---

Changes from v3:

    make sure ->cleanup is called for NICs

Changes from v2:
fix crash on repeated netdev_del

  net.c |   49 ++++++++++++++++++++++++++++++++++++++++++-------
  net.h |    1 +
  2 files changed, 43 insertions(+), 7 deletions(-)
net.c +42 -7
index 3d0fde7..ed74c7f 100644
@@ -281,29 +281,64 @@ NICState *qemu_new_nic(NetClientInfo *info,
      return nic;
  }
 
-void qemu_del_vlan_client(VLANClientState *vc)
+static void qemu_cleanup_vlan_client(VLANClientState *vc)
  {
      if (vc->vlan) {
          QTAILQ_REMOVE(&vc->vlan->clients, vc, next);
      } else {
-        if (vc->send_queue) {
-            qemu_del_net_queue(vc->send_queue);
-        }
          QTAILQ_REMOVE(&non_vlan_clients, vc, next);
-        if (vc->peer) {
-            vc->peer->peer = NULL;
-        }
      }
 
      if (vc->info->cleanup) {
          vc->info->cleanup(vc);
      }
+}
 
+static void qemu_free_vlan_client(VLANClientState *vc)
+{
+    if (!vc->vlan) {
+        if (vc->send_queue) {
+            qemu_del_net_queue(vc->send_queue);
+        }
+        if (vc->peer) {
+            vc->peer->peer = NULL;
+        }
+    }
      qemu_free(vc->name);
      qemu_free(vc->model);
      qemu_free(vc);
  }
 
+void qemu_del_vlan_client(VLANClientState *vc)
+{
+    /* If there is a peer NIC, delete and cleanup client, but do not free. */
+    if (!vc->vlan&&  vc->peer&&  vc->peer->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc->peer);
+        if (nic->peer_deleted) {
+            return;
+        }
+        nic->peer_deleted = true;
+        /* Let NIC know peer is gone. */
+        vc->peer->link_down = true;
+        if (vc->peer->info->link_status_changed) {
+            vc->peer->info->link_status_changed(vc->peer);
+        }
+        qemu_cleanup_vlan_client(vc);
+        return;
+    }
+
+    /* If this is a peer NIC and peer has already been deleted, free it now. */
+    if (!vc->vlan&&  vc->peer&&  vc->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc);
+        if (nic->peer_deleted) {
+            qemu_free_vlan_client(vc->peer);
+        }
+    }
+
+    qemu_cleanup_vlan_client(vc);
+    qemu_free_vlan_client(vc);
+}
+
  VLANClientState *
  qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                                const char *client_str)
net.h +1 -0
index 518cf9c..44c31a9 100644
@@ -72,6 +72,7 @@ typedef struct NICState {
      VLANClientState nc;
      NICConf *conf;
      void *opaque;
+    bool peer_deleted;
  } NICState;
 
  struct VLANState {
#3 lore raw
Quoted 23 lines · Anthony Liguori
Quoted 14 lines · Michael S. Tsirkin

With -netdev, virtio devices present offload features to guest, depending on the backend used. Thus, removing host netdev peer while guest is active leads to guest-visible inconsistency and/or crashes.

As a solution, while guest (NIC) peer device exists, we prevent the host peer from being deleted.
This patch does this by adding peer_deleted flag in nic state:
if host device is going away while guest device
is around, set this flag and keep a shell of
the host device around for as long as guest device exists.

The link is put down so all packets will get discarded.

Quoted 7 lines

At the moment, management can detect that device deletion is delayed by doing info net. As a next step, we shall add commands that control hotplug/unplug without removing the device, and an event to report that guest has responded to the hotplug event.

Quoted 7 lines

I don't think this is a very good idea. Making netdev_del conditionally succeed is going to break management tools and is very non-intuitive.

Regards,

Anthony Liguori

But this is not what this does. With this patch netdev_del always succeeds, and clears the device state leaving just an empty shell around. The shell goes away together with the NIC.

Quoted 99 lines
Quoted 99 lines

---

Changes from v3:
make sure ->cleanup is called for NICs
Changes from v2:
fix crash on repeated netdev_del

 net.c |   49 ++++++++++++++++++++++++++++++++++++++++++-------
 net.h |    1 +
 2 files changed, 43 insertions(+), 7 deletions(-)
net.c +42 -7
index 3d0fde7..ed74c7f 100644
@@ -281,29 +281,64 @@ NICState *qemu_new_nic(NetClientInfo *info,
     return nic;
 }
 
-void qemu_del_vlan_client(VLANClientState *vc)
+static void qemu_cleanup_vlan_client(VLANClientState *vc)
 {
     if (vc->vlan) {
         QTAILQ_REMOVE(&vc->vlan->clients, vc, next);
     } else {
-        if (vc->send_queue) {
-            qemu_del_net_queue(vc->send_queue);
-        }
         QTAILQ_REMOVE(&non_vlan_clients, vc, next);
-        if (vc->peer) {
-            vc->peer->peer = NULL;
-        }
     }
 
     if (vc->info->cleanup) {
         vc->info->cleanup(vc);
     }
+}
 
+static void qemu_free_vlan_client(VLANClientState *vc)
+{
+    if (!vc->vlan) {
+        if (vc->send_queue) {
+            qemu_del_net_queue(vc->send_queue);
+        }
+        if (vc->peer) {
+            vc->peer->peer = NULL;
+        }
+    }
     qemu_free(vc->name);
     qemu_free(vc->model);
     qemu_free(vc);
 }
 
+void qemu_del_vlan_client(VLANClientState *vc)
+{
+    /* If there is a peer NIC, delete and cleanup client, but do not free. */
+    if (!vc->vlan&&  vc->peer&&  vc->peer->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc->peer);
+        if (nic->peer_deleted) {
+            return;
+        }
+        nic->peer_deleted = true;
+        /* Let NIC know peer is gone. */
+        vc->peer->link_down = true;
+        if (vc->peer->info->link_status_changed) {
+            vc->peer->info->link_status_changed(vc->peer);
+        }
+        qemu_cleanup_vlan_client(vc);
+        return;
+    }
+
+    /* If this is a peer NIC and peer has already been deleted, free it now. */
+    if (!vc->vlan&&  vc->peer&&  vc->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc);
+        if (nic->peer_deleted) {
+            qemu_free_vlan_client(vc->peer);
+        }
+    }
+
+    qemu_cleanup_vlan_client(vc);
+    qemu_free_vlan_client(vc);
+}
+
 VLANClientState *
 qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                               const char *client_str)
net.h +1 -0
index 518cf9c..44c31a9 100644
@@ -72,6 +72,7 @@ typedef struct NICState {
     VLANClientState nc;
     NICConf *conf;
     void *opaque;
+    bool peer_deleted;
 } NICState;
 
 struct VLANState {
#4 lore raw
Quoted 30 lines · Michael S. Tsirkin
Quoted 28 lines · Anthony Liguori
Quoted 14 lines · Michael S. Tsirkin

With -netdev, virtio devices present offload features to guest, depending on the backend used. Thus, removing host netdev peer while guest is active leads to guest-visible inconsistency and/or crashes.

As a solution, while guest (NIC) peer device exists, we prevent the host peer from being deleted.
This patch does this by adding peer_deleted flag in nic state:
if host device is going away while guest device
is around, set this flag and keep a shell of
the host device around for as long as guest device exists.

The link is put down so all packets will get discarded.

Quoted 8 lines

At the moment, management can detect that device deletion is delayed by doing info net. As a next step, we shall add commands that control hotplug/unplug without removing the device, and an event to report that guest has responded to the hotplug event.

I don't think this is a very good idea. Making netdev_del conditionally succeed is going to break management tools and is very non-intuitive.

Quoted 8 lines
Quoted 4 lines

Regards,

Anthony Liguori

But this is not what this does. With this patch netdev_del always succeeds, and clears the device state leaving just an empty shell around. The shell goes away together with the NIC.

Sorry, I confused myself. I'm a little confused by the statement:

"At the moment, management can detect that device deletion is delayed by doing info net."

Is this still the case since you remove the netdev from the list?

Regards,

Anthony Liguori

Quoted 100 lines
Quoted 100 lines
Quoted 100 lines

---

Changes from v3:

    make sure ->cleanup is called for NICs

Changes from v2:
fix crash on repeated netdev_del

  net.c |   49 ++++++++++++++++++++++++++++++++++++++++++-------
  net.h |    1 +
  2 files changed, 43 insertions(+), 7 deletions(-)
net.c +42 -7
index 3d0fde7..ed74c7f 100644
@@ -281,29 +281,64 @@ NICState *qemu_new_nic(NetClientInfo *info,
      return nic;
  }
 
-void qemu_del_vlan_client(VLANClientState *vc)
+static void qemu_cleanup_vlan_client(VLANClientState *vc)
  {
      if (vc->vlan) {
          QTAILQ_REMOVE(&vc->vlan->clients, vc, next);
      } else {
-        if (vc->send_queue) {
-            qemu_del_net_queue(vc->send_queue);
-        }
          QTAILQ_REMOVE(&non_vlan_clients, vc, next);
-        if (vc->peer) {
-            vc->peer->peer = NULL;
-        }
      }
 
      if (vc->info->cleanup) {
          vc->info->cleanup(vc);
      }
+}
 
+static void qemu_free_vlan_client(VLANClientState *vc)
+{
+    if (!vc->vlan) {
+        if (vc->send_queue) {
+            qemu_del_net_queue(vc->send_queue);
+        }
+        if (vc->peer) {
+            vc->peer->peer = NULL;
+        }
+    }
      qemu_free(vc->name);
      qemu_free(vc->model);
      qemu_free(vc);
  }
 
+void qemu_del_vlan_client(VLANClientState *vc)
+{
+    /* If there is a peer NIC, delete and cleanup client, but do not free. */
+    if (!vc->vlan&&   vc->peer&&   vc->peer->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc->peer);
+        if (nic->peer_deleted) {
+            return;
+        }
+        nic->peer_deleted = true;
+        /* Let NIC know peer is gone. */
+        vc->peer->link_down = true;
+        if (vc->peer->info->link_status_changed) {
+            vc->peer->info->link_status_changed(vc->peer);
+        }
+        qemu_cleanup_vlan_client(vc);
+        return;
+    }
+
+    /* If this is a peer NIC and peer has already been deleted, free it now. */
+    if (!vc->vlan&&   vc->peer&&   vc->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc);
+        if (nic->peer_deleted) {
+            qemu_free_vlan_client(vc->peer);
+        }
+    }
+
+    qemu_cleanup_vlan_client(vc);
+    qemu_free_vlan_client(vc);
+}
+
  VLANClientState *
  qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                                const char *client_str)
net.h +1 -0
index 518cf9c..44c31a9 100644
@@ -72,6 +72,7 @@ typedef struct NICState {
      VLANClientState nc;
      NICConf *conf;
      void *opaque;
+    bool peer_deleted;
  } NICState;
 
  struct VLANState {
#5 lore raw
Quoted 37 lines · Anthony Liguori
Quoted 27 lines · Michael S. Tsirkin
Quoted 26 lines · Anthony Liguori
Quoted 14 lines · Michael S. Tsirkin

With -netdev, virtio devices present offload features to guest, depending on the backend used. Thus, removing host netdev peer while guest is active leads to guest-visible inconsistency and/or crashes.

As a solution, while guest (NIC) peer device exists, we prevent the host peer from being deleted.
This patch does this by adding peer_deleted flag in nic state:
if host device is going away while guest device
is around, set this flag and keep a shell of
the host device around for as long as guest device exists.

The link is put down so all packets will get discarded.

Quoted 7 lines

At the moment, management can detect that device deletion is delayed by doing info net. As a next step, we shall add commands that control hotplug/unplug without removing the device, and an event to report that guest has responded to the hotplug event.

I don't think this is a very good idea. Making netdev_del conditionally succeed is going to break management tools and is very non-intuitive.

Quoted 6 lines
Quoted 3 lines

Regards,

Anthony Liguori

But this is not what this does. With this patch netdev_del always succeeds, and clears the device state leaving just an empty shell around. The shell goes away together with the NIC.

Sorry, I confused myself. I'm a little confused by the statement:

Quoted 7 lines

"At the moment, management can detect that device deletion is delayed by doing info net."

Is this still the case since you remove the netdev from the list?

Regards,

Anthony Liguori

The device in question here is the PCI device removed with device_del. The unplug there is asynchronous and I did not change this.

Quoted 100 lines
Quoted 100 lines
Quoted 100 lines
Quoted 100 lines

---

Changes from v3:
make sure ->cleanup is called for NICs
Changes from v2:
fix crash on repeated netdev_del

 net.c |   49 ++++++++++++++++++++++++++++++++++++++++++-------
 net.h |    1 +
 2 files changed, 43 insertions(+), 7 deletions(-)
net.c +42 -7
index 3d0fde7..ed74c7f 100644
@@ -281,29 +281,64 @@ NICState *qemu_new_nic(NetClientInfo *info,
     return nic;
 }
 
-void qemu_del_vlan_client(VLANClientState *vc)
+static void qemu_cleanup_vlan_client(VLANClientState *vc)
 {
     if (vc->vlan) {
         QTAILQ_REMOVE(&vc->vlan->clients, vc, next);
     } else {
-        if (vc->send_queue) {
-            qemu_del_net_queue(vc->send_queue);
-        }
         QTAILQ_REMOVE(&non_vlan_clients, vc, next);
-        if (vc->peer) {
-            vc->peer->peer = NULL;
-        }
     }
 
     if (vc->info->cleanup) {
         vc->info->cleanup(vc);
     }
+}
 
+static void qemu_free_vlan_client(VLANClientState *vc)
+{
+    if (!vc->vlan) {
+        if (vc->send_queue) {
+            qemu_del_net_queue(vc->send_queue);
+        }
+        if (vc->peer) {
+            vc->peer->peer = NULL;
+        }
+    }
     qemu_free(vc->name);
     qemu_free(vc->model);
     qemu_free(vc);
 }
 
+void qemu_del_vlan_client(VLANClientState *vc)
+{
+    /* If there is a peer NIC, delete and cleanup client, but do not free. */
+    if (!vc->vlan&&   vc->peer&&   vc->peer->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc->peer);
+        if (nic->peer_deleted) {
+            return;
+        }
+        nic->peer_deleted = true;
+        /* Let NIC know peer is gone. */
+        vc->peer->link_down = true;
+        if (vc->peer->info->link_status_changed) {
+            vc->peer->info->link_status_changed(vc->peer);
+        }
+        qemu_cleanup_vlan_client(vc);
+        return;
+    }
+
+    /* If this is a peer NIC and peer has already been deleted, free it now. */
+    if (!vc->vlan&&   vc->peer&&   vc->info->type == NET_CLIENT_TYPE_NIC) {
+        NICState *nic = DO_UPCAST(NICState, nc, vc);
+        if (nic->peer_deleted) {
+            qemu_free_vlan_client(vc->peer);
+        }
+    }
+
+    qemu_cleanup_vlan_client(vc);
+    qemu_free_vlan_client(vc);
+}
+
 VLANClientState *
 qemu_find_vlan_client_by_name(Monitor *mon, int vlan_id,
                               const char *client_str)
net.h +1 -0
index 518cf9c..44c31a9 100644
@@ -72,6 +72,7 @@ typedef struct NICState {
     VLANClientState nc;
     NICConf *conf;
     void *opaque;
+    bool peer_deleted;
 } NICState;
 
 struct VLANState {