Support live migration with mlx5_vdpa
RFC PATCH 3/3 vdpa/mlx5: Disable VLAN support to support live migration
In reply to Eli CohenCurrent qemu code does not support live migration for devices supporting VLAN. Disable it.
Note: this patch is provided just to enable testing with current qemu.
--- drivers/vdpa/mlx5/net/mlx5_vnet.c | 1 - 1 file changed, 1 deletion(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -1
@@ -2172,7 +2172,6 @@ static u64 get_supported_features(struct mlx5_core_dev *mdev) mlx_vdpa_features |=BIT_ULL (VIRTIO_NET_F_MQ ); mlx_vdpa_features |=BIT_ULL (VIRTIO_NET_F_STATUS ); mlx_vdpa_features |=BIT_ULL (VIRTIO_NET_F_MTU ); - mlx_vdpa_features |=BIT_ULL (VIRTIO_NET_F_CTRL_VLAN );return mlx_vdpa_features; }
Signature · 1 line
2.35.1
cover Support live migration with mlx5_vdpa
Hi all,
I am sending this patch set with adressing some of the comments.
Please note that I will be on vacation for the next three weeks so any comments will be addressed after I return.
Eugenio and myself were able to actually see live migation working over mlx5_vdpa so that's a good sign too :-)
Following patchset supports the suspend callback to suspend VQs to allow sshdow VQs to kick in. It also adds address space support also required for live migration. We use two groups, one for the control virtqueue and one for the data virtqueues.
Finally there is a patch that disables VLAN support since it is not supported currently.
The aim of this patchset is to enable development/testing of live migration over a real hardware device.
The below two patches are also required to be applied but I did not include them in the series since they were sent separately.
https://lore.kernel.org/all/[email protected]/T/
https://lore.kernel.org/all/[email protected]/T/
v0 -> v1:
1. Fix a few typos
2. handle memory map updates for all groups with a given asid.
Eli Cohen (3):
vdpa/mlx5: Implement susupend virtqueue callback vdpa/mlx5: Support different address spaces for control and data vdpa/mlx5: Disable VLAN support to support live migration
drivers/vdpa/mlx5/core/mlx5_vdpa.h | 11 ++ drivers/vdpa/mlx5/net/mlx5_vnet.c | 157 ++++++++++++++++++++++++++--- include/linux/mlx5/mlx5_ifc_vdpa.h | 8 ++ 3 files changed, 163 insertions(+), 13 deletions(-)
Signature · 1 line
2.35.1
RFC PATCH 2/3 vdpa/mlx5: Support different address spaces for control and data
Partition virtqueues to two different address spaces: one for control virtqueue which is implemented in software, and one for data virtqueues.
Based-on: <[email protected]>
--- drivers/vdpa/mlx5/core/mlx5_vdpa.h | 11 ++++ drivers/vdpa/mlx5/net/mlx5_vnet.c | 88 ++++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 11 deletions(-)
drivers/vdpa/mlx5/core/mlx5_vdpa.h +11 -0
@@ -70,6 +70,16 @@ struct mlx5_vdpa_wq_ent {struct mlx5_vdpa_dev *mvdev; }; +enum { +MLX5_VDPA_DATAVQ_GROUP , +MLX5_VDPA_CVQ_GROUP , +MLX5_VDPA_NUMVQ_GROUPS +}; + +enum { +MLX5_VDPA_NUM_AS =MLX5_VDPA_NUMVQ_GROUPS +}; +struct mlx5_vdpa_dev {struct vdpa_device vdev;struct mlx5_core_dev *mdev;
@@ -85,6 +95,7 @@ struct mlx5_vdpa_dev {struct mlx5_vdpa_mr mr;struct mlx5_control_vq cvq;struct workqueue_struct *wq; +unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS ]; };int mlx5_vdpa_alloc_pd (struct mlx5_vdpa_dev *dev,u32 *pdn,u16 uid);
drivers/vdpa/mlx5/net/mlx5_vnet.c +77 -11
144 lines not on this page · raw message
Signature · 1 line
2.35.1
RFC PATCH 1/3 vdpa/mlx5: Implement susupend virtqueue callback
In reply to Eli CohenImplement the suspend callback allowing to suspend the virtqueues so they stop processing descriptors. This is required to allow to query a consistent state of the virtqueue while live migration is taking place.
--- drivers/vdpa/mlx5/net/mlx5_vnet.c | 68 +++++++++++++++++++++++++++++- include/linux/mlx5/mlx5_ifc_vdpa.h | 8 ++++ 2 files changed, 75 insertions(+), 1 deletion(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +67 -1
@@ -895,6 +895,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueif (err) goto err_cmd; + mvq->fw_state =MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT ;kfree (in); mvq->virtq_id =MLX5_GET (general_obj_out_cmd_hdr, out, obj_id);
@@ -922,6 +923,7 @@ static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqmlx5_vdpa_warn (&ndev->mvdev,"destroy virtqueue 0x%x\n" , mvq->virtq_id);return ; } + mvq->fw_state =MLX5_VIRTIO_NET_Q_OBJECT_NONE ;umems_destroy (ndev, mvq); }
@@ -1121,6 +1123,20 @@ static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueureturn err; } +static bool is_valid_state_change (int oldstate,int newstate) +{ +switch (oldstate) { +case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT : +return newstate ==MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY ; +case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY : +return newstate ==MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND ; +case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND : +case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR : +default : +return false; + } +} +static int modify_virtqueue (struct mlx5_vdpa_net *ndev,struct mlx5_vdpa_virtqueue *mvq,int state) {int inlen =MLX5_ST_SZ_BYTES (modify_virtio_net_q_in);
@@ -1130,6 +1146,12 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtquevoid *in;int err; +if (mvq->fw_state ==MLX5_VIRTIO_NET_Q_OBJECT_NONE ) +return 0 ; + +if (!is_valid_state_change (mvq->fw_state, state)) +return -EINVAL ; + in =kzalloc (inlen,GFP_KERNEL );if (!in)return -ENOMEM ;
@@ -1991,6 +2013,7 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool readystruct mlx5_vdpa_dev *mvdev =to_mvdev (vdev);struct mlx5_vdpa_net *ndev =to_mlx5_vdpa_ndev (mvdev);struct mlx5_vdpa_virtqueue *mvq; +int err;if (!mvdev->actual_features)return ;
@@ -2004,8 +2027,16 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready } mvq = &ndev->vqs[idx]; -if (!ready) +if (!ready) {suspend_vq (ndev, mvq); + }else { + err =modify_virtqueue (ndev, mvq,MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY ); +if (err) { +mlx5_vdpa_warn (mvdev,"modify VQ %d to ready failed (%d)\n" , idx, err); + ready = false; + } + } + mvq->ready = ready; }
@@ -2732,6 +2763,39 @@ static int mlx5_vdpa_get_vendor_vq_stats(struct vdpa_device *vdev, u16 idx,return err; } +static void mlx5_vdpa_cvq_suspend (struct mlx5_vdpa_dev *mvdev,bool suspend) +{ +struct mlx5_control_vq *cvq; + +if (!(mvdev->actual_features &BIT_ULL (VIRTIO_NET_F_CTRL_VQ ))) +return ; + + cvq = &mvdev->cvq; + cvq->ready = !suspend; +} + +static int mlx5_vdpa_suspend (struct vdpa_device *vdev,bool suspend) +{ +struct mlx5_vdpa_dev *mvdev =to_mvdev (vdev); +struct mlx5_vdpa_net *ndev =to_mlx5_vdpa_ndev (mvdev); +struct mlx5_vdpa_virtqueue *mvq; +int i; + +if (!suspend) { +mlx5_vdpa_warn (mvdev,"Resume of virtqueues is not supported\n" ); +return -EOPNOTSUPP ; + } + +down_write (&ndev->reslock); +for (i =0 ; i < ndev->cur_num_vqs; i++) { + mvq = &ndev->vqs[i]; +suspend_vq (ndev, mvq); + } +mlx5_vdpa_cvq_suspend (mvdev, suspend); +up_write (&ndev->reslock); +return 0 ; +} +static const struct vdpa_config_ops mlx5_vdpa_ops = { .set_vq_address = mlx5_vdpa_set_vq_address, .set_vq_num = mlx5_vdpa_set_vq_num,
@@ -2762,6 +2826,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = { .get_generation = mlx5_vdpa_get_generation, .set_map = mlx5_vdpa_set_map, .free = mlx5_vdpa_free, + .suspend = mlx5_vdpa_suspend, };static int query_mtu (struct mlx5_core_dev *mdev,u16 *mtu)
@@ -2827,6 +2892,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev) mvq->index = i; mvq->ndev = ndev; mvq->fwqp.fw = true; + mvq->fw_state =MLX5_VIRTIO_NET_Q_OBJECT_NONE ; }for (; i < ndev->mvdev.max_vqs; i++) { mvq = &ndev->vqs[i];
include/linux/mlx5/mlx5_ifc_vdpa.h +8 -0
@@ -150,6 +150,14 @@ enum { MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR = 0x3, }; +/* This indicates that the object was not created or has already +* been desroyed. It is very safe to assume that this object will never +* have so many states +*/ +enum { +MLX5_VIRTIO_NET_Q_OBJECT_NONE =0xffffffff +}; +enum {MLX5_RQTC_LIST_Q_TYPE_RQ =0x0 ,MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q =0x1 ,
Signature · 1 line
2.35.1