bugfix some driver issues
Longfang Liu
maintainerv5 cover bugfix some driver issues
As the test scenarios for the live migration function become more and more extensive. Some previously undiscovered driver issues were found.
Update and fix through this patchset.
Change v4 -> v5
Update version matching strategy
Change v3 -> v4
Modify version matching scheme
Change v2 -> v3
Modify the magic digital field segment
Change v1 -> v2
Add fixes line for patch comment
Longfang Liu (5):
hisi_acc_vfio_pci: fix XQE dma address error
hisi_acc_vfio_pci: add eq and aeq interruption restore
hisi_acc_vfio_pci: bugfix cache write-back issue
hisi_acc_vfio_pci: bugfix the problem of uninstalling driver
hisi_acc_vfio_pci: bugfix live migration function without VF device
driver
.../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 97 +++++++++++++++---- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h | 14 ++- 2 files changed, 89 insertions(+), 22 deletions(-)
Signature · 1 line
2.24.0
Longfang Liu
maintainerPATCH v5 1/5 hisi_acc_vfio_pci: fix XQE dma address error
The dma addresses of EQE and AEQE are wrong after migration and results in guest kernel-mode encryption services failure. Comparing the definition of hardware registers, we found that there was an error when the data read from the register was combined into an address. Therefore, the address combination sequence needs to be corrected.
Even after fixing the above problem, we still have an issue where the Guest from an old kernel can get migrated to new kernel and may result in wrong data.
In order to ensure that the address is correct after migration, if an old magic number is detected, the dma address needs to be updated.
--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 41 ++++++++++++++++--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h | 14 ++++++- 2 files changed, 47 insertions(+), 8 deletions(-)
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +35 -6
@@ -350,6 +350,32 @@ static int vf_qm_func_stop(struct hisi_qm *qm)return hisi_qm_mb (qm,QM_MB_CMD_PAUSE_QM ,0 ,0 ,0 ); } +static int vf_qm_version_check (struct acc_vf_data *vf_data,struct device *dev) +{ +switch (vf_data->acc_magic) { +case ACC_DEV_MAGIC_V2 : +if (vf_data->major_ver !=ACC_DRV_MAJOR_VER ) { +dev_info (dev,"migration driver version<%u.%u> not match!\n" , + vf_data->major_ver, vf_data->minor_ver); +return -EINVAL ; + } +break ; +case ACC_DEV_MAGIC_V1 : +/* Correct dma address */ + vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH ]; + vf_data->eqe_dma <<=QM_XQC_ADDR_OFFSET ; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW ]; + vf_data->aeqe_dma = vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH ]; + vf_data->aeqe_dma <<=QM_XQC_ADDR_OFFSET ; + vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW ]; +break ; +default : +return -EINVAL ; + } + +return 0 ; +} +static int vf_qm_check_match (struct hisi_acc_vf_core_device *hisi_acc_vdev,struct hisi_acc_vf_migration_file *migf) {
@@ -363,7 +389,8 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,if (migf->total_length <QM_MATCH_SIZE || hisi_acc_vdev->match_done)return 0 ; -if (vf_data->acc_magic !=ACC_DEV_MAGIC ) { + ret =vf_qm_version_check (vf_data, dev); +if (ret) {dev_err (dev,"failed to match ACC_DEV_MAGIC\n" );return -EINVAL ; }
@@ -418,7 +445,9 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,int vf_id = hisi_acc_vdev->vf_id;int ret; - vf_data->acc_magic =ACC_DEV_MAGIC ; + vf_data->acc_magic =ACC_DEV_MAGIC_V2 ; + vf_data->major_ver =ACC_DRV_MAJOR_VER ; + vf_data->minor_ver =ACC_DRV_MINOR_VER ;/* Save device id */ vf_data->dev_id = hisi_acc_vdev->vf_dev->device;
@@ -496,12 +525,12 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, struct acc_vf_data *vf_data)return -EINVAL ;/* Every reg is 32 bit, the dma address is 64 bit. */ - vf_data->eqe_dma = vf_data->qm_eqc_dw[1 ]; + vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH ]; vf_data->eqe_dma <<=QM_XQC_ADDR_OFFSET ; - vf_data->eqe_dma |= vf_data->qm_eqc_dw[0 ]; - vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1 ]; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW ]; + vf_data->aeqe_dma = vf_data->qm_aeqc_dw[QM_XQC_ADDR_HIGH ]; vf_data->aeqe_dma <<=QM_XQC_ADDR_OFFSET ; - vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0 ]; + vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW ];/* Through SQC_BT/CQC_BT to get sqc and cqc address */ ret =qm_get_sqc (vf_qm, &vf_data->sqc_dma);
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +12 -2
@@ -39,6 +39,9 @@#define QM_REG_ADDR_OFFSET 0x0004#define QM_XQC_ADDR_OFFSET 32U +#define QM_XQC_ADDR_LOW 0x1 +#define QM_XQC_ADDR_HIGH 0x2 +#define QM_VF_AEQ_INT_MASK 0x0004#define QM_VF_EQ_INT_MASK 0x000c#define QM_IFC_INT_SOURCE_V 0x0020
@@ -50,10 +53,15 @@#define QM_EQC_DW0 0X8000#define QM_AEQC_DW0 0X8020 +#define ACC_DRV_MAJOR_VER 1 +#define ACC_DRV_MINOR_VER 0 + +#define ACC_DEV_MAGIC_V1 0XCDCDCDCDFEEDAACC +#define ACC_DEV_MAGIC_V2 0xAACCFEEDDECADEDE +struct acc_vf_data {#define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)/* QM match information */ -#define ACC_DEV_MAGIC 0XCDCDCDCDFEEDAACCu64 acc_magic;u32 qp_num;u32 dev_id;
@@ -61,7 +69,9 @@ struct acc_vf_data {u32 qp_base;u32 vf_qm_state;/* QM reserved match information */ -u32 qm_rsv_state[3 ]; +u16 major_ver; +u16 minor_ver; +u32 qm_rsv_state[2 ];/* QM RW regs */ u32 aeq_int_mask;
Signature · 1 line
2.24.0
RE: [PATCH v5 1/5] hisi_acc_vfio_pci: fix XQE dma address error
Quoted 23 lines
-----Original Message-----
From: liulongfang <[email protected]>
Sent: Thursday, March 13, 2025 7:20 AM
To: [email protected]; [email protected]; Shameerali Kolothum Thodi <[email protected]>; Jonathan Cameron <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; liulongfang <[email protected]>
Subject: [PATCH v5 1/5]hisi_acc_vfio_pci: fix XQE dma address errorThe dma addresses of EQE and AEQE are wrong after migration and results in guest kernel-mode encryption services failure. Comparing the definition of hardware registers, we found that there was an error when the data read from the register was combined into an address. Therefore, the address combination sequence needs to be corrected.
Even after fixing the above problem, we still have an issue where the Guest from an old kernel can get migrated to new kernel and may result in wrong data.
Quoted 7 lines
In order to ensure that the address is correct after migration, if an old magic number is detected, the dma address needs to be updated.
Fixes b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO livemigration")
Signed-off-by Longfang Liu <[email protected]>
LGTM,
Thanks,
Shameer
Quoted 139 lines
--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 41 ++++++++++++++++--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h | 14 ++++++- 2 files changed, 47 insertions(+), 8 deletions(-)drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +15
-0@@ -350,6 +350,32 @@ static int vf_qm_func_stop(struct hisi_qm *qm)return hisi_qm_mb (qm,QM_MB_CMD_PAUSE_QM ,0 ,0 ,0 ); } +static int vf_qm_version_check (struct acc_vf_data *vf_data,struct device *dev) +{ +switch (vf_data->acc_magic) { +case ACC_DEV_MAGIC_V2 : +if (vf_data->major_ver !=ACC_DRV_MAJOR_VER ) { +dev_info (dev,"migration driver version<%u.%u> not match!\n", + vf_data->major_ver, vf_data->minor_ver); +return -EINVAL ; + } +break ; +case ACC_DEV_MAGIC_V1 : +/* Correct dma address */ + vf_data->eqe_dma = vf_data- >qm_eqc_dw[QM_XQC_ADDR_HIGH]; + vf_data->eqe_dma <<=QM_XQC_ADDR_OFFSET ; + vf_data->eqe_dma |= vf_data-Quoted 1 line
qm_eqc_dw[QM_XQC_ADDR_LOW];
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-0+ vf_data->aeqe_dma = vf_data-Quoted 1 line
qm_aeqc_dw[QM_XQC_ADDR_HIGH];
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +2
-0+ vf_data->aeqe_dma <<=QM_XQC_ADDR_OFFSET ; + vf_data->aeqe_dma |= vf_data-Quoted 1 line
qm_aeqc_dw[QM_XQC_ADDR_LOW];
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +8
-0+break ; +default : +return -EINVAL ; + } + +return 0 ; +} + static int vf_qm_check_match(struct hisi_acc_vf_core_device
*hisi_acc_vdev,drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +2
-1struct hisi_acc_vf_migration_file *migf) {@@ -363,7 +389,8 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev- >match_done)return 0 ; -if (vf_data->acc_magic !=ACC_DEV_MAGIC ) { + ret =vf_qm_version_check (vf_data, dev); +if (ret) {dev_err (dev,"failed to match ACC_DEV_MAGIC\n" );drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +7
-5return -EINVAL ; }@@ -418,7 +445,9 @@ static int vf_qm_get_match_data(structhisi_acc_vf_core_device *hisi_acc_vdev, int vf_id = hisi_acc_vdev->vf_id;int ret; - vf_data->acc_magic =ACC_DEV_MAGIC ; + vf_data->acc_magic =ACC_DEV_MAGIC_V2 ; + vf_data->major_ver =ACC_DRV_MAJOR_VER ; + vf_data->minor_ver =ACC_DRV_MINOR_VER ;/* Save device id */ vf_data->dev_id = hisi_acc_vdev->vf_dev->device;@@ -496,12 +525,12 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, struct acc_vf_data *vf_data)return -EINVAL ;/* Every reg is 32 bit, the dma address is 64 bit. */ - vf_data->eqe_dma = vf_data->qm_eqc_dw[1 ]; + vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH ]; vf_data->eqe_dma <<=QM_XQC_ADDR_OFFSET ; - vf_data->eqe_dma |= vf_data->qm_eqc_dw[0 ]; - vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1 ]; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW ]; + vf_data->aeqe_dma = vf_data- >qm_aeqc_dw[QM_XQC_ADDR_HIGH]; vf_data->aeqe_dma <<=QM_XQC_ADDR_OFFSET ; - vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0 ]; + vf_data->aeqe_dma |= vf_data- >qm_aeqc_dw[QM_XQC_ADDR_LOW];drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +0
-0/* Through SQC_BT/CQC_BT to get sqc and cqc address */ ret =qm_get_sqc (vf_qm, &vf_data->sqc_dma);drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +12
-3@@ -39,6 +39,9 @@#define QM_REG_ADDR_OFFSET 0x0004#define QM_XQC_ADDR_OFFSET 32U +#define QM_XQC_ADDR_LOW 0x1 +#define QM_XQC_ADDR_HIGH 0x2 +#define QM_VF_AEQ_INT_MASK 0x0004#define QM_VF_EQ_INT_MASK 0x000c#define QM_IFC_INT_SOURCE_V 0x0020@@ -50,10 +53,15 @@#define QM_EQC_DW0 0X8000#define QM_AEQC_DW0 0X8020 +#define ACC_DRV_MAJOR_VER 1 +#define ACC_DRV_MINOR_VER 0 + +#define ACC_DEV_MAGIC_V1 0XCDCDCDCDFEEDAACC +#define ACC_DEV_MAGIC_V2 0xAACCFEEDDECADEDE +struct acc_vf_data {#define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)/* QM match information */ -#define ACC_DEV_MAGIC 0XCDCDCDCDFEEDAACCu64 acc_magic;u32 qp_num;u32 dev_id;@@ -61,7 +69,9 @@ struct acc_vf_data {u32 qp_base;u32 vf_qm_state;/* QM reserved match information */ -u32 qm_rsv_state[3 ]; +u16 major_ver; +u16 minor_ver; +u32 qm_rsv_state[2 ];/* QM RW regs */ u32 aeq_int_mask; --2.24.0
liulongfang
maintainerRe: [PATCH v5 1/5] hisi_acc_vfio_pci: fix XQE dma address error
Quoted 31 lines · Shameerali Kolothum Thodi
Quoted 23 lines
-----Original Message-----
From: liulongfang <[email protected]>
Sent: Thursday, March 13, 2025 7:20 AM
To: [email protected]; [email protected]; Shameerali Kolothum Thodi <[email protected]>; Jonathan Cameron <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; liulongfang <[email protected]>
Subject: [PATCH v5 1/5]hisi_acc_vfio_pci: fix XQE dma address errorThe dma addresses of EQE and AEQE are wrong after migration and results in guest kernel-mode encryption services failure. Comparing the definition of hardware registers, we found that there was an error when the data read from the register was combined into an address. Therefore, the address combination sequence needs to be corrected.
Even after fixing the above problem, we still have an issue where the Guest from an old kernel can get migrated to new kernel and may result in wrong data.
Quoted 7 lines
In order to ensure that the address is correct after migration, if an old magic number is detected, the dma address needs to be updated.
Fixes b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO livemigration")
Signed-off-by Longfang Liu <[email protected]>
Quoted 7 lines
LGTM,
Reviewed-by Shameer Kolothum <[email protected]>Thanks,
Shameer
Thanks.
Longfang.
Quoted 142 lines
Quoted 138 lines
--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 41 ++++++++++++++++--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.h | 14 ++++++- 2 files changed, 47 insertions(+), 8 deletions(-)drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +15
-0@@ -350,6 +350,32 @@ static int vf_qm_func_stop(struct hisi_qm *qm)return hisi_qm_mb (qm,QM_MB_CMD_PAUSE_QM ,0 ,0 ,0 ); } +static int vf_qm_version_check (struct acc_vf_data *vf_data,struct device *dev) +{ +switch (vf_data->acc_magic) { +case ACC_DEV_MAGIC_V2 : +if (vf_data->major_ver !=ACC_DRV_MAJOR_VER ) { +dev_info (dev,"migration driver version<%u.%u> not match!\n", + vf_data->major_ver, vf_data->minor_ver); +return -EINVAL ; + } +break ; +case ACC_DEV_MAGIC_V1 : +/* Correct dma address */ + vf_data->eqe_dma = vf_data- > qm_eqc_dw[QM_XQC_ADDR_HIGH]; + vf_data->eqe_dma <<=QM_XQC_ADDR_OFFSET ; + vf_data->eqe_dma |= vf_data-Quoted 1 line
qm_eqc_dw[QM_XQC_ADDR_LOW];
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-0+ vf_data->aeqe_dma = vf_data-Quoted 1 line
qm_aeqc_dw[QM_XQC_ADDR_HIGH];
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +2
-0+ vf_data->aeqe_dma <<=QM_XQC_ADDR_OFFSET ; + vf_data->aeqe_dma |= vf_data-Quoted 1 line
qm_aeqc_dw[QM_XQC_ADDR_LOW];
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +8
-0+break ; +default : +return -EINVAL ; + } + +return 0 ; +} + static int vf_qm_check_match(struct hisi_acc_vf_core_device
*hisi_acc_vdev,drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +2
-1struct hisi_acc_vf_migration_file *migf) {@@ -363,7 +389,8 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev- > match_done)return 0 ; -if (vf_data->acc_magic !=ACC_DEV_MAGIC ) { + ret =vf_qm_version_check (vf_data, dev); +if (ret) {dev_err (dev,"failed to match ACC_DEV_MAGIC\n" );drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +7
-5return -EINVAL ; }@@ -418,7 +445,9 @@ static int vf_qm_get_match_data(structhisi_acc_vf_core_device *hisi_acc_vdev, int vf_id = hisi_acc_vdev->vf_id;int ret; - vf_data->acc_magic =ACC_DEV_MAGIC ; + vf_data->acc_magic =ACC_DEV_MAGIC_V2 ; + vf_data->major_ver =ACC_DRV_MAJOR_VER ; + vf_data->minor_ver =ACC_DRV_MINOR_VER ;/* Save device id */ vf_data->dev_id = hisi_acc_vdev->vf_dev->device;@@ -496,12 +525,12 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, struct acc_vf_data *vf_data)return -EINVAL ;/* Every reg is 32 bit, the dma address is 64 bit. */ - vf_data->eqe_dma = vf_data->qm_eqc_dw[1 ]; + vf_data->eqe_dma = vf_data->qm_eqc_dw[QM_XQC_ADDR_HIGH ]; vf_data->eqe_dma <<=QM_XQC_ADDR_OFFSET ; - vf_data->eqe_dma |= vf_data->qm_eqc_dw[0 ]; - vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1 ]; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[QM_XQC_ADDR_LOW ]; + vf_data->aeqe_dma = vf_data- > qm_aeqc_dw[QM_XQC_ADDR_HIGH]; vf_data->aeqe_dma <<=QM_XQC_ADDR_OFFSET ; - vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0 ]; + vf_data->aeqe_dma |= vf_data- > qm_aeqc_dw[QM_XQC_ADDR_LOW];drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +0
-0/* Through SQC_BT/CQC_BT to get sqc and cqc address */ ret =qm_get_sqc (vf_qm, &vf_data->sqc_dma);drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +12
-3@@ -39,6 +39,9 @@#define QM_REG_ADDR_OFFSET 0x0004#define QM_XQC_ADDR_OFFSET 32U +#define QM_XQC_ADDR_LOW 0x1 +#define QM_XQC_ADDR_HIGH 0x2 +#define QM_VF_AEQ_INT_MASK 0x0004#define QM_VF_EQ_INT_MASK 0x000c#define QM_IFC_INT_SOURCE_V 0x0020@@ -50,10 +53,15 @@#define QM_EQC_DW0 0X8000#define QM_AEQC_DW0 0X8020 +#define ACC_DRV_MAJOR_VER 1 +#define ACC_DRV_MINOR_VER 0 + +#define ACC_DEV_MAGIC_V1 0XCDCDCDCDFEEDAACC +#define ACC_DEV_MAGIC_V2 0xAACCFEEDDECADEDE +struct acc_vf_data {#define QM_MATCH_SIZE offsetofend(struct acc_vf_data, qm_rsv_state)/* QM match information */ -#define ACC_DEV_MAGIC 0XCDCDCDCDFEEDAACCu64 acc_magic;u32 qp_num;u32 dev_id;@@ -61,7 +69,9 @@ struct acc_vf_data {u32 qp_base;u32 vf_qm_state;/* QM reserved match information */ -u32 qm_rsv_state[3 ]; +u16 major_ver; +u16 minor_ver; +u32 qm_rsv_state[2 ];/* QM RW regs */ u32 aeq_int_mask; --Quoted 1 line
2.24.0
.
Longfang Liu
maintainerPATCH v5 2/5 hisi_acc_vfio_pci: add eq and aeq interruption restore
In reply to Longfang LiuIn order to ensure that the task packets of the accelerator device are not lost during the migration process, it is necessary to send an EQ and AEQ command to the device after the live migration is completed and to update the completion position of the task queue.
Let the device recheck the completed tasks data and if there are uncollected packets, device resend a task completion interrupt to the software.
--- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +16 -0
@@ -470,6 +470,19 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,return 0 ; } +static void vf_qm_xeqc_save (struct hisi_qm *qm, +struct hisi_acc_vf_migration_file *migf) +{ +struct acc_vf_data *vf_data = &migf->vf_data; +u16 eq_head, aeq_head; + + eq_head = vf_data->qm_eqc_dw[0 ] &0xFFFF ; +qm_db (qm,0 ,QM_DOORBELL_CMD_EQ , eq_head,0 ); + + aeq_head = vf_data->qm_aeqc_dw[0 ] &0xFFFF ; +qm_db (qm,0 ,QM_DOORBELL_CMD_AEQ , aeq_head,0 ); +} +static int vf_qm_load_data (struct hisi_acc_vf_core_device *hisi_acc_vdev,struct hisi_acc_vf_migration_file *migf) {
@@ -578,6 +591,9 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,return -EINVAL ; migf->total_length =sizeof (struct acc_vf_data ); +/* Save eqc and aeqc interrupt information */ +vf_qm_xeqc_save (vf_qm, migf); +return 0 ; }
Signature · 1 line
2.24.0
Longfang Liu
maintainerPATCH v5 3/5 hisi_acc_vfio_pci: bugfix cache write-back issue
In reply to Longfang LiuAt present, cache write-back is placed in the device data copy stage after stopping the device operation. Writing back to the cache at this stage will cause the data obtained by the cache to be written back to be empty.
In order to ensure that the cache data is written back successfully, the data needs to be written back into the stop device stage.
--- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +7 -7
@@ -566,7 +566,6 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, {struct acc_vf_data *vf_data = &migf->vf_data;struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; -struct device *dev = &vf_qm->pdev->dev;int ret;if (unlikely (qm_wait_dev_not_ready (vf_qm))) {
@@ -580,12 +579,6 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, vf_data->vf_qm_state =QM_READY ; hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; - ret =vf_qm_cache_wb (vf_qm); -if (ret) { -dev_err (dev,"failed to writeback QM Cache!\n" ); -return ret; - } - ret =vf_qm_read_data (vf_qm, vf_data);if (ret)return -EINVAL ;
@@ -1012,6 +1005,13 @@ static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdevdev_err (dev,"failed to check QM INT state!\n" );return ret; } + + ret =vf_qm_cache_wb (vf_qm); +if (ret) { +dev_err (dev,"failed to writeback QM cache!\n" ); +return ret; + } +return 0 ; }
Signature · 1 line
2.24.0
Longfang Liu
maintainerPATCH v5 4/5 hisi_acc_vfio_pci: bugfix the problem of uninstalling driver
In reply to Longfang LiuIn a live migration scenario. If the number of VFs at the destination is greater than the source, the recovery operation will fail and qemu will not be able to complete the process and exit after shutting down the device FD.
This will cause the driver to be unable to be unloaded normally due to abnormal reference counting of the live migration driver caused by the abnormal closing operation of fd.
--- drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 1 + 1 file changed, 1 insertion(+)
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1 -0
@@ -1508,6 +1508,7 @@ static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)struct hisi_acc_vf_core_device *hisi_acc_vdev =hisi_acc_get_vf_dev (core_vdev);struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm; +hisi_acc_vf_disable_fds (hisi_acc_vdev);mutex_lock (&hisi_acc_vdev->open_mutex); hisi_acc_vdev->dev_opened = false;iounmap (vf_qm->io_base);
Signature · 1 line
2.24.0
Longfang Liu
maintainerPATCH v5 5/5 hisi_acc_vfio_pci: bugfix live migration function without VF device driver
In reply to Longfang LiuIf the VF device driver is not loaded in the Guest OS and we attempt to perform device data migration, the address of the migrated data will be NULL.
The live migration recovery operation on the destination side will access a null address value, which will cause access errors.
Therefore, live migration of VMs without added VF device drivers does not require device data migration.
In addition, when the queue address data obtained by the destination is empty, device queue recovery processing will not be performed.
--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-)
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +18 -7
@@ -426,13 +426,6 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,return -EINVAL ; } - ret =qm_write_regs (vf_qm,QM_VF_STATE , &vf_data->vf_qm_state,1 ); -if (ret) { -dev_err (dev,"failed to write QM_VF_STATE\n" ); -return ret; - } - - hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; hisi_acc_vdev->match_done = true;return 0 ; }
@@ -498,6 +491,13 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,if (migf->total_length <sizeof (struct acc_vf_data ))return -EINVAL ; + ret =qm_write_regs (qm,QM_VF_STATE , &vf_data->vf_qm_state,1 ); +if (ret) { +dev_err (dev,"failed to write QM_VF_STATE\n" ); +return -EINVAL ; + } + hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; + qm->eqe_dma = vf_data->eqe_dma; qm->aeqe_dma = vf_data->aeqe_dma; qm->sqc_dma = vf_data->sqc_dma;
@@ -506,6 +506,12 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, qm->qp_base = vf_data->qp_base; qm->qp_num = vf_data->qp_num; +if (!vf_data->eqe_dma || !vf_data->aeqe_dma || + !vf_data->sqc_dma || !vf_data->cqc_dma) { +dev_err (dev,"resume dma addr is NULL!\n" ); +return -EINVAL ; + } + ret =qm_set_regs (qm, vf_data);if (ret) {dev_err (dev,"set VF regs failed\n" );
@@ -726,8 +732,12 @@ static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) {struct device *dev = &hisi_acc_vdev->vf_dev->dev;struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf; +struct acc_vf_data *vf_data = &migf->vf_data;int ret; +if (vf_data->vf_qm_state !=QM_READY ) +return 0 ; +/* Recover data to VF */ ret =vf_qm_load_data (hisi_acc_vdev, migf);if (ret) {
@@ -1531,6 +1541,7 @@ static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) hisi_acc_vdev->vf_id =pci_iov_vf_id (pdev) +1 ; hisi_acc_vdev->pf_qm = pf_qm; hisi_acc_vdev->vf_dev = pdev; + hisi_acc_vdev->vf_qm_state =QM_NOT_READY ;mutex_init (&hisi_acc_vdev->state_mutex);mutex_init (&hisi_acc_vdev->open_mutex);
Signature · 1 line
2.24.0
RE: [PATCH v5 5/5] hisi_acc_vfio_pci: bugfix live migration function without VF device driver
Quoted 82 lines
-----Original Message-----
From: liulongfang <[email protected]>
Sent: Thursday, March 13, 2025 7:20 AM
To: [email protected]; [email protected]; Shameerali Kolothum Thodi <[email protected]>; Jonathan Cameron <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; liulongfang <[email protected]>
Subject: [PATCH v5 5/5]hisi_acc_vfio_pci: bugfix live migration function without VF device driverIf the VF device driver is not loaded in the Guest OS and we attempt to perform device data migration, the address of the migrated data will be NULL.
The live migration recovery operation on the destination side will access a null address value, which will cause access errors.Therefore, live migration of VMs without added VF device drivers does not require device data migration.
In addition, when the queue address data obtained by the destination is empty, device queue recovery processing will not be performed.Fixes b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO livemigration")
Signed-off-by Longfang Liu <[email protected]>--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-)drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +0
-7@@ -426,13 +426,6 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, return -EINVAL ; } - ret =qm_write_regs (vf_qm,QM_VF_STATE , &vf_data->vf_qm_state, 1); -if (ret) { -dev_err (dev,"failed to write QM_VF_STATE\n" ); -return ret; - } - - hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; hisi_acc_vdev->match_done = true;drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +7
-0return 0 ; }@@ -498,6 +491,13 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (migf->total_length < sizeof(struct acc_vf_data)) return -EINVAL; + ret =qm_write_regs (qm,QM_VF_STATE , &vf_data->vf_qm_state,1 ); +if (ret) { +dev_err (dev,"failed to write QM_VF_STATE\n" ); +return -EINVAL ; + } + hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; + qm->eqe_dma = vf_data->eqe_dma; qm->aeqe_dma = vf_data->aeqe_dma;drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +6
-0qm->sqc_dma = vf_data->sqc_dma;@@ -506,6 +506,12 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, qm->qp_base = vf_data->qp_base; qm->qp_num = vf_data->qp_num; +if (!vf_data->eqe_dma || !vf_data->aeqe_dma || + !vf_data->sqc_dma || !vf_data->cqc_dma) { +dev_err (dev,"resume dma addr is NULL!\n" ); +return -EINVAL ; + } + ret =qm_set_regs (qm, vf_data);if (ret) {drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +0
-0dev_err (dev,"set VF regs failed\n" );
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c static int hisi_acc_vf_load_state(struct · line 741
Quoted 10 lines
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +3
-0@@ -726,8 +732,12 @@ static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) {struct device *dev = &hisi_acc_vdev->vf_dev->dev;struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev- >resuming_migf; +struct acc_vf_data *vf_data = &migf->vf_data;int ret; +if (vf_data->vf_qm_state !=QM_READY ) +return 0 ;
I don't think we need to check the above. In vf_qm_satte_save(), If vf_qm_state != QM_READY, we set the
migf->total_length = QM_MATCH_SIZE.
Hence it will return 0 in the below vf_qm_load_data() anyway.
With that corrected,
Thanks,
Shameer
Quoted 15 lines
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-0+/* Recover data to VF */ drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-1ret =vf_qm_load_data (hisi_acc_vdev, migf);if (ret) {@@ -1531,6 +1541,7 @@ static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1; hisi_acc_vdev->pf_qm = pf_qm; hisi_acc_vdev->vf_dev = pdev; + hisi_acc_vdev->vf_qm_state =QM_NOT_READY ;mutex_init (&hisi_acc_vdev->state_mutex);mutex_init (&hisi_acc_vdev->open_mutex); --2.24.0
liulongfang
maintainerRe: [PATCH v5 5/5] hisi_acc_vfio_pci: bugfix live migration function without VF device driver
Quoted 90 lines · Shameerali Kolothum Thodi
Quoted 90 lines
-----Original Message-----
From: liulongfang <[email protected]>
Sent: Thursday, March 13, 2025 7:20 AM
To: [email protected]; [email protected]; Shameerali Kolothum Thodi <[email protected]>; Jonathan Cameron <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; liulongfang <[email protected]>
Subject: [PATCH v5 5/5]hisi_acc_vfio_pci: bugfix live migration function without VF device driverIf the VF device driver is not loaded in the Guest OS and we attempt to perform device data migration, the address of the migrated data will be NULL.
The live migration recovery operation on the destination side will access a null address value, which will cause access errors.Therefore, live migration of VMs without added VF device drivers does not require device data migration.
In addition, when the queue address data obtained by the destination is empty, device queue recovery processing will not be performed.Fixes b0eed085903e ("hisi_acc_vfio_pci: Add support for VFIO livemigration")
Signed-off-by Longfang Liu <[email protected]>--- .../vfio/pci/hisilicon/hisi_acc_vfio_pci.c | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-)drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +0
-7@@ -426,13 +426,6 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, return -EINVAL ; } - ret =qm_write_regs (vf_qm,QM_VF_STATE , &vf_data->vf_qm_state, 1); -if (ret) { -dev_err (dev,"failed to write QM_VF_STATE\n" ); -return ret; - } - - hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; hisi_acc_vdev->match_done = true;drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +7
-0return 0 ; }@@ -498,6 +491,13 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (migf->total_length < sizeof(struct acc_vf_data)) return -EINVAL; + ret =qm_write_regs (qm,QM_VF_STATE , &vf_data->vf_qm_state,1 ); +if (ret) { +dev_err (dev,"failed to write QM_VF_STATE\n" ); +return -EINVAL ; + } + hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state; + qm->eqe_dma = vf_data->eqe_dma; qm->aeqe_dma = vf_data->aeqe_dma;drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +6
-0qm->sqc_dma = vf_data->sqc_dma;@@ -506,6 +506,12 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, qm->qp_base = vf_data->qp_base; qm->qp_num = vf_data->qp_num; +if (!vf_data->eqe_dma || !vf_data->aeqe_dma || + !vf_data->sqc_dma || !vf_data->cqc_dma) { +dev_err (dev,"resume dma addr is NULL!\n" ); +return -EINVAL ; + } + ret =qm_set_regs (qm, vf_data);if (ret) {drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-0dev_err (dev,"set VF regs failed\n" );@@ -726,8 +732,12 @@ static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) {struct device *dev = &hisi_acc_vdev->vf_dev->dev;struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev- > resuming_migf; +struct acc_vf_data *vf_data = &migf->vf_data;int ret;
Quoted 9 lines
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) · line 739
Quoted 2 lines
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +2
-0+if (vf_data->vf_qm_state !=QM_READY ) +return 0 ;I don't think we need to check the above. In
vf_qm_satte_save(), Ifvf_qm_state!=QM_READY, we set the
migf->total_length=QM_MATCH_SIZE.Hence it will return 0 in the below
vf_qm_load_data()anyway.
After removing this judgment code, the live migration function works normally without loading the VM driver.
I will remove this judgment in the next version.
Thanks.
Longfang.
Quoted 25 lines
With that corrected,
Reviewed-by Shameer Kolothum <[email protected]>Thanks,
ShameerQuoted 14 lines
drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-0+/* Recover data to VF */ drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +1
-1ret =vf_qm_load_data (hisi_acc_vdev, migf);if (ret) {@@ -1531,6 +1541,7 @@ static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1; hisi_acc_vdev->pf_qm = pf_qm; hisi_acc_vdev->vf_dev = pdev; + hisi_acc_vdev->vf_qm_state =QM_NOT_READY ;mutex_init (&hisi_acc_vdev->state_mutex);mutex_init (&hisi_acc_vdev->open_mutex); --Quoted 1 line
2.24.0
.