vfio/pds: Rework and simplify reset flows
Subsystems (4)
Brett Creeley
maintainerPATCH 1/1 vfio/pds: Rework and simplify reset flows
The current logic for handling resets based on whether they were initiated from the DSC or host/VMM is slightly confusing and incorrect. The incorrect behavior can cause the VF device to be unusable on the destination on failed migrations due to incompatible configurations. Fix this by setting the state back to VFIO_DEVICE_STATE_RUNNING when an FLR is triggered, so the VF device is put back in an "initial" pre-configured state after failures.
Also, while here clean-up the reset logic to make the source of the reset more obvious.
--- drivers/vfio/pci/pds/pci_drv.c | 2 +- drivers/vfio/pci/pds/vfio_dev.c | 14 +++++++------- drivers/vfio/pci/pds/vfio_dev.h | 7 ++++++- 3 files changed, 14 insertions(+), 9 deletions(-)
drivers/vfio/pci/pds/pci_drv.c +1 -1
@@ -57,7 +57,7 @@ static void pds_vfio_recovery(struct pds_vfio_pci_device *pds_vfio)if (deferred_reset_needed) {mutex_lock (&pds_vfio->reset_mutex); pds_vfio->deferred_reset = true; - pds_vfio->deferred_reset_state =VFIO_DEVICE_STATE_ERROR ; + pds_vfio->deferred_reset_type =PDS_VFIO_DEVICE_RESET ;mutex_unlock (&pds_vfio->reset_mutex); } }
drivers/vfio/pci/pds/vfio_dev.c +7 -7
@@ -32,13 +32,14 @@ void pds_vfio_state_mutex_unlock(struct pds_vfio_pci_device *pds_vfio)mutex_lock (&pds_vfio->reset_mutex);if (pds_vfio->deferred_reset) { pds_vfio->deferred_reset = false; -if (pds_vfio->state ==VFIO_DEVICE_STATE_ERROR ) { -pds_vfio_put_restore_file (pds_vfio); -pds_vfio_put_save_file (pds_vfio); +pds_vfio_put_restore_file (pds_vfio); +pds_vfio_put_save_file (pds_vfio); +if (pds_vfio->deferred_reset_type ==PDS_VFIO_HOST_RESET ) { + pds_vfio->state =VFIO_DEVICE_STATE_RUNNING ; + }else {pds_vfio_dirty_disable (pds_vfio, false); + pds_vfio->state =VFIO_DEVICE_STATE_ERROR ; } - pds_vfio->state = pds_vfio->deferred_reset_state; - pds_vfio->deferred_reset_state =VFIO_DEVICE_STATE_RUNNING ;mutex_unlock (&pds_vfio->reset_mutex); goto again; }
@@ -50,7 +51,7 @@ void pds_vfio_reset(struct pds_vfio_pci_device *pds_vfio) {mutex_lock (&pds_vfio->reset_mutex); pds_vfio->deferred_reset = true; - pds_vfio->deferred_reset_state =VFIO_DEVICE_STATE_RUNNING ; + pds_vfio->deferred_reset_type =PDS_VFIO_HOST_RESET ;if (!mutex_trylock (&pds_vfio->state_mutex)) {mutex_unlock (&pds_vfio->reset_mutex);return ;
@@ -194,7 +195,6 @@ static int pds_vfio_open_device(struct vfio_device *vdev)return err; pds_vfio->state =VFIO_DEVICE_STATE_RUNNING ; - pds_vfio->deferred_reset_state =VFIO_DEVICE_STATE_RUNNING ;vfio_pci_core_finish_enable (&pds_vfio->vfio_coredev);
drivers/vfio/pci/pds/vfio_dev.h +6 -1
@@ -10,6 +10,11 @@#include "dirty.h" #include "lm.h" +enum pds_vfio_reset_type { +PDS_VFIO_HOST_RESET =0 , +PDS_VFIO_DEVICE_RESET =1 , +}; +struct pds_vfio_pci_device {struct vfio_pci_core_device vfio_coredev;
@@ -20,7 +25,7 @@ struct pds_vfio_pci_device {enum vfio_device_mig_state state;struct mutex reset_mutex;/* protect reset_done flow */ u8 deferred_reset; -enum vfio_device_mig_state deferred_reset_state; +enum pds_vfio_reset_type deferred_reset_type;struct notifier_block nb;int vf_id;
Signature · 1 line
2.17.1
Tian, Kevin
reviewerQuoted 13 lines
From: Brett Creeley <[email protected]>
Sent: Saturday, January 27, 2024 2:32 AMThe current logic for handling resets based on whether they were initiated from the DSC or host/VMM is slightly confusing and incorrect. The incorrect behavior can cause the VF device to be unusable on the destination on failed migrations due to incompatible configurations. Fix this by setting the state back to
VFIO_DEVICE_STATE_RUNNINGwhen an FLR is triggered, so the VF device is put back in an "initial" pre-configured state after failures.
any reason for putting short lines (<50 chars) in commit msg?
Quoted 3 lines
Also, while here clean-up the reset logic to make the source of the reset more obvious.
as a fix the a 'Fixed' tag is preferred and CC stable
also separate the real fix from the cleanup so stable kernel doesn't need to backport unnecessary code.
btw the commit msg is not clear to me. It says fixing the problem by setting the state to _ERROR for the DSC path and to _RUNNING for the FLR path.
But looks it's already such case with old code:
pds_vfio_recovery()
pds_vfio->deferred_reset = true; pds_vfio->deferred_reset_state = VFIO_DEVICE_STATE_ERROR;
pds_vfio_reset()
pds_vfio->deferred_reset = true; pds_vfio->deferred_reset_state = VFIO_DEVICE_STATE_RUNNING;
pds_vfio_state_mutex_unlock()
if (pds_vfio->deferred_reset) {
...
pds_vfio->state = pds_vfio->deferred_reset_state;
...
}it's same as what this patch does:
pds_vfio_recovery()
pds_vfio->deferred_reset_type = PDS_VFIO_DEVICE_RESET;
pds_vfio_reset()
pds_vfio->deferred_reset_state = PDS_VFIO_HOST_RESET;
pds_vfio_state_mutex_unlock()
if (pds_vfio->deferred_reset) {
...
if (pds_vfio->deferred_reset_type == PDS_VFIO_HOST_RESET)
pds_vfio->state = VFIO_DEVICE_STATE_RUNNING;
else
pds_vfio->state = VFIO_DEVICE_STATE_ERROR;
...
}looks the actual functional difference is from below change:
drivers/vfio/pci/pds/vfio_dev.c void pds_vfio_state_mutex_unlock(struct · line 37
Quoted 10 lines
drivers/vfio/pci/pds/vfio_dev.c +2
-3@@ -32,13 +32,14 @@ void pds_vfio_state_mutex_unlock(struct pds_vfio_pci_device *pds_vfio) mutex_lock(&pds_vfio->reset_mutex);if (pds_vfio->deferred_reset) { pds_vfio->deferred_reset = false; -if (pds_vfio->state ==VFIO_DEVICE_STATE_ERROR ) { -pds_vfio_put_restore_file (pds_vfio); -pds_vfio_put_save_file (pds_vfio); +pds_vfio_put_restore_file (pds_vfio); +pds_vfio_put_save_file (pds_vfio);
above two are changed from conditional to always.
drivers/vfio/pci/pds/vfio_dev.c void pds_vfio_state_mutex_unlock(struct pds_vfio_pci_device *pds_vfio) · line 40
Quoted 5 lines
drivers/vfio/pci/pds/vfio_dev.c +3
-0+if (pds_vfio->deferred_reset_type ==PDS_VFIO_HOST_RESET ) { + pds_vfio->state =VFIO_DEVICE_STATE_RUNNING ; + }else {pds_vfio_dirty_disable (pds_vfio, false);
and this is now only for the DSC path.
need a better explanation here.
Quoted 17 lines · Kevin
Caution: This message originated from an External Source. Use proper caution when opening attachments, clicking links, or responding.
Quoted 13 lines
From: Brett Creeley <[email protected]>
Sent: Saturday, January 27, 2024 2:32 AMThe current logic for handling resets based on whether they were initiated from the DSC or host/VMM is slightly confusing and incorrect. The incorrect behavior can cause the VF device to be unusable on the destination on failed migrations due to incompatible configurations. Fix this by setting the state back to
VFIO_DEVICE_STATE_RUNNINGwhen an FLR is triggered, so the VF device is put back in an "initial" pre-configured state after failures.
Quoted 1 line
any reason for putting short lines (<50 chars) in commit msg?
No, I will make the lines longer in the next commits.
Quoted 9 lines
Quoted 4 lines
Also, while here clean-up the reset logic to make the source of the reset more obvious.
as a fix the a 'Fixed' tag is preferred and CC stable
also separate the real fix from the cleanup so stable kernel doesn't need to backport unnecessary code.
Sure, I can split this into 2 changes as you suggested.
Quoted 61 lines
btw the commit msg is not clear to me. It says fixing the problem by setting the state to
_ERRORfor the DSC path and to_RUNNINGfor the FLR path.But looks it's already such case with old code:
pds_vfio_recovery()pds_vfio->deferred_reset = true; pds_vfio->deferred_reset_state = VFIO_DEVICE_STATE_ERROR;
pds_vfio_reset()pds_vfio->deferred_reset = true; pds_vfio->deferred_reset_state = VFIO_DEVICE_STATE_RUNNING;
pds_vfio_state_mutex_unlock()if (pds_vfio->deferred_reset) { ... pds_vfio->state = pds_vfio->deferred_reset_state; ... }it's same as what this patch does:
pds_vfio_recovery()pds_vfio->deferred_reset_type = PDS_VFIO_DEVICE_RESET;
pds_vfio_reset()pds_vfio->deferred_reset_state = PDS_VFIO_HOST_RESET;
pds_vfio_state_mutex_unlock()if (pds_vfio->deferred_reset) { ... if (pds_vfio->deferred_reset_type == PDS_VFIO_HOST_RESET) pds_vfio->state = VFIO_DEVICE_STATE_RUNNING; else pds_vfio->state = VFIO_DEVICE_STATE_ERROR; ... }looks the actual functional difference is from below change:
drivers/vfio/pci/pds/vfio_dev.c void pds_vfio_state_mutex_unlock(struct · line 37
Quoted 10 lines
drivers/vfio/pci/pds/vfio_dev.c +2
-3@@ -32,13 +32,14 @@ void pds_vfio_state_mutex_unlock(struct pds_vfio_pci_device *pds_vfio) mutex_lock(&pds_vfio->reset_mutex);if (pds_vfio->deferred_reset) { pds_vfio->deferred_reset = false; -if (pds_vfio->state ==VFIO_DEVICE_STATE_ERROR ) { -pds_vfio_put_restore_file (pds_vfio); -pds_vfio_put_save_file (pds_vfio); +pds_vfio_put_restore_file (pds_vfio); +pds_vfio_put_save_file (pds_vfio);above two are changed from conditional to always.
drivers/vfio/pci/pds/vfio_dev.c void pds_vfio_state_mutex_unlock(struct pds_vfio_pci_device *pds_vfio) · line 40
Quoted 5 lines
drivers/vfio/pci/pds/vfio_dev.c +3
-0+if (pds_vfio->deferred_reset_type ==PDS_VFIO_HOST_RESET ) { + pds_vfio->state =VFIO_DEVICE_STATE_RUNNING ; + }else {pds_vfio_dirty_disable (pds_vfio, false);
Quoted 3 lines
and this is now only for the DSC path.
need a better explanation here.
I will clean up this patch by separating it into 2 patches and improving the commit descriptions before sending a v2.
Thanks for the review,
Brett