s390/vfio-ap: do not use open locks during VFIO_GROUP_NOTIFY_SET_KVM notification
Subsystems (3)
Tony Krowiak
maintainerPATCH 1/1 s390/vfio-ap: do not use open locks during VFIO_GROUP_NOTIFY_SET_KVM notification
The fix to resolve a lockdep splat while handling the VFIO_GROUP_NOTIFY_SET_KVM event introduced a kvm_busy flag indicating that the vfio_ap device driver is busy setting or unsetting the KVM pointer. A wait queue was employed to allow functions requiring access to the KVM pointer to wait for the kvm_busy flag to be cleared. For the duration of the wait period, the mdev lock was unlocked then acquired again after the kvm_busy flag was cleared. This got rid of the lockdep report, but didn't really resolve the problem.
This patch removes the the kvm_busy flag and wait queue as they are not necessary to resolve the lockdep splat problem. The wait queue was introduced to prevent changes to the matrix used to update the guest's AP configuration. The idea was that whenever an adapter, domain or control domain was being assigned to or unassigned from the matrix, the function would wait until the group notifier function was no longer busy with the KVM pointer.
The thing is, the KVM pointer value (matrix_mdev->kvm) is always set and cleared while holding the matrix_dev->lock mutex. The assignment and unassignment interfaces also lock the matrix_dev->lock mutex prior to checking whether the matrix_mdev->kvm pointer is set and if so, returns the -EBUSY error from the function. Consequently, there is no chance for an update to the matrix to occur while the guest's AP configuration is being updated.
--- drivers/s390/crypto/vfio_ap_ops.c | 77 +++++++-------------------- drivers/s390/crypto/vfio_ap_private.h | 2 - 2 files changed, 20 insertions(+), 59 deletions(-)
drivers/s390/crypto/vfio_ap_ops.c +20 -57
@@ -294,15 +294,6 @@ static int handle_pqap(struct kvm_vcpu *vcpu) matrix_mdev =container_of (vcpu->kvm->arch.crypto.pqap_hook, struct ap_matrix_mdev, pqap_hook); -/* -* If the KVM pointer is in the process of being set, wait until the -* process has completed. -*/ -wait_event_cmd (matrix_mdev->wait_for_kvm, - !matrix_mdev->kvm_busy, -mutex_unlock (&matrix_dev->lock), -mutex_lock (&matrix_dev->lock)); -/* If the there is no guest using the mdev, there is nothing to do */ if (!matrix_mdev->kvm) goto out_unlock;
@@ -350,7 +341,6 @@ static int vfio_ap_mdev_create(struct mdev_device *mdev) matrix_mdev->mdev = mdev;vfio_ap_matrix_init (&matrix_dev->info, &matrix_mdev->matrix); -init_waitqueue_head (&matrix_mdev->wait_for_kvm);mdev_set_drvdata (mdev, matrix_mdev); matrix_mdev->pqap_hook = handle_pqap;mutex_lock (&matrix_dev->lock);
@@ -623,7 +613,7 @@ static ssize_t assign_adapter_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of adapter */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }
@@ -696,7 +686,7 @@ static ssize_t unassign_adapter_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of adapter */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }
@@ -786,7 +776,7 @@ static ssize_t assign_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * assignment of domain */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }
@@ -854,7 +844,7 @@ static ssize_t unassign_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of domain */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }
@@ -908,7 +898,7 @@ static ssize_t assign_control_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * assignment of control domain. */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }
@@ -967,7 +957,7 @@ static ssize_t unassign_control_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of control domain. */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }
@@ -1108,14 +1098,17 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,struct ap_matrix_mdev *m;if (kvm->arch.crypto.crycbd) { +mutex_lock (&matrix_dev->lock); +list_for_each_entry (m, &matrix_dev->mdev_list, node) { -if (m != matrix_mdev && m->kvm == kvm) +if (m != matrix_mdev && m->kvm == kvm) { +mutex_unlock (&matrix_dev->lock);return -EPERM ; + } }kvm_get_kvm (kvm); matrix_mdev->kvm = kvm; - matrix_mdev->kvm_busy = true;mutex_unlock (&matrix_dev->lock);down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem);
@@ -1126,10 +1119,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, matrix_mdev->matrix.apm, matrix_mdev->matrix.aqm, matrix_mdev->matrix.adm); - -mutex_lock (&matrix_dev->lock); - matrix_mdev->kvm_busy = false; -wake_up_all (&matrix_mdev->wait_for_kvm); }return 0 ;
@@ -1181,33 +1170,21 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb,*/ static void vfio_ap_mdev_unset_kvm (struct ap_matrix_mdev *matrix_mdev) { -/* -* If the KVM pointer is in the process of being set, wait until the -* process has completed. -*/ -wait_event_cmd (matrix_mdev->wait_for_kvm, - !matrix_mdev->kvm_busy, -mutex_unlock (&matrix_dev->lock), -mutex_lock (&matrix_dev->lock)); +mutex_lock (&matrix_dev->lock); -if (matrix_mdev->kvm) { - matrix_mdev->kvm_busy = true; +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock); +down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); + matrix_mdev->kvm->arch.crypto.pqap_hook =NULL ; +up_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); -if (matrix_mdev->kvm->arch.crypto.crycbd) { -down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); - matrix_mdev->kvm->arch.crypto.pqap_hook =NULL ; -up_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); - -kvm_arch_crypto_clear_masks (matrix_mdev->kvm); - } +kvm_arch_crypto_clear_masks (matrix_mdev->kvm);mutex_lock (&matrix_dev->lock);vfio_ap_mdev_reset_queues (matrix_mdev->mdev);kvm_put_kvm (matrix_mdev->kvm); matrix_mdev->kvm =NULL ; - matrix_mdev->kvm_busy = false; -wake_up_all (&matrix_mdev->wait_for_kvm); +mutex_unlock (&matrix_dev->lock); } }
@@ -1220,7 +1197,6 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,if (action !=VFIO_GROUP_NOTIFY_SET_KVM )return NOTIFY_OK ; -mutex_lock (&matrix_dev->lock); matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);if (!data)
@@ -1228,8 +1204,6 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,else if (vfio_ap_mdev_set_kvm (matrix_mdev, data)) notify_rc =NOTIFY_DONE ; -mutex_unlock (&matrix_dev->lock); -return notify_rc; }
@@ -1363,14 +1337,12 @@ static void vfio_ap_mdev_release(struct mdev_device *mdev) {struct ap_matrix_mdev *matrix_mdev =mdev_get_drvdata (mdev); -mutex_lock (&matrix_dev->lock); -vfio_ap_mdev_unset_kvm (matrix_mdev); -mutex_unlock (&matrix_dev->lock); -vfio_unregister_notifier (mdev_dev (mdev),VFIO_IOMMU_NOTIFY , &matrix_mdev->iommu_notifier);vfio_unregister_notifier (mdev_dev (mdev),VFIO_GROUP_NOTIFY , &matrix_mdev->group_notifier); + +vfio_ap_mdev_unset_kvm (matrix_mdev);module_put (THIS_MODULE ); }
@@ -1412,15 +1384,6 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,break ; } -/* -* If the KVM pointer is in the process of being set, wait until -* the process has completed. -*/ -wait_event_cmd (matrix_mdev->wait_for_kvm, - !matrix_mdev->kvm_busy, -mutex_unlock (&matrix_dev->lock), -mutex_lock (&matrix_dev->lock)); - ret =vfio_ap_mdev_reset_queues (mdev);break ;default :
drivers/s390/crypto/vfio_ap_private.h +0 -2
@@ -83,8 +83,6 @@ struct ap_matrix_mdev {struct ap_matrix matrix;struct notifier_block group_notifier;struct notifier_block iommu_notifier; -bool kvm_busy; -wait_queue_head_t wait_for_kvm;struct kvm *kvm;crypto_hook pqap_hook;struct mdev_device *mdev;
Signature · 1 line
2.30.2
Quoted 8 lines · Tony Krowiak
static void
vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev){drivers/s390/crypto/vfio_ap_ops.c +4
-0+mutex_lock (&matrix_dev->lock); +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock); +down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); + matrix_mdev->kvm->arch.crypto.pqap_hook =NULL ;
drivers/s390/crypto/vfio_ap_ops.c static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · line 1188
Quoted 10 lines
drivers/s390/crypto/vfio_ap_ops.c +3
-0+up_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); +kvm_arch_crypto_clear_masks (matrix_mdev->kvm);mutex_lock (&matrix_dev->lock);vfio_ap_mdev_reset_queues (matrix_mdev->mdev);kvm_put_kvm (matrix_mdev->kvm); matrix_mdev->kvm =NULL ; +mutex_unlock (&matrix_dev->lock); }
Doesn't a flow exit the function with matrix_dev->lock held here?
Write it with 'success oriented flow'
I didn't check if everything makes sense, but it sure looks clean.
Jason
Tony Krowiak
maintainerQuoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Tony Krowiak
static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) {drivers/s390/crypto/vfio_ap_ops.c +6
-0+mutex_lock (&matrix_dev->lock); +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock); +down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); + matrix_mdev->kvm->arch.crypto.pqap_hook =NULL ; +up_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); +kvm_arch_crypto_clear_masks (matrix_mdev->kvm);
Quoted 7 lines
Review
Quoted 6 lines
mutex_lock(&matrix_dev->lock); vfio_ap_mdev_reset_queues(matrix_mdev->mdev); kvm_put_kvm(matrix_mdev->kvm); matrix_mdev->kvm = NULL;(quoted diff) +1
-0+ mutex_unlock(&matrix_dev->lock); }Doesn't a flow exit the function with
matrix_dev->lockheld he
How can that happen? What flow?
Quoted 2 lines
Write it with 'success oriented flow'
I'm not sure what you mean, can you clarify this statement?
Quoted 4 lines
I didn't check if everything makes sense, but it sure looks clean.
Jason
Quoted 21 lines · Tony Krowiak
Quoted 17 lines · Jason Gunthorpe
Quoted 5 lines · Tony Krowiak
static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) {drivers/s390/crypto/vfio_ap_ops.c +2
-0+mutex_lock (&matrix_dev->lock); +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock);drivers/s390/crypto/vfio_ap_ops.c static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · line 1188
Quoted 10 lines
drivers/s390/crypto/vfio_ap_ops.c +5
-0+down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); + matrix_mdev->kvm->arch.crypto.pqap_hook =NULL ; +up_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); +kvm_arch_crypto_clear_masks (matrix_mdev->kvm);mutex_lock (&matrix_dev->lock);vfio_ap_mdev_reset_queues (matrix_mdev->mdev);kvm_put_kvm (matrix_mdev->kvm); matrix_mdev->kvm =NULL ; +mutex_unlock (&matrix_dev->lock); }Doesn't a flow exit the function with
matrix_dev->lockheld he
Quoted 1 line
How can that happen? What flow?
When the if isn't taken
Quoted 3 lines
Quoted 1 line
Write it with 'success oriented flow'
I'm not sure what you mean, can you clarify this statement?
Basically, don't write the bulk of the function under an if statement
mutex_lock(&matrix_dev->lock);
if (!matrix_mdev->kvm || !matrix_mdev->kvm->arch.crypto.crycbd) {
mutex_unlock(&matrix_dev->lock);
return;
}Jason
Tony Krowiak
maintainerQuoted 4 lines · Jason Gunthorpe
Quoted 3 lines · Tony Krowiak
Quoted 1 line · Jason Gunthorpe
On Fri, Jun 25, 2021 at 06:07:58PM -0400, Tony Krowiak wrote:
Quoted 16 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) {drivers/s390/crypto/vfio_ap_ops.c +5
-0+mutex_lock (&matrix_dev->lock); +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock); +down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem); + matrix_mdev->kvm->arch.crypto.pqap_hook =NULL ; +up_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem);Quoted 8 lines
Quoted 8 lines
drivers/s390/crypto/vfio_ap_ops.c static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · line 1188
Quoted 7 lines
drivers/s390/crypto/vfio_ap_ops.c +2
-0+kvm_arch_crypto_clear_masks (matrix_mdev->kvm);mutex_lock (&matrix_dev->lock);vfio_ap_mdev_reset_queues (matrix_mdev->mdev);kvm_put_kvm (matrix_mdev->kvm); matrix_mdev->kvm =NULL ; +mutex_unlock (&matrix_dev->lock); }Doesn't a flow exit the function with
matrix_dev->lockheld he
Yes, you are correct. Stupid mistake.
Quoted 4 lines
Quoted 1 line
How can that happen? What flow?
When the if isn't taken
Quoted 1 line
Quoted 1 line
Write it with 'success oriented flow'
Quoted 8 lines
Quoted 1 line
I'm not sure what you mean, can you clarify this statement?
Basically, don't write the bulk of the function under an if statement
mutex_lock(&matrix_dev->lock);if (!matrix_mdev->kvm || !matrix_mdev->kvm->arch.crypto.crycbd) { mutex_unlock(&matrix_dev->lock); return; }
Sure.
Quoted 3 lines
Jason
Halil Pasic
maintainerOn Fri, 25 Jun 2021 18:07:58 -0400
Tony Krowiak <[email protected]> wrote:
What is a suitable base for this patch. I've tried the usual suspects, but none of them worked.
Quoted 8 lines
The fix to resolve a lockdep splat while handling the
VFIO_GROUP_NOTIFY_SET_KVMevent introduced akvm_busyflag indicating that thevfio_apdevice driver is busy setting or unsetting the KVM pointer. A wait queue was employed to allow functions requiring access to the KVM pointer to wait for thekvm_busyflag to be cleared. For the duration of the wait period, the mdev lock was unlocked then acquired again after thekvm_busyflag was cleared. This got rid of the lockdep report, but didn't really resolve the problem.
Can you please elaborate on the last point. You mean that we can have circular locking even after 0cc00c8d4050, but instead of getting stuck in on a lock we will get stuck on wait_event_cmd()? If that is it, please state it clearly in the description, and if you can to it in the short description.
Quoted 20 lines
This patch removes the the
kvm_busyflag and wait queue as they are not necessary to resolve the lockdep splat problem. The wait queue was introduced to prevent changes to the matrix used to update the guest's AP configuration. The idea was that whenever an adapter, domain or control domain was being assigned to or unassigned from the matrix, the function would wait until the group notifier function was no longer busy with the KVM pointer.The thing is, the KVM pointer value (
matrix_mdev->kvm) is always set and cleared while holding thematrix_dev->lockmutex. The assignment and unassignment interfaces also lock thematrix_dev->lockmutex prior to checking whether thematrix_mdev->kvmpointer is set and if so, returns the-EBUSYerror from the function. Consequently, there is no chance for an update to the matrix to occur while the guest's AP configuration is being updated.Fixes 0cc00c8d4050 ("s390/vfio-ap: fix circular lockdep when setting/clearing crypto masks")Signed-off-by Tony Krowiak <[email protected]>
Tony Krowiak
maintainerQuoted 5 lines · Halil Pasic
On Fri, 25 Jun 2021 18:07:58 -0400
Tony Krowiak <[email protected]> wrote:What is a suitable base for this patch. I've tried the usual suspects, but none of them worked.
I discovered what the problem is here. The patch is based on our master branch along with the two pre-requisite patches that were recently reviewed and are currently being merged. The two patches
of which I speak are:
* [PATCH v6 1/2] s390/vfio-ap: clean up mdev resources when remove callback invoked Message ID: <[email protected]>
* [PATCH v6 2/2] s390/vfio-ap: r/w lock for PQAP interception handler function pointer <[email protected]>
I probably should have included those along with this one.
Quoted 9 lines
Quoted 9 lines
The fix to resolve a lockdep splat while handling the
VFIO_GROUP_NOTIFY_SET_KVMevent introduced akvm_busyflag indicating that thevfio_apdevice driver is busy setting or unsetting the KVM pointer. A wait queue was employed to allow functions requiring access to the KVM pointer to wait for thekvm_busyflag to be cleared. For the duration of the wait period, the mdev lock was unlocked then acquired again after thekvm_busyflag was cleared. This got rid of the lockdep report, but didn't really resolve the problem.
Quoted 5 lines
Can you please elaborate on the last point. You mean that we can have circular locking even after
0cc00c8d4050, but instead of getting stuck in on a lock we will get stuck onwait_event_cmd()? If that is it, please state it clearly in the description, and if you can to it in the short description.
This patch was in response to the following review comments made by Jason Gunthorpe:
* Message ID: <[email protected]>
"... the kvm_busy should be replaced by a proper rwsem, don't try to open code locks like that - it just defeats lockdep analysis".
* Message ID: <[email protected]> "Usually when people start open coding locks it is often because lockdep complained. Open coding a lock makes lockdep stop because the lockdep code
is removed, but it doesn't fix anything. The kvm_busy should be replaced by a proper rwsem, don't try to open code locks like that - it just defeats lockdep analysis."
I will paraphrase and include the information from Jason's comments in the description.
Quoted 19 lines
Quoted 19 lines
This patch removes the the
kvm_busyflag and wait queue as they are not necessary to resolve the lockdep splat problem. The wait queue was introduced to prevent changes to the matrix used to update the guest's AP configuration. The idea was that whenever an adapter, domain or control domain was being assigned to or unassigned from the matrix, the function would wait until the group notifier function was no longer busy with the KVM pointer.The thing is, the KVM pointer value (
matrix_mdev->kvm) is always set and cleared while holding thematrix_dev->lockmutex. The assignment and unassignment interfaces also lock thematrix_dev->lockmutex prior to checking whether thematrix_mdev->kvmpointer is set and if so, returns the-EBUSYerror from the function. Consequently, there is no chance for an update to the matrix to occur while the guest's AP configuration is being updated.Fixes 0cc00c8d4050 ("s390/vfio-ap: fix circular lockdep when setting/clearing crypto masks")Signed-off-by Tony Krowiak <[email protected]>
Halil Pasic
maintainerQuoted 15 lines · On Wed, 30 Jun 2021 10:31:22 -0400 Tony Krowiak <[email protected]> wrote:
Quoted 5 lines · Halil Pasic
On Fri, 25 Jun 2021 18:07:58 -0400
Tony Krowiak <[email protected]> wrote:What is a suitable base for this patch. I've tried the usual suspects, but none of them worked.
I discovered what the problem is here. The patch is based on our master branch along with the two pre-requisite patches that were recently reviewed and are currently being merged. The two patches of which I speak are:
* [PATCH v6 1/2] s390/vfio-ap: clean up mdev resources when remove callback invoked
Message ID: <[email protected]>
Quoted 5 lines
* [PATCH v6 2/2] s390/vfio-ap: r/w lock for PQAP interception handler function pointer <[email protected]>
I probably should have included those along with this one.
Either that, or state in the cover letter that those are prerequisites.
Quoted 33 lines
Quoted 10 lines
Quoted 10 lines
The fix to resolve a lockdep splat while handling the
VFIO_GROUP_NOTIFY_SET_KVMevent introduced akvm_busyflag indicating that thevfio_apdevice driver is busy setting or unsetting the KVM pointer. A wait queue was employed to allow functions requiring access to the KVM pointer to wait for thekvm_busyflag to be cleared. For the duration of the wait period, the mdev lock was unlocked then acquired again after thekvm_busyflag was cleared. This got rid of the lockdep report, but didn't really resolve the problem.Quoted 5 lines
Can you please elaborate on the last point. You mean that we can have circular locking even after
0cc00c8d4050, but instead of getting stuck in on a lock we will get stuck onwait_event_cmd()? If that is it, please state it clearly in the description, and if you can to it in the short description.This patch was in response to the following review comments made by Jason Gunthorpe:
* Message ID: <[email protected]>
"... thekvm_busyshould be replaced by a proper rwsem, don't try to open code locks like that - it just defeats lockdep analysis".* Message ID: <[email protected]> "Usually when people start open coding locks it is often because lockdep complained. Open coding a lock makes lockdep stop because the lockdep code
is removed, but it doesn't fix anything. Thekvm_busyshould be replaced by a proper rwsem, don't try to open code locks like that - it just defeats lockdep analysis."
Quoted 3 lines
I will paraphrase and include the information from Jason's comments in the description.
This does not answer my questions.
I'm in favor of Jason's proposal, because it is much easier to comprehend simple rwsem protected than a mutex + wait_queue dance.
I think Jason was talking about open coding locks in general. I don't consider it as proof of commit 0cc00c8d4050 not doing what it advertised. You can add a Suggested-by tag if you like, but you should be able to tell us what is the merit of your patch.
Regards,
Halil
Tony Krowiak
maintainerQuoted 17 lines · Halil Pasic
Quoted 14 lines · On Wed, 30 Jun 2021 10:31:22 -0400 Tony Krowiak <[email protected]> wrote:
Quoted 5 lines · Halil Pasic
On Fri, 25 Jun 2021 18:07:58 -0400
Tony Krowiak <[email protected]> wrote:What is a suitable base for this patch. I've tried the usual suspects, but none of them worked.
I discovered what the problem is here. The patch is based on our master branch along with the two pre-requisite patches that were recently reviewed and are currently being merged. The two patches of which I speak are:
* [PATCH v6 1/2] s390/vfio-ap: clean up mdev resources when remove callback invoked
Message ID: <[email protected]>
Quoted 21 lines
Quoted 5 lines
* [PATCH v6 2/2] s390/vfio-ap: r/w lock for PQAP interception handler function pointer <[email protected]>
I probably should have included those along with this one.
Either that, or state in the cover letter that those are prerequisites.
Quoted 14 lines
Quoted 14 lines
Quoted 9 lines
The fix to resolve a lockdep splat while handling the
VFIO_GROUP_NOTIFY_SET_KVMevent introduced akvm_busyflag indicating that thevfio_apdevice driver is busy setting or unsetting the KVM pointer. A wait queue was employed to allow functions requiring access to the KVM pointer to wait for thekvm_busyflag to be cleared. For the duration of the wait period, the mdev lock was unlocked then acquired again after thekvm_busyflag was cleared. This got rid of the lockdep report, but didn't really resolve the problem.Can you please elaborate on the last point. You mean that we can have circular locking even after
0cc00c8d4050, but instead of getting stuck in on a lock we will get stuck onwait_event_cmd()? If that is it, please state it clearly in the description, and if you can to it in the short description.
I created this patch in response to Jason G's review comments I copied below. He did not mention anything about getting stuck in a wait_event_cmd(), so you may want to ask him about that. To answer your
question, I don't see how we can get stuck in a wait_event_cmd() unless one of the functions that set the matrix_mdev->kvm_busy flag does not complete for some reason.
You asked for me to elaborate on the last point in the preceding paragraph; I did so in my response to your comments/question above.
Quoted 18 lines
Quoted 18 lines
Quoted 1 line
This patch was in response to the following review comments made by Jason Gunthorpe:
* Message ID: <[email protected]>
"... thekvm_busyshould be replaced by a proper rwsem, don't try to open code locks like that - it just defeats lockdep analysis".* Message ID: <[email protected]> "Usually when people start open coding locks it is often because lockdep complained. Open coding a lock makes lockdep stop because the lockdep code
is removed, but it doesn't fix anything. Thekvm_busyshould be replaced by a proper rwsem, don't try to open code locks like that - it just defeats lockdep analysis."
Quoted 4 lines
Quoted 3 lines
I will paraphrase and include the information from Jason's comments in the description.
This does not answer my questions.
See above.
Quoted 3 lines
I'm in favor of Jason's proposal, because it is much easier to comprehend simple rwsem protected than a mutex +
wait_queuedance.
That is a matter of opinion. I have no trouble understanding the "mutex + wait_queue dance", but then again, I wrote the code, so maybe that is why.
Quoted 3 lines
I think Jason was talking about open coding locks in general.
That may be so, but his comments were in support of his statement that the mutex + wait_queue did not resolve the issue reported vai the lockdep splat because it turned off lockdep.
Quoted 3 lines
I don't
consider it as proof of commit0cc00c8d4050not doing what it advertised.
I think I agree with this statement. Maybe I misunderstood what Jason meant by "open coding locks like that". Since that comment directly related to replacing the kvm->busy, I assumed that he was referring to the "mutex + wait_queue dance" as you called it. I probably should have probed deeper to discern exactly what Jason meant by "open coding locks". I took Jason at his word that "the kvm->busy ... just defeats lockdep analysis" because I don't have deep knowledge about lockdep.
Even if the kvm->busy does defeat lockdep analysis, I still believe it fixes the problem for which commit 0cc00c8d4050 was created. If we don't hold the matrix_dev->lock while the kvm->lock is held during update of the guest's matrix, lockdep does not report a problem. That is proven by the tests of this patch.
If commit 0cc00c8d4050 does in fact resolve the issue for which it was created, then there really is no need for this patch. It certainly would reduce the amount of change that will be required to integrate this patch with the "s390/vfio-ap: dynamic configuration support" patch series currently under review.
Quoted 2 lines
You can add a Suggested-by tag if you like, but you should be able to tell us what is the merit of your patch.
The patch removes the matrix_mdev->kvm_busy flag and the wait_queue. The reason for removing them was because, as Jason stated, "the kvm->busy ... just defeats lockdep analysis".
Quoted 4 lines
Regards,
Halil
Quoted 6 lines · Tony Krowiak
Quoted 1 line
I think Jason was talking about open coding locks in general.
That may be so, but his comments were in support of his statement that the mutex +
wait_queuedid not resolve the issue reported vai the lockdep splat because it turned off lockdep.
Rgiht, if this used to be proper locks and lockdep complained then whatever potential deadlock it found is not magically removed by going to a wait_queue. It just removes the lockdep annotations that would identify the issue early.
This is why people should not open code locks, it completely defeats lockdep. That alone is merit enough for this patch.
Jason
Tony Krowiak
maintainerQuoted 7 lines · Jason Gunthorpe
Quoted 5 lines · Tony Krowiak
Quoted 1 line
I think Jason was talking about open coding locks in general.
That may be so, but his comments were in support of his statement that the mutex +
wait_queuedid not resolve the issue reported vai the lockdep splat because it turned off lockdep.
Quoted 7 lines
Rgiht, if this used to be proper locks and lockdep complained then whatever potential deadlock it found is not magically removed by going to a
wait_queue. It just removes the lockdep annotations that would identify the issue early.This is why people should not open code locks, it completely defeats lockdep. That alone is merit enough for this patch.
When you use the phrase "open code locks", to what are you specifically referring? I am confused by the use of the phrase "open code" in this context because open coding, at least as I understand it, has to do with data analysis.
Quoted 2 lines
Jason
Quoted 15 lines · Tony Krowiak
Quoted 12 lines · Jason Gunthorpe
Quoted 5 lines · Tony Krowiak
Quoted 1 line
I think Jason was talking about open coding locks in general.
That may be so, but his comments were in support of his statement that the mutex +
wait_queuedid not resolve the issue reported vai the lockdep splat because it turned off lockdep.Rgiht, if this used to be proper locks and lockdep complained then whatever potential deadlock it found is not magically removed by going to a
wait_queue. It just removes the lockdep annotations that would identify the issue early.
Quoted 7 lines
Quoted 2 lines
This is why people should not open code locks, it completely defeats lockdep. That alone is merit enough for this patch.
When you use the phrase "open code locks", to what are you specifically referring? I am confused by the use of the phrase "open code" in this context because open coding, at least as I understand it, has to do with data analysis.
"open code" here means you write the algorithm of a standard lock in your own functions instead of calling the standard library.
Testing/setting the busy and sleeping on a wait_event is exactly a standard lock.
Ie if I write
for (len = 0; str[len] != 0; len++)
;
Then I have open coded strlen()
Jason
Tony Krowiak
maintainerQuoted 27 lines · Jason Gunthorpe
Quoted 14 lines · Tony Krowiak
Quoted 12 lines · Jason Gunthorpe
Quoted 5 lines · Tony Krowiak
Quoted 1 line
I think Jason was talking about open coding locks in general.
That may be so, but his comments were in support of his statement that the mutex +
wait_queuedid not resolve the issue reported vai the lockdep splat because it turned off lockdep.Rgiht, if this used to be proper locks and lockdep complained then whatever potential deadlock it found is not magically removed by going to a
wait_queue. It just removes the lockdep annotations that would identify the issue early.Quoted 6 lines
Quoted 2 lines
This is why people should not open code locks, it completely defeats lockdep. That alone is merit enough for this patch.
When you use the phrase "open code locks", to what are you specifically referring? I am confused by the use of the phrase "open code" in this context because open coding, at least as I understand it, has to do with data analysis.
"open code" here means you write the algorithm of a standard lock in your own functions instead of calling the standard library.
Testing/setting the busy and sleeping on a
wait_eventis exactly a standard lock.
Quoted 8 lines
Ie if I write
for (len = 0; str[len] != 0; len++) ;Then I have open coded
strlen()Jason
Thanks for the explanation.
Jason J. Herne
maintainerQuoted 154 lines · Tony Krowiak
The fix to resolve a lockdep splat while handling the
VFIO_GROUP_NOTIFY_SET_KVMevent introduced akvm_busyflag indicating that thevfio_apdevice driver is busy setting or unsetting the KVM pointer. A wait queue was employed to allow functions requiring access to the KVM pointer to wait for thekvm_busyflag to be cleared. For the duration of the wait period, the mdev lock was unlocked then acquired again after thekvm_busyflag was cleared. This got rid of the lockdep report, but didn't really resolve the problem.This patch removes the the
kvm_busyflag and wait queue as they are not necessary to resolve the lockdep splat problem. The wait queue was introduced to prevent changes to the matrix used to update the guest's AP configuration. The idea was that whenever an adapter, domain or control domain was being assigned to or unassigned from the matrix, the function would wait until the group notifier function was no longer busy with the KVM pointer.The thing is, the KVM pointer value (
matrix_mdev->kvm) is always set and cleared while holding thematrix_dev->lockmutex. The assignment and unassignment interfaces also lock thematrix_dev->lockmutex prior to checking whether thematrix_mdev->kvmpointer is set and if so, returns the-EBUSYerror from the function. Consequently, there is no chance for an update to the matrix to occur while the guest's AP configuration is being updated.Fixes 0cc00c8d4050 ("s390/vfio-ap: fix circular lockdep when setting/clearing crypto masks")Signed-off-by Tony Krowiak <[email protected]>--- drivers/s390/crypto/vfio_ap_ops.c | 77 +++++++-------------------- drivers/s390/crypto/vfio_ap_private.h | 2 - 2 files changed, 20 insertions(+), 59 deletions(-)drivers/s390/crypto/vfio_ap_ops.c +11
-26@@ -294,15 +294,6 @@ static int handle_pqap(struct kvm_vcpu *vcpu) matrix_mdev =container_of (vcpu->kvm->arch.crypto.pqap_hook, struct ap_matrix_mdev, pqap_hook); -/* -* If the KVM pointer is in the process of being set, wait until the -* process has completed. -*/ -wait_event_cmd (matrix_mdev->wait_for_kvm, - !matrix_mdev->kvm_busy, -mutex_unlock (&matrix_dev->lock), -mutex_lock (&matrix_dev->lock)); -/* If the there is no guest using the mdev, there is nothing to do */ if (!matrix_mdev->kvm) goto out_unlock;@@ -350,7 +341,6 @@ static int vfio_ap_mdev_create(struct mdev_device *mdev) matrix_mdev->mdev = mdev;vfio_ap_matrix_init (&matrix_dev->info, &matrix_mdev->matrix); -init_waitqueue_head (&matrix_mdev->wait_for_kvm);mdev_set_drvdata (mdev, matrix_mdev); matrix_mdev->pqap_hook = handle_pqap;mutex_lock (&matrix_dev->lock);@@ -623,7 +613,7 @@ static ssize_t assign_adapter_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of adapter */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }@@ -696,7 +686,7 @@ static ssize_t unassign_adapter_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of adapter */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }@@ -786,7 +776,7 @@ static ssize_t assign_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * assignment of domain */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }@@ -854,7 +844,7 @@ static ssize_t unassign_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of domain */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }@@ -908,7 +898,7 @@ static ssize_t assign_control_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * assignment of control domain. */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }@@ -967,7 +957,7 @@ static ssize_t unassign_control_domain_store(struct device *dev,* If the KVM pointer is in flux or the guest is running, disallow * un-assignment of control domain. */ -if (matrix_mdev->kvm_busy || matrix_mdev->kvm) { +if (matrix_mdev->kvm) { ret = -EBUSY ; goto done; }@@ -1108,14 +1098,17 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,struct ap_matrix_mdev *m;if (kvm->arch.crypto.crycbd) { +mutex_lock (&matrix_dev->lock); +list_for_each_entry (m, &matrix_dev->mdev_list, node) { -if (m != matrix_mdev && m->kvm == kvm) +if (m != matrix_mdev && m->kvm == kvm) { +mutex_unlock (&matrix_dev->lock);return -EPERM ; + } }kvm_get_kvm (kvm); matrix_mdev->kvm = kvm; - matrix_mdev->kvm_busy = true;mutex_unlock (&matrix_dev->lock);down_write (&matrix_mdev->kvm->arch.crypto.pqap_hook_rwsem);@@ -1126,10 +1119,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, matrix_mdev->matrix.apm, matrix_mdev->matrix.aqm, matrix_mdev->matrix.adm); - -mutex_lock (&matrix_dev->lock); - matrix_mdev->kvm_busy = false; -wake_up_all (&matrix_mdev->wait_for_kvm); }return 0 ;@@ -1181,33 +1170,21 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb,*/ static void vfio_ap_mdev_unset_kvm (struct ap_matrix_mdev *matrix_mdev) { -/* -* If the KVM pointer is in the process of being set, wait until the -* process has completed. -*/
drivers/s390/crypto/vfio_ap_ops.c static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · line 1176
Quoted 10 lines
drivers/s390/crypto/vfio_ap_ops.c +2
-6-wait_event_cmd (matrix_mdev->wait_for_kvm, - !matrix_mdev->kvm_busy, -mutex_unlock (&matrix_dev->lock), -mutex_lock (&matrix_dev->lock)); +mutex_lock (&matrix_dev->lock); -if (matrix_mdev->kvm) { - matrix_mdev->kvm_busy = true; +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock);
If this function cannot be entered concurrently on separate threads then I think we can remove this mutex_unlock of matrix_dev->lock, (and the above mutex_lock) All that happens while holding the lock is the examination of the matrix_mdev->kvm pointer and then the subsequent examination of matrix_mdev->kvm->arch.crypto.crycbd. And since this function is the only place that the kvm pointer is NULLed I don't see how the kvm pointer could go away between the two parts of the conditional. Again, this is only true if this function cannot be entered concurrently.
Signature · 1 line
-- Jason J. Herne ([email protected])
Quoted 9 lines · Jason J. Herne
drivers/s390/crypto/vfio_ap_ops.c static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · line 1176
Quoted 5 lines
drivers/s390/crypto/vfio_ap_ops.c +2
-2+mutex_lock (&matrix_dev->lock); -if (matrix_mdev->kvm) { - matrix_mdev->kvm_busy = true; +if ((matrix_mdev->kvm) && (matrix_mdev->kvm->arch.crypto.crycbd)) {mutex_unlock (&matrix_dev->lock);If this function cannot be entered concurrently on separate threads then I think we can remove this
mutex_unlockofmatrix_dev->lock,
Don't remove locking around data. If the data is written under a mutex it should be read under a mutex too.
Jason