KVM: arm64: Handle KVM_REQ_VM_DEAD in vgic_init test
cover KVM: arm64: Handle KVM_REQ_VM_DEAD in vgic_init test
Mark reported that df5fd75ee305 ("KVM: arm64: Don't eagerly teardown the vgic on init error") causes vgic_init test assertion failure. This is due to the fact that now an incomplete vgic setup causes a KVM_REQ_VM_DEAD request to be sent. vgic_init test checks such kind of incomplete setups. As a consequence some KVM IOCTL's fail on the clean up path (kvm_vm_free) and cause assertion failures in the test.
This series fixes that by checking the VM state and avoid to call KVM IOCTL's after the occcurence of such failure.
Best regards
Eric
This series can be found at:
https://github.com/eauger/linux/tree/vgic_init_dead_v1
Eric Auger (3):
KVM: selftests: Introduce vm_dead()
KVM: selftests: Introduce kvm_vm_dead_free
KVM: selftests: Handle dead VM in vgic_init test
.../testing/selftests/kvm/aarch64/vgic_init.c | 41 +++++++++++-------- .../testing/selftests/kvm/include/kvm_util.h | 28 +++++++++---- tools/testing/selftests/kvm/lib/kvm_util.c | 25 ++++++++--- 3 files changed, 62 insertions(+), 32 deletions(-)
Signature · 1 line
2.41.0
PATCH 1/3 KVM: selftests: Introduce vm_dead()
Introduce a new helper that detects whether the VM was turned dead by a KVM_REQ_VM_DEAD request.
--- .../testing/selftests/kvm/include/kvm_util.h | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-)
tools/testing/selftests/kvm/include/kvm_util.h +19 -8
@@ -299,13 +299,26 @@ static __always_inline void static_assert_is_vm(struct kvm_vm *vm) { } })/* -* Assert that a VM or vCPU ioctl() succeeded, with extra magic to detect if -* the ioctl() failed because KVM killed/bugged the VM. To detect a dead VM, -* probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM since before -* selftests existed and (b) should never outright fail, i.e. is supposed to -* return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the +* To detect a dead VM, probe KVM_CAP_USER_MEMORY, which (a) has been supported by KVM +* since before selftests existed and (b) should never outright fail, i.e. is supposed +* to return 0 or 1. If KVM kills a VM, KVM returns -EIO for all ioctl()s for the * VM and its vCPUs, including KVM_CHECK_EXTENSION. */ +static inline bool vm_dead (struct kvm_vm *vm) +{ +int ret =__vm_ioctl (vm,KVM_CHECK_EXTENSION , (void *)KVM_CAP_USER_MEMORY ); + +if (ret <1 ) { +TEST_ASSERT (errno ==EIO ,"KVM killed the VM, should return -EIO" ); +return true; + } +return false; +} + +/* +* Assert that a VM or vCPU ioctl() succeeded, also handling the case when +* the ioctl() failed because KVM killed/bugged the VM. +*/ #define __TEST_ASSERT_VM_VCPU_IOCTL (cond, name, ret, vm) \ do { \ int __errno = errno; \
@@ -315,9 +328,7 @@ do { \if (cond) \break ; \ \ -if (errno ==EIO && \ -__vm_ioctl (vm,KVM_CHECK_EXTENSION , (void *)KVM_CAP_USER_MEMORY ) <0 ) { \ -TEST_ASSERT (errno ==EIO ,"KVM killed the VM, should return -EIO" ); \ +if (vm_dead (vm)) { \TEST_FAIL ("KVM killed/bugged the VM, check the kernel log for clues" ); \ } \ errno = __errno; \
Signature · 1 line
2.41.0
PATCH 2/3 KVM: selftests: Introduce kvm_vm_dead_free
In reply to Eric AugerIn case a KVM_REQ_VM_DEAD request was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where the kvm_vm_free() fails. Let's add a new kvm_vm_dead_free() helper that does all the deallocation besides the KVM_SET_USER_MEMORY_REGION2 ioctl.
--- .../testing/selftests/kvm/include/kvm_util.h | 1 + tools/testing/selftests/kvm/lib/kvm_util.c | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-)
tools/testing/selftests/kvm/include/kvm_util.h +1 -0
@@ -437,6 +437,7 @@ void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);const char *vm_guest_mode_string (uint32_t i);void kvm_vm_free (struct kvm_vm *vmp); +void kvm_vm_dead_free (struct kvm_vm *vmp);void kvm_vm_restart (struct kvm_vm *vmp);void kvm_vm_release (struct kvm_vm *vmp);void kvm_vm_elf_load (struct kvm_vm *vm,const char *filename);
tools/testing/selftests/kvm/lib/kvm_util.c +19 -6
@@ -712,7 +712,8 @@ void kvm_vm_release(struct kvm_vm *vmp) }static void __vm_mem_region_delete (struct kvm_vm *vm, -struct userspace_mem_region *region) +struct userspace_mem_region *region, +bool dead) {int ret;
@@ -720,8 +721,10 @@ static void __vm_mem_region_delete(struct kvm_vm *vm,rb_erase (®ion->hva_node, &vm->regions.hva_tree);hash_del (®ion->slot_node); - region->region.memory_size =0 ; -vm_ioctl (vm,KVM_SET_USER_MEMORY_REGION2 , ®ion->region); +if (!dead) { + region->region.memory_size =0 ; +vm_ioctl (vm,KVM_SET_USER_MEMORY_REGION2 , ®ion->region); + }sparsebit_free (®ion->unused_phy_pages);sparsebit_free (®ion->protected_phy_pages);
@@ -742,7 +745,7 @@ static void __vm_mem_region_delete(struct kvm_vm *vm,/* * Destroys and frees the VM pointed to by vmp. */ -void kvm_vm_free (struct kvm_vm *vmp) +static void __kvm_vm_free (struct kvm_vm *vmp,bool dead) {int ctr;struct hlist_node *node;
@@ -759,7 +762,7 @@ void kvm_vm_free(struct kvm_vm *vmp)/* Free userspace_mem_regions. */ hash_for_each_safe (vmp->regions.slot_hash, ctr, node, region, slot_node) -__vm_mem_region_delete (vmp, region); +__vm_mem_region_delete (vmp, region, dead);/* Free sparsebit arrays. */ sparsebit_free (&vmp->vpages_valid);
@@ -771,6 +774,16 @@ void kvm_vm_free(struct kvm_vm *vmp)free (vmp); } +void kvm_vm_free (struct kvm_vm *vmp) +{ +__kvm_vm_free (vmp, false); +} + +void kvm_vm_dead_free (struct kvm_vm *vmp) +{ +__kvm_vm_free (vmp, true); +} +int kvm_memfd_alloc (size_t size,bool hugepages) {int memfd_flags =MFD_CLOEXEC ;
@@ -1197,7 +1210,7 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)*/ void vm_mem_region_delete (struct kvm_vm *vm,uint32_t slot) { -__vm_mem_region_delete (vm,memslot2region (vm, slot)); +__vm_mem_region_delete (vm,memslot2region (vm, slot), false); } void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
Signature · 1 line
2.41.0
Sean Christopherson
reviewerRe: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.
Please no. I don't want to bleed the kvm->vm_dead behavior all over selftests. The hack in __TEST_ASSERT_VM_VCPU_IOCTL() is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.
IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return -EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.
Marking a VM dead should be a _very_ rare event; it's not something that I think we should encourage, i.e. we shouldn't make it easier to deal with. Ideally, use of kvm_vm_dead() should be limited to things like sev_vm_move_enc_context_from(), where KVM needs to prever accessing the source VM to protect the host. IMO, the vGIC case and x86's enter_smm() are hacks. E.g. I don't see any reason why the enter_smm() case can't synthesize a triple fault.
If memory utilization is a concern for the vgic_init test (which seems unlikely), I would much rather solve that problem by expanding KVM selftests support for the selftests harness so that each testcase runs as a child process.
Re: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 5 lines · Sean Christopherson
If memory utilization is a concern for the
vgic_inittest (which seems unlikely), I would much rather solve that problem by expanding KVM selftests support for the selftests harness so that each testcase runs as a child process.
That would in general be useful, having individual test cases be visible is helpful from a UI and triage point of view and having them all run even if one of them has failed even more so.
Re: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
In reply to Sean ChristophersonQuoted 12 lines · Sean Christopherson
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.Please no. I don't want to bleed the
kvm->vm_deadbehavior all over selftests. The hack in__TEST_ASSERT_VM_VCPU_IOCTL()is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.
Quoted 4 lines
IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return
-EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.
Encouraging tests to explicitly leak resources to fudge around assertions in the selftests library seems off to me.
IMO, the better approach would be to provide a helper that gives the impression of freeing the VM but implicitly leaks it, paired with some reasoning for it.
tools/testing/selftests/kvm/include/kvm_util.h +1 -0
@@ -426,6 +426,7 @@ void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);const char *vm_guest_mode_string (uint32_t i);void kvm_vm_free (struct kvm_vm *vmp); +void __kvm_vm_free_dead (struct kvm_vm *vmp);void kvm_vm_restart (struct kvm_vm *vmp);void kvm_vm_release (struct kvm_vm *vmp);void kvm_vm_elf_load (struct kvm_vm *vm,const char *filename);
tools/testing/selftests/kvm/lib/kvm_util.c +15 -0
@@ -771,6 +771,21 @@ void kvm_vm_free(struct kvm_vm *vmp)free (vmp); } +/* +* For use with the *extremely* uncommon case that a test expects a VM to be +* marked as dead. +* +* Deliberately leak the VM to avoid hacking up kvm_vm_free() to cope with a +* dead VM while giving the outward impression of 'doing the right thing'. +*/ +void __kvm_vm_dead_free (struct kvm_vm *vmp) +{ +if (!vmp) +return ; + +TEST_ASSERT (vm_dead (vmp)); +} +int kvm_memfd_alloc (size_t size,bool hugepages) {int memfd_flags =MFD_CLOEXEC ;
Quoted 6 lines
Marking a VM dead should be a _very_ rare event; it's not something that I think we should encourage, i.e. we shouldn't make it easier to deal with. Ideally, use of
kvm_vm_dead()should be limited to things likesev_vm_move_enc_context_from(), where KVM needs to prever accessing the source VM to protect the host. IMO, the vGIC case and x86'senter_smm()are hacks. E.g. I don't see any reason why theenter_smm()case can't synthesize a triple fault.
The VGIC case is at least better than the alternative of slapping bandaids all over the shop to cope with a half-baked VM and ensure we tear it down correctly. Userspace is far up shit creek at the point the VM is marked as dead, so I don't see any value in hobbling along afterwards.
Signature · 2 lines
Thanks, Oliver
Sean Christopherson
reviewerRe: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 13 lines · Oliver Upton
Quoted 12 lines · Sean Christopherson
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.Please no. I don't want to bleed the
kvm->vm_deadbehavior all over selftests. The hack in__TEST_ASSERT_VM_VCPU_IOCTL()is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.
Quoted 7 lines
Quoted 4 lines
IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return
-EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.Encouraging tests to explicitly leak resources to fudge around assertions in the selftests library seems off to me.
I don't disagree, but I really, really don't want to add vm_dead().
Quoted 3 lines
IMO, the better approach would be to provide a helper that gives the impression of freeing the VM but implicitly leaks it, paired with some reasoning for it.
Actually, duh. There's no need to manually delete KVM memslots for *any* VM, dead or alive. Just skip that unconditionally when freeing the VM, and then the vGIC test just needs to assert on -EIO instead -ENXIO/-EBUSY.
---
From: Sean Christopherson <[email protected]>
Date: Thu, 7 Nov 2024 11:39:59 -0800
Subject: [PATCH] KVM: selftests: Don't bother deleting memslots in KVM when
freeing VMs
When freeing a VM, don't call into KVM to manually remove each memslot, simply cleanup and free any userspace assets associated with the memory region. KVM is ultimately responsible for ensuring kernel resources are freed when the VM is destroyed, deleting memslots one-by-one is unnecessarily slow, and unless a test is already leaking the VM fd, the VM will be destroyed when kvm_vm_release() is called.
Not deleting KVM's memslot also allows cleaning up dead VMs without having to care whether or not the to-be-freed VM is dead or alive.
--- tools/testing/selftests/kvm/lib/kvm_util.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-)
tools/testing/selftests/kvm/lib/kvm_util.c +6 -4
@@ -720,9 +720,6 @@ static void __vm_mem_region_delete(struct kvm_vm *vm,rb_erase (®ion->hva_node, &vm->regions.hva_tree);hash_del (®ion->slot_node); - region->region.memory_size =0 ; -vm_ioctl (vm,KVM_SET_USER_MEMORY_REGION2 , ®ion->region); -sparsebit_free (®ion->unused_phy_pages);sparsebit_free (®ion->protected_phy_pages); ret =munmap (region->mmap_start, region->mmap_size);
@@ -1197,7 +1194,12 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)*/ void vm_mem_region_delete (struct kvm_vm *vm,uint32_t slot) { -__vm_mem_region_delete (vm,memslot2region (vm, slot)); +struct userspace_mem_region *region =memslot2region (vm, slot); + + region->region.memory_size =0 ; +vm_ioctl (vm,KVM_SET_USER_MEMORY_REGION2 , ®ion->region); + +__vm_mem_region_delete (vm, region); } void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
base-commit: c88416ba074a8913cf6d61b789dd834bbca6681c
Quoted 7 lines
Quoted 6 lines
Marking a VM dead should be a _very_ rare event; it's not something that I think we should encourage, i.e. we shouldn't make it easier to deal with. Ideally, use of
kvm_vm_dead()should be limited to things likesev_vm_move_enc_context_from(), where KVM needs to prever accessing the source VM to protect the host. IMO, the vGIC case and x86'senter_smm()are hacks. E.g. I don't see any reason why theenter_smm()case can't synthesize a triple fault.
Quoted 5 lines
The VGIC case is at least better than the alternative of slapping bandaids all over the shop to cope with a half-baked VM and ensure we tear it down correctly. Userspace is far up shit creek at the point the VM is marked as dead, so I don't see any value in hobbling along afterwards.
Again, I don't disagree, but I don't want to normalize shooting the VM on errors.
Oliver Upton
Reviewed-byRe: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 19 lines · Sean Christopherson
Quoted 18 lines · Oliver Upton
Quoted 16 lines · Sean Christopherson
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.Please no. I don't want to bleed the
kvm->vm_deadbehavior all over selftests. The hack in__TEST_ASSERT_VM_VCPU_IOCTL()is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return
-EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.
Quoted 4 lines
Quoted 2 lines
Encouraging tests to explicitly leak resources to fudge around assertions in the selftests library seems off to me.
I don't disagree, but I really, really don't want to add
vm_dead().
It'd still be valuable to test that the VM is properly dead and subsequent ioctls also return EIO, but I understand the hesitation.
Quoted 7 lines
Quoted 3 lines
IMO, the better approach would be to provide a helper that gives the impression of freeing the VM but implicitly leaks it, paired with some reasoning for it.
Actually, duh. There's no need to manually delete KVM memslots for *any* VM, dead or alive. Just skip that unconditionally when freeing the VM, and then the vGIC test just needs to assert on
-EIOinstead -ENXIO/-EBUSY.
Yeah, that'd tighten up the assertions a bit more to the exact ioctl where we expect the VM to go sideways.
Quoted 13 lines
---
From: Sean Christopherson <[email protected]>
Date: Thu, 7 Nov 2024 11:39:59 -0800
Subject: [PATCH] KVM: selftests: Don't bother deleting memslots in KVM when
freeing VMsWhen freeing a VM, don't call into KVM to manually remove each memslot, simply cleanup and free any userspace assets associated with the memory region. KVM is ultimately responsible for ensuring kernel resources are freed when the VM is destroyed, deleting memslots one-by-one is unnecessarily slow, and unless a test is already leaking the VM fd, the VM will be destroyed when
kvm_vm_release()is called.
Quoted 2 lines
Not deleting KVM's memslot also allows cleaning up dead VMs without having to care whether or not the to-be-freed VM is dead or alive.
Can you add a comment to kvm_vm_free() about why we want to avoid ioctls in that helper? It'd help discourage this situation from happening again in the future in the unlikely case someone wants to park an ioctl there.
Quoted 3 lines
Reported-by Eric Auger <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Sean Christopherson <[email protected]>
I'm assuming you want to take this, happy to grab it otherwise.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Marking a VM dead should be a _very_ rare event; it's not something that I think we should encourage, i.e. we shouldn't make it easier to deal with. Ideally, use of
kvm_vm_dead()should be limited to things likesev_vm_move_enc_context_from(), where KVM needs to prever accessing the source VM to protect the host. IMO, the vGIC case and x86'senter_smm()are hacks. E.g. I don't see any reason why theenter_smm()case can't synthesize a triple fault.
Quoted 7 lines
Quoted 5 lines
The VGIC case is at least better than the alternative of slapping bandaids all over the shop to cope with a half-baked VM and ensure we tear it down correctly. Userspace is far up shit creek at the point the VM is marked as dead, so I don't see any value in hobbling along afterwards.
Again, I don't disagree, but I don't want to normalize shooting the VM on errors.
Definitely not. It is very much a break-glass situation where this is even somewhat OK.
Signature · 2 lines
Thanks, Oliver
Sean Christopherson
reviewerRe: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 21 lines · Oliver Upton
Quoted 13 lines · Sean Christopherson
---
From: Sean Christopherson <[email protected]>
Date: Thu, 7 Nov 2024 11:39:59 -0800
Subject: [PATCH] KVM: selftests: Don't bother deleting memslots in KVM when
freeing VMsWhen freeing a VM, don't call into KVM to manually remove each memslot, simply cleanup and free any userspace assets associated with the memory region. KVM is ultimately responsible for ensuring kernel resources are freed when the VM is destroyed, deleting memslots one-by-one is unnecessarily slow, and unless a test is already leaking the VM fd, the VM will be destroyed when
kvm_vm_release()is called.Quoted 2 lines
Not deleting KVM's memslot also allows cleaning up dead VMs without having to care whether or not the to-be-freed VM is dead or alive.
Can you add a comment to
kvm_vm_free()about why we want to avoid ioctls in that helper? It'd help discourage this situation from happening again in the future in the unlikely case someone wants to park an ioctl there.
Quoted 5 lines
Quoted 3 lines
Reported-by Eric Auger <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Sean Christopherson <[email protected]>I'm assuming you want to take this, happy to grab it otherwise.
You take it. Unless my git foo is off the rails, this is needs to go into 6.12, along with a fix for the vGIC test. That, and I already sent Paolo a pull request for rc7; I don't want to overwork myself ;-)
Re: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 26 lines · Sean Christopherson
Quoted 25 lines · Oliver Upton
Quoted 13 lines · Sean Christopherson
---
From: Sean Christopherson <[email protected]>
Date: Thu, 7 Nov 2024 11:39:59 -0800
Subject: [PATCH] KVM: selftests: Don't bother deleting memslots in KVM when
freeing VMsWhen freeing a VM, don't call into KVM to manually remove each memslot, simply cleanup and free any userspace assets associated with the memory region. KVM is ultimately responsible for ensuring kernel resources are freed when the VM is destroyed, deleting memslots one-by-one is unnecessarily slow, and unless a test is already leaking the VM fd, the VM will be destroyed when
kvm_vm_release()is called.Quoted 2 lines
Not deleting KVM's memslot also allows cleaning up dead VMs without having to care whether or not the to-be-freed VM is dead or alive.
Can you add a comment to
kvm_vm_free()about why we want to avoid ioctls in that helper? It'd help discourage this situation from happening again in the future in the unlikely case someone wants to park an ioctl there.Quoted 3 lines
Reported-by Eric Auger <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Sean Christopherson <[email protected]>
Quoted 5 lines
Quoted 1 line
I'm assuming you want to take this, happy to grab it otherwise.
You take it. Unless my git foo is off the rails, this is needs to go into 6.12, along with a fix for the vGIC test. That, and I already sent Paolo a pull request for rc7; I don't want to overwork myself ;-)
Fine -- I _guess_ I'll do it then :) It'll just go in the 6.13 pull request.
Signature · 2 lines
Thanks, Oliver
Re: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
In reply to Oliver UptonHi Sean, Oliver,
Quoted 52 lines · Oliver Upton
Quoted 13 lines · Sean Christopherson
Quoted 12 lines · Oliver Upton
Quoted 11 lines · Sean Christopherson
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.Please no. I don't want to bleed the
kvm->vm_deadbehavior all over selftests. The hack in__TEST_ASSERT_VM_VCPU_IOCTL()is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.Quoted 7 lines
Quoted 6 lines
Quoted 4 lines
IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return
-EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.Encouraging tests to explicitly leak resources to fudge around assertions in the selftests library seems off to me.
I don't disagree, but I really, really don't want to add
vm_dead().It'd still be valuable to test that the VM is properly dead and subsequent ioctls also return EIO, but I understand the hesitation.
Quoted 6 lines
Quoted 3 lines
IMO, the better approach would be to provide a helper that gives the impression of freeing the VM but implicitly leaks it, paired with some reasoning for it.
Actually, duh. There's no need to manually delete KVM memslots for *any* VM, dead or alive. Just skip that unconditionally when freeing the VM, and then the vGIC test just needs to assert on
-EIOinstead -ENXIO/-EBUSY.Yeah, that'd tighten up the assertions a bit more to the exact ioctl where we expect the VM to go sideways.
Quoted 13 lines
---
From: Sean Christopherson <[email protected]>
Date: Thu, 7 Nov 2024 11:39:59 -0800
Subject: [PATCH] KVM: selftests: Don't bother deleting memslots in KVM when
freeing VMsWhen freeing a VM, don't call into KVM to manually remove each memslot, simply cleanup and free any userspace assets associated with the memory region. KVM is ultimately responsible for ensuring kernel resources are freed when the VM is destroyed, deleting memslots one-by-one is unnecessarily slow, and unless a test is already leaking the VM fd, the VM will be destroyed when
kvm_vm_release()is called.Quoted 2 lines
Not deleting KVM's memslot also allows cleaning up dead VMs without having to care whether or not the to-be-freed VM is dead or alive.
Can you add a comment to
kvm_vm_free()about why we want to avoid ioctls in that helper? It'd help discourage this situation from happening again in the future in the unlikely case someone wants to park an ioctl there.
Quoted 6 lines
Quoted 3 lines
Reported-by Eric Auger <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Sean Christopherson <[email protected]>I'm assuming you want to take this, happy to grab it otherwise.
Reviewed-by Oliver Upton <[email protected]>
Thanks
Eric
Quoted 16 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Marking a VM dead should be a _very_ rare event; it's not something that I think we should encourage, i.e. we shouldn't make it easier to deal with. Ideally, use of
kvm_vm_dead()should be limited to things likesev_vm_move_enc_context_from(), where KVM needs to prever accessing the source VM to protect the host. IMO, the vGIC case and x86'senter_smm()are hacks. E.g. I don't see any reason why theenter_smm()case can't synthesize a triple fault.Quoted 6 lines
Quoted 5 lines
The VGIC case is at least better than the alternative of slapping bandaids all over the shop to cope with a half-baked VM and ensure we tear it down correctly. Userspace is far up shit creek at the point the VM is marked as dead, so I don't see any value in hobbling along afterwards.
Again, I don't disagree, but I don't want to normalize shooting the VM on errors.
Definitely not. It is very much a break-glass situation where this is even somewhat OK.
Re: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
In reply to Oliver UptonHi Oliver,
Quoted 18 lines · Oliver Upton
Quoted 17 lines · Sean Christopherson
Quoted 16 lines · Oliver Upton
Quoted 15 lines · Sean Christopherson
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.Please no. I don't want to bleed the
kvm->vm_deadbehavior all over selftests. The hack in__TEST_ASSERT_VM_VCPU_IOCTL()is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return
-EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines
Encouraging tests to explicitly leak resources to fudge around assertions in the selftests library seems off to me.
I don't disagree, but I really, really don't want to add
vm_dead().It'd still be valuable to test that the VM is properly dead and subsequent ioctls also return EIO, but I understand the hesitation.
Currently the vgic_test does not check that the VM is dead, it just checks the first expected errno according to the uapi documentation. Besides AFAIK this latter has not been updated according to the new VM dead implementation.
Eric
Quoted 51 lines
Quoted 7 lines
Quoted 4 lines
IMO, the better approach would be to provide a helper that gives the impression of freeing the VM but implicitly leaks it, paired with some reasoning for it.
Actually, duh. There's no need to manually delete KVM memslots for *any* VM, dead or alive. Just skip that unconditionally when freeing the VM, and then the vGIC test just needs to assert on
-EIOinstead -ENXIO/-EBUSY.Yeah, that'd tighten up the assertions a bit more to the exact ioctl where we expect the VM to go sideways.
Quoted 13 lines
---
From: Sean Christopherson <[email protected]>
Date: Thu, 7 Nov 2024 11:39:59 -0800
Subject: [PATCH] KVM: selftests: Don't bother deleting memslots in KVM when
freeing VMsWhen freeing a VM, don't call into KVM to manually remove each memslot, simply cleanup and free any userspace assets associated with the memory region. KVM is ultimately responsible for ensuring kernel resources are freed when the VM is destroyed, deleting memslots one-by-one is unnecessarily slow, and unless a test is already leaking the VM fd, the VM will be destroyed when
kvm_vm_release()is called.Quoted 2 lines
Not deleting KVM's memslot also allows cleaning up dead VMs without having to care whether or not the to-be-freed VM is dead or alive.
Can you add a comment to
kvm_vm_free()about why we want to avoid ioctls in that helper? It'd help discourage this situation from happening again in the future in the unlikely case someone wants to park an ioctl there.Quoted 3 lines
Reported-by Eric Auger <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Sean Christopherson <[email protected]>I'm assuming you want to take this, happy to grab it otherwise.
Reviewed-by Oliver Upton <[email protected]>Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Marking a VM dead should be a _very_ rare event; it's not something that I think we should encourage, i.e. we shouldn't make it easier to deal with. Ideally, use of
kvm_vm_dead()should be limited to things likesev_vm_move_enc_context_from(), where KVM needs to prever accessing the source VM to protect the host. IMO, the vGIC case and x86'senter_smm()are hacks. E.g. I don't see any reason why theenter_smm()case can't synthesize a triple fault.Quoted 6 lines
Quoted 5 lines
The VGIC case is at least better than the alternative of slapping bandaids all over the shop to cope with a half-baked VM and ensure we tear it down correctly. Userspace is far up shit creek at the point the VM is marked as dead, so I don't see any value in hobbling along afterwards.
Again, I don't disagree, but I don't want to normalize shooting the VM on errors.
Definitely not. It is very much a break-glass situation where this is even somewhat OK.
Re: [PATCH 2/3] KVM: selftests: Introduce kvm_vm_dead_free
Quoted 23 lines · Eric Auger
Hi Oliver,
Quoted 20 lines · Oliver Upton
Quoted 19 lines · Sean Christopherson
Quoted 18 lines · Oliver Upton
Quoted 11 lines · Sean Christopherson
Quoted 5 lines · Eric Auger
In case a
KVM_REQ_VM_DEADrequest was sent to a VM, subsequent KVM ioctls will fail and cause test failure. This now happens with an aarch64 vgic test where thekvm_vm_free()fails. Let's add a newkvm_vm_dead_free()helper that does all the deallocation besides theKVM_SET_USER_MEMORY_REGION2ioctl.Please no. I don't want to bleed the
kvm->vm_deadbehavior all over selftests. The hack in__TEST_ASSERT_VM_VCPU_IOCTL()is there purely to provide users with a more helpful error message, it is most definitely not intended to be an "official" way to detect and react to the VM being dead.Quoted 4 lines
IMO, tests that intentionally result in a dead VM should assert that subsequent VM/vCPU ioctls return
-EIO, and that's all. Attempting to gracefully free resources adds complexity and pollutes the core selftests APIs, with very little benefit.Encouraging tests to explicitly leak resources to fudge around assertions in the selftests library seems off to me.
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
I don't disagree, but I really, really don't want to add
vm_dead().It'd still be valuable to test that the VM is properly dead and subsequent ioctls also return EIO, but I understand the hesitation.
Currently the
vgic_testdoes not check that the VM is dead, it just checks the first expected errno according to the uapi documentation. Besides AFAIK this latter has not been updated according to the new VM dead implementation.
Ah yep, of course. We'll only return EIO for subsequent ioctls. It'd be nice to assert that is the case, but it isn't _that_ big of a deal.
Signature · 2 lines
Thanks, Oliver
PATCH 3/3 KVM: selftests: Handle dead VM in vgic_init test
In reply to Eric AugerCommit df5fd75ee305 ("KVM: arm64: Don't eagerly teardown the vgic on init error") changed the way the error is handled in case of incomplete vgic setup. Now a KVM_REQ_VM_DEAD request is sent causing subsequent KVM ioctl to fail. This now triggers a test assertion failure in kvm_vm_free() which calls KVM_SET_USER_MEMORY_REGION2 ioctl.
Update the test so that it checks that after the partial vgic setup the KVM_REQ_VM_DEAD has been sent and use the new kvm_vm_dead_free() helper to free the resources.
--- .../testing/selftests/kvm/aarch64/vgic_init.c | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-)
tools/testing/selftests/kvm/aarch64/vgic_init.c +23 -18
@@ -101,6 +101,12 @@ static void vm_gic_destroy(struct vm_gic *v)kvm_vm_free (v->vm); } +static void vm_dead_gic_destroy (struct vm_gic *v) +{ +close (v->gic_fd); +kvm_vm_dead_free (v->vm); +} +struct vgic_region_attr {uint64_t attr;uint64_t size;
@@ -335,7 +341,7 @@ static void test_vgic_then_vcpus(uint32_t gic_dev_type) {struct kvm_vcpu *vcpus[NR_VCPUS ];struct vm_gic v; -int ret, i; +int i; v =vm_gic_create_with_vcpus (gic_dev_type,1 , vcpus);
@@ -345,10 +351,10 @@ static void test_vgic_then_vcpus(uint32_t gic_dev_type)for (i =1 ; i <NR_VCPUS ; ++i) vcpus[i] =vm_vcpu_add (v.vm, i, guest_code); - ret =run_vcpu (vcpus[3 ]); -TEST_ASSERT (ret == -EINVAL ,"dist/rdist overlap detected on 1st vcpu run" ); +run_vcpu (vcpus[3 ]); +TEST_ASSERT (vm_dead (v.vm),"dist/rdist overlap detected on 1st vcpu run" ); -vm_gic_destroy (&v); +vm_dead_gic_destroy (&v); }/* All the VCPUs are created before the VGIC KVM device gets initialized */
@@ -356,16 +362,15 @@ static void test_vcpus_then_vgic(uint32_t gic_dev_type) {struct kvm_vcpu *vcpus[NR_VCPUS ];struct vm_gic v; -int ret; v =vm_gic_create_with_vcpus (gic_dev_type,NR_VCPUS , vcpus);subtest_dist_rdist (&v); - ret =run_vcpu (vcpus[3 ]); -TEST_ASSERT (ret == -EINVAL ,"dist/rdist overlap detected on 1st vcpu run" ); +run_vcpu (vcpus[3 ]); +TEST_ASSERT (vm_dead (v.vm),"dist/rdist overlap detected on 1st vcpu run" ); -vm_gic_destroy (&v); +vm_dead_gic_destroy (&v); }#define KVM_VGIC_V2_ATTR (offset, cpu) \
@@ -415,9 +420,9 @@ static void test_v3_new_redist_regions(void)kvm_device_attr_set (v.gic_fd,KVM_DEV_ARM_VGIC_GRP_CTRL ,KVM_DEV_ARM_VGIC_CTRL_INIT ,NULL ); - ret =run_vcpu (vcpus[3 ]); -TEST_ASSERT (ret == -ENXIO ,"running without sufficient number of rdists" ); -vm_gic_destroy (&v); +run_vcpu (vcpus[3 ]); +TEST_ASSERT (vm_dead (v.vm),"running without sufficient number of rdists" ); +vm_dead_gic_destroy (&v);/* step2 */
@@ -428,10 +433,10 @@ static void test_v3_new_redist_regions(void)kvm_device_attr_set (v.gic_fd,KVM_DEV_ARM_VGIC_GRP_ADDR ,KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION , &addr); - ret =run_vcpu (vcpus[3 ]); -TEST_ASSERT (ret == -EBUSY ,"running without vgic explicit init" ); +run_vcpu (vcpus[3 ]); +TEST_ASSERT (vm_dead (v.vm),"running without vgic explicit init" ); -vm_gic_destroy (&v); +vm_dead_gic_destroy (&v);/* step 3 */
@@ -604,8 +609,8 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void) {struct kvm_vcpu *vcpus[NR_VCPUS ];struct vm_gic v; -int ret, i;uint64_t addr; +int i; v =vm_gic_create_with_vcpus (KVM_DEV_TYPE_ARM_VGIC_V3 ,1 , vcpus);
@@ -626,11 +631,11 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void) KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);/* Attempt to run a vcpu without enough redist space. */ - ret =run_vcpu (vcpus[2 ]); -TEST_ASSERT (ret && errno ==EINVAL , +run_vcpu (vcpus[2 ]); +TEST_ASSERT (vm_dead (v.vm),"redist base+size above PA range detected on 1st vcpu run" ); -vm_gic_destroy (&v); +vm_dead_gic_destroy (&v); }static void test_v3_its_region (void )
Signature · 1 line
2.41.0