KVM: Pass MMU notifier range flags to kvm_unmap_hva_range()
PATCH 1/2 KVM: Pass MMU notifier range flags to kvm_unmap_hva_range()
commit fdfe7cbd58806522e799e2a50a15aee7f2cbb7b6 upstream.
The 'flags' field of 'struct mmu_notifier_range' is used to indicate whether invalidate_range_{start,end}() are permitted to block. In the case of kvm_mmu_notifier_invalidate_range_start(), this field is not forwarded on to the architecture-specific implementation of kvm_unmap_hva_range() and therefore the backend cannot sensibly decide whether or not to block.
Add an extra 'flags' parameter to kvm_unmap_hva_range() so that architectures are aware as to whether or not they are permitted to block.
Cc <[email protected]> # v5.7 only
Cc Marc Zyngier <[email protected]>
Cc Suzuki K Poulose <[email protected]>
Cc James Morse <[email protected]>
Signed-off-by Will Deacon <[email protected]>
Message-Id: <[email protected]>
Signed-off-by Paolo Bonzini <[email protected]>
Signed-off-by Will Deacon <[email protected]>
--- arch/arm64/include/asm/kvm_host.h | 2 +- arch/mips/include/asm/kvm_host.h | 2 +- arch/mips/kvm/mmu.c | 3 ++- arch/powerpc/include/asm/kvm_host.h | 3 ++- arch/powerpc/kvm/book3s.c | 3 ++- arch/powerpc/kvm/e500_mmu_host.c | 3 ++- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/mmu/mmu.c | 3 ++- virt/kvm/arm/mmu.c | 2 +- virt/kvm/kvm_main.c | 3 ++- 10 files changed, 17 insertions(+), 10 deletions(-)
arch/arm64/include/asm/kvm_host.h +1 -1
@@ -440,7 +440,7 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,#define KVM_ARCH_WANT_MMU_NOTIFIER int kvm_unmap_hva_range (struct kvm *kvm, -unsigned long start,unsigned long end); +unsigned long start,unsigned long end,unsigned flags);int kvm_set_spte_hva (struct kvm *kvm,unsigned long hva,pte_t pte);int kvm_age_hva (struct kvm *kvm,unsigned long start,unsigned long end);int kvm_test_age_hva (struct kvm *kvm,unsigned long hva);
arch/mips/include/asm/kvm_host.h +1 -1
@@ -939,7 +939,7 @@ enum kvm_mips_fault_result kvm_trap_emul_gva_fault(struct kvm_vcpu *vcpu,#define KVM_ARCH_WANT_MMU_NOTIFIER int kvm_unmap_hva_range (struct kvm *kvm, -unsigned long start,unsigned long end); +unsigned long start,unsigned long end,unsigned flags);int kvm_set_spte_hva (struct kvm *kvm,unsigned long hva,pte_t pte);int kvm_age_hva (struct kvm *kvm,unsigned long start,unsigned long end);int kvm_test_age_hva (struct kvm *kvm,unsigned long hva);
arch/mips/kvm/mmu.c +2 -1
@@ -518,7 +518,8 @@ static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,return 1 ; } -int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end) +int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end, +unsigned flags) {handle_hva_to_gpa (kvm, start, end, &kvm_unmap_hva_handler,NULL );
arch/powerpc/include/asm/kvm_host.h +2 -1
@@ -58,7 +58,8 @@#define KVM_ARCH_WANT_MMU_NOTIFIER extern int kvm_unmap_hva_range (struct kvm *kvm, -unsigned long start,unsigned long end); +unsigned long start,unsigned long end, +unsigned flags);extern int kvm_age_hva (struct kvm *kvm,unsigned long start,unsigned long end);extern int kvm_test_age_hva (struct kvm *kvm,unsigned long hva);extern int kvm_set_spte_hva (struct kvm *kvm,unsigned long hva,pte_t pte);
arch/powerpc/kvm/book3s.c +2 -1
@@ -837,7 +837,8 @@ void kvmppc_core_commit_memory_region(struct kvm *kvm, kvm->arch.kvm_ops->commit_memory_region (kvm, mem, old, new, change); } -int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end) +int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end, +unsigned flags) {return kvm->arch.kvm_ops->unmap_hva_range (kvm, start, end); }
arch/powerpc/kvm/e500_mmu_host.c +2 -1
@@ -734,7 +734,8 @@ static int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)return 0 ; } -int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end) +int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end, +unsigned flags) {/* kvm_unmap_hva flushes everything anyways */ kvm_unmap_hva (kvm, start);
arch/x86/include/asm/kvm_host.h +2 -1
@@ -1606,7 +1606,8 @@ asmlinkage void kvm_spurious_fault(void); _ASM_EXTABLE(666b, 667b)#define KVM_ARCH_WANT_MMU_NOTIFIER -int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end); +int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end, +unsigned flags);int kvm_age_hva (struct kvm *kvm,unsigned long start,unsigned long end);int kvm_test_age_hva (struct kvm *kvm,unsigned long hva);int kvm_set_spte_hva (struct kvm *kvm,unsigned long hva,pte_t pte);
arch/x86/kvm/mmu/mmu.c +2 -1
@@ -1972,7 +1972,8 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,return kvm_handle_hva_range (kvm, hva, hva +1 , data, handler); } -int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end) +int kvm_unmap_hva_range (struct kvm *kvm,unsigned long start,unsigned long end, +unsigned flags) {return kvm_handle_hva_range (kvm, start, end,0 , kvm_unmap_rmapp); }
virt/kvm/arm/mmu.c +1 -1
@@ -2046,7 +2046,7 @@ static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *dat }int kvm_unmap_hva_range (struct kvm *kvm, -unsigned long start,unsigned long end) +unsigned long start,unsigned long end,unsigned flags) {if (!kvm->arch.pgd)return 0 ;
virt/kvm/kvm_main.c +2 -1
@@ -428,7 +428,8 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,* count is also read inside the mmu_lock critical section. */ kvm->mmu_notifier_count++; - need_tlb_flush =kvm_unmap_hva_range (kvm, range->start, range->end); + need_tlb_flush =kvm_unmap_hva_range (kvm, range->start, range->end, + range->flags); need_tlb_flush |= kvm->tlbs_dirty;/* we've to flush the tlb before the pages can be freed */ if (need_tlb_flush)
Signature · 1 line
2.28.0.297.g1956fa8f8d-goog
PATCH 2/2 KVM: arm64: Only reschedule if MMU_NOTIFIER_RANGE_BLOCKABLE is not set
commit b5331379bc62611d1026173a09c73573384201d9 upstream.
When an MMU notifier call results in unmapping a range that spans multiple PGDs, we end up calling into cond_resched_lock() when crossing a PGD boundary, since this avoids running into RCU stalls during VM teardown. Unfortunately, if the VM is destroyed as a result of OOM, then blocking is not permitted and the call to the scheduler triggers the following BUG():
| BUG: sleeping function called from invalid context at arch/arm64/kvm/mmu.c:394 | in_atomic(): 1, irqs_disabled(): 0, non_block: 1, pid: 36, name: oom_reaper | INFO: lockdep is turned off. | CPU: 3 PID: 36 Comm: oom_reaper Not tainted 5.8.0 #1 | Hardware name: QEMU QEMU Virtual Machine, BIOS 0.0.0 02/06/2015 | Call trace: | dump_backtrace+0x0/0x284 | show_stack+0x1c/0x28 | dump_stack+0xf0/0x1a4 | ___might_sleep+0x2bc/0x2cc | unmap_stage2_range+0x160/0x1ac | kvm_unmap_hva_range+0x1a0/0x1c8 | kvm_mmu_notifier_invalidate_range_start+0x8c/0xf8 | __mmu_notifier_invalidate_range_start+0x218/0x31c | mmu_notifier_invalidate_range_start_nonblock+0x78/0xb0 | __oom_reap_task_mm+0x128/0x268 | oom_reap_task+0xac/0x298 | oom_reaper+0x178/0x17c | kthread+0x1e4/0x1fc | ret_from_fork+0x10/0x30
Use the new 'flags' argument to kvm_unmap_hva_range() to ensure that we only reschedule if MMU_NOTIFIER_RANGE_BLOCKABLE is set in the notifier flags.
Cc <[email protected]> # v5.7 only
Fixes 8b3405e345b5 ("kvm: arm/arm64: Fix locking for kvm_free_stage2_pgd")
Cc Marc Zyngier <[email protected]>
Cc Suzuki K Poulose <[email protected]>
Cc James Morse <[email protected]>
Signed-off-by Will Deacon <[email protected]>
Message-Id: <[email protected]>
Signed-off-by Paolo Bonzini <[email protected]>
Signed-off-by Will Deacon <[email protected]>
--- virt/kvm/arm/mmu.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)
virt/kvm/arm/mmu.c +13 -4
@@ -331,7 +331,8 @@ static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,* destroying the VM), otherwise another faulting VCPU may come in and mess * with things behind our backs. */ -static void unmap_stage2_range (struct kvm *kvm,phys_addr_t start,u64 size) +static void __unmap_stage2_range (struct kvm *kvm,phys_addr_t start,u64 size, +bool may_block) {pgd_t *pgd;phys_addr_t addr = start, end = start + size;
@@ -356,11 +357,16 @@ static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)* If the range is too large, release the kvm->mmu_lock * to prevent starvation and lockup detector warnings. */ -if (next != end) +if (may_block && next != end)cond_resched_lock (&kvm->mmu_lock); }while (pgd++, addr = next, addr != end); } +static void unmap_stage2_range (struct kvm *kvm,phys_addr_t start,u64 size) +{ +__unmap_stage2_range (kvm, start, size, true); +} +static void stage2_flush_ptes (struct kvm *kvm,pmd_t *pmd,phys_addr_t addr,phys_addr_t end) {
@@ -2041,7 +2047,10 @@ static int handle_hva_to_gpa(struct kvm *kvm,static int kvm_unmap_hva_handler (struct kvm *kvm,gpa_t gpa,u64 size,void *data) { -unmap_stage2_range (kvm, gpa, size); +unsigned flags = *(unsigned *)data; +bool may_block = flags &MMU_NOTIFIER_RANGE_BLOCKABLE ; + +__unmap_stage2_range (kvm, gpa, size, may_block);return 0 ; }
@@ -2052,7 +2061,7 @@ int kvm_unmap_hva_range(struct kvm *kvm,return 0 ;trace_kvm_unmap_hva_range (start, end); -handle_hva_to_gpa (kvm, start, end, &kvm_unmap_hva_handler,NULL ); +handle_hva_to_gpa (kvm, start, end, &kvm_unmap_hva_handler, &flags);return 0 ; }
Signature · 1 line
2.28.0.297.g1956fa8f8d-goog