KVM: MMU: Do not treat ZONE_DEVICE pages as being reserved
PATCH 1/1 KVM: MMU: Do not treat ZONE_DEVICE pages as being reserved
[ Upstream commit a78986aae9b2988f8493f9f65a587ee433e83bc3 ]
Explicitly exempt ZONE_DEVICE pages from kvm_is_reserved_pfn() and instead manually handle ZONE_DEVICE on a case-by-case basis. For things like page refcounts, KVM needs to treat ZONE_DEVICE pages like normal pages, e.g. put pages grabbed via gup(). But for flows such as setting A/D bits or shifting refcounts for transparent huge pages, KVM needs to to avoid processing ZONE_DEVICE pages as the flows in question lack the underlying machinery for proper handling of ZONE_DEVICE pages.
This fixes a hang reported by Adam Borowski[*] in dev_pagemap_cleanup() when running a KVM guest backed with /dev/dax memory, as KVM straight up doesn't put any references to ZONE_DEVICE pages acquired by gup().
Note, Dan Williams proposed an alternative solution of doing put_page() on ZONE_DEVICE pages immediately after gup() in order to simplify the auditing needed to ensure is_zone_device_page() is called if and only if the backing device is pinned (via gup()). But that approach would break kvm_vcpu_{un}map() as KVM requires the page to be pinned from map() 'til unmap() when accessing guest memory, unlike KVM's secondary MMU, which coordinates with mmu_notifier invalidations to avoid creating stale page references, i.e. doesn't rely on pages being pinned.
[*] http://lkml.kernel.org/r/[email protected]
--- arch/x86/kvm/mmu.c | 8 ++++---- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 26 +++++++++++++++++++++++--- 3 files changed, 28 insertions(+), 7 deletions(-)
arch/x86/kvm/mmu.c +4 -4
@@ -3177,7 +3177,7 @@ static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,* here. */ if (!is_error_noslot_pfn (pfn) && !kvm_is_reserved_pfn (pfn) && - level ==PT_PAGE_TABLE_LEVEL && + !kvm_is_zone_device_pfn (pfn) && level ==PT_PAGE_TABLE_LEVEL &&PageTransCompoundMap (pfn_to_page (pfn)) && !mmu_gfn_lpage_is_disallowed (vcpu, gfn,PT_DIRECTORY_LEVEL )) {unsigned long mask;
@@ -5344,9 +5344,9 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,* the guest, and the guest page table is using 4K page size * mapping if the indirect sp has level = 1. */ -if (sp->role.direct && - !kvm_is_reserved_pfn (pfn) && -PageTransCompoundMap (pfn_to_page (pfn))) { +if (sp->role.direct && !kvm_is_reserved_pfn (pfn) && + !kvm_is_zone_device_pfn (pfn) && +PageTransCompoundMap (pfn_to_page (pfn))) {drop_spte (kvm, sptep); need_tlb_flush =1 ; goto restart;
include/linux/kvm_host.h +1 -0
@@ -890,6 +890,7 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);void kvm_vcpu_kick (struct kvm_vcpu *vcpu);bool kvm_is_reserved_pfn (kvm_pfn_t pfn); +bool kvm_is_zone_device_pfn (kvm_pfn_t pfn);struct kvm_irq_ack_notifier {struct hlist_node link;
virt/kvm/kvm_main.c +23 -3
@@ -142,10 +142,30 @@ __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, { } +bool kvm_is_zone_device_pfn (kvm_pfn_t pfn) +{ +/* +* The metadata used by is_zone_device_page() to determine whether or +* not a page is ZONE_DEVICE is guaranteed to be valid if and only if +* the device has been pinned, e.g. by get_user_pages(). WARN if the +* page_count() is zero to help detect bad usage of this helper. +*/ +if (!pfn_valid (pfn) ||WARN_ON_ONCE (!page_count (pfn_to_page (pfn)))) +return false; + +return is_zone_device_page (pfn_to_page (pfn)); +} +bool kvm_is_reserved_pfn (kvm_pfn_t pfn) { +/* +* ZONE_DEVICE pages currently set PG_reserved, but from a refcounting +* perspective they are "normal" pages, albeit with slightly different +* usage rules. +*/ if (pfn_valid (pfn)) -return PageReserved (pfn_to_page (pfn)); +return PageReserved (pfn_to_page (pfn)) && + !kvm_is_zone_device_pfn (pfn);return true; }
@@ -1730,7 +1750,7 @@ static void kvm_release_pfn_dirty(kvm_pfn_t pfn)void kvm_set_pfn_dirty (kvm_pfn_t pfn) { -if (!kvm_is_reserved_pfn (pfn)) { +if (!kvm_is_reserved_pfn (pfn) && !kvm_is_zone_device_pfn (pfn)) {struct page *page =pfn_to_page (pfn);if (!PageReserved (page))
@@ -1741,7 +1761,7 @@ EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);void kvm_set_pfn_accessed (kvm_pfn_t pfn) { -if (!kvm_is_reserved_pfn (pfn)) +if (!kvm_is_reserved_pfn (pfn) && !kvm_is_zone_device_pfn (pfn))mark_page_accessed (pfn_to_page (pfn)); }EXPORT_SYMBOL_GPL (kvm_set_pfn_accessed);
Signature · 1 line
2.24.0
Quoted 125 lines · Sean Christopherson
[ Upstream commit
a78986aae9b2988f8493f9f65a587ee433e83bc3]Explicitly exempt
ZONE_DEVICEpages fromkvm_is_reserved_pfn()and instead manually handleZONE_DEVICEon a case-by-case basis. For things like page refcounts, KVM needs to treatZONE_DEVICEpages like normal pages, e.g. put pages grabbed viagup(). But for flows such as setting A/D bits or shifting refcounts for transparent huge pages, KVM needs to to avoid processingZONE_DEVICEpages as the flows in question lack the underlying machinery for proper handling ofZONE_DEVICEpages.This fixes a hang reported by Adam Borowski[*] in
dev_pagemap_cleanup()when running a KVM guest backed with/dev/daxmemory, as KVM straight up doesn't put any references toZONE_DEVICEpages acquired bygup().Note, Dan Williams proposed an alternative solution of doing
put_page()onZONE_DEVICEpages immediately aftergup()in order to simplify the auditing needed to ensureis_zone_device_page()is called if and only if the backing device is pinned (viagup()). But that approach would break kvm_vcpu_{un}map() as KVM requires the page to be pinned frommap()'tilunmap()when accessing guest memory, unlike KVM's secondary MMU, which coordinates withmmu_notifierinvalidations to avoid creating stale page references, i.e. doesn't rely on pages being pinned.[*] http://lkml.kernel.org/r/[email protected]
Reported-by Adam Borowski <[email protected]>Analyzed-by David Hildenbrand <[email protected]>Acked-by Dan Williams <[email protected]>Fixes 3565fce3a659 ("mm, x86: get_user_pages() for dax mappings")Signed-off-by Sean Christopherson <[email protected]>Signed-off-by Paolo Bonzini <[email protected]>[sean: backport to 4.x; resolve conflict in mmu.c]Signed-off-by Sean Christopherson <[email protected]>--- arch/x86/kvm/mmu.c | 8 ++++---- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 26 +++++++++++++++++++++++--- 3 files changed, 28 insertions(+), 7 deletions(-)arch/x86/kvm/mmu.c +4
-4@@ -3177,7 +3177,7 @@ static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,* here. */ if (!is_error_noslot_pfn (pfn) && !kvm_is_reserved_pfn (pfn) && - level ==PT_PAGE_TABLE_LEVEL && + !kvm_is_zone_device_pfn (pfn) && level ==PT_PAGE_TABLE_LEVEL &&PageTransCompoundMap (pfn_to_page (pfn)) && !mmu_gfn_lpage_is_disallowed (vcpu, gfn,PT_DIRECTORY_LEVEL )) {unsigned long mask;@@ -5344,9 +5344,9 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,* the guest, and the guest page table is using 4K page size * mapping if the indirect sp has level = 1. */ -if (sp->role.direct && - !kvm_is_reserved_pfn (pfn) && -PageTransCompoundMap (pfn_to_page (pfn))) { +if (sp->role.direct && !kvm_is_reserved_pfn (pfn) && + !kvm_is_zone_device_pfn (pfn) && +PageTransCompoundMap (pfn_to_page (pfn))) {drop_spte (kvm, sptep); need_tlb_flush =1 ; goto restart;include/linux/kvm_host.h +1
-0@@ -890,6 +890,7 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);void kvm_vcpu_kick (struct kvm_vcpu *vcpu);bool kvm_is_reserved_pfn (kvm_pfn_t pfn); +bool kvm_is_zone_device_pfn (kvm_pfn_t pfn);struct kvm_irq_ack_notifier {struct hlist_node link;virt/kvm/kvm_main.c +22
-2@@ -142,10 +142,30 @@ __weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, { } +bool kvm_is_zone_device_pfn (kvm_pfn_t pfn) +{ +/* +* The metadata used by is_zone_device_page() to determine whether or +* not a page is ZONE_DEVICE is guaranteed to be valid if and only if +* the device has been pinned, e.g. by get_user_pages(). WARN if the +* page_count() is zero to help detect bad usage of this helper. +*/ +if (!pfn_valid (pfn) ||WARN_ON_ONCE (!page_count (pfn_to_page (pfn)))) +return false; + +return is_zone_device_page (pfn_to_page (pfn)); +} +bool kvm_is_reserved_pfn (kvm_pfn_t pfn) { +/* +* ZONE_DEVICE pages currently set PG_reserved, but from a refcounting +* perspective they are "normal" pages, albeit with slightly different +* usage rules. +*/ if (pfn_valid (pfn)) -return PageReserved (pfn_to_page (pfn)); +return PageReserved (pfn_to_page (pfn)) && + !kvm_is_zone_device_pfn (pfn);return true; }@@ -1730,7 +1750,7 @@ static void kvm_release_pfn_dirty(kvm_pfn_t pfn)void kvm_set_pfn_dirty (kvm_pfn_t pfn) { -if (!kvm_is_reserved_pfn (pfn)) { +if (!kvm_is_reserved_pfn (pfn) && !kvm_is_zone_device_pfn (pfn)) {struct page *page =pfn_to_page (pfn);if (!PageReserved (page))
virt/kvm/kvm_main.c EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty); · line 1767
Quoted 8 lines
virt/kvm/kvm_main.c +1
-1@@ -1741,7 +1761,7 @@ EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);void kvm_set_pfn_accessed (kvm_pfn_t pfn) { -if (!kvm_is_reserved_pfn (pfn)) +if (!kvm_is_reserved_pfn (pfn) && !kvm_is_zone_device_pfn (pfn))mark_page_accessed (pfn_to_page (pfn)); }EXPORT_SYMBOL_GPL (kvm_set_pfn_accessed);