mm: add follow_pte_pmd()
PATCH 1/4 mm: add follow_pte_pmd()
From: Ross Zwisler <[email protected]>
commit 097963959594c5eccaba42510f7033f703211bda upstream.
Patch series "Write protect DAX PMDs in *sync path".
Currently dax_mapping_entry_mkclean() fails to clean and write protect the pmd_t of a DAX PMD entry during an *sync operation. This can result in data loss, as detailed in patch 2.
This series is based on Dan's "libnvdimm-pending" branch, which is the current home for Jan's "dax: Page invalidation fixes" series. You can find a working tree here:
https://git.kernel.org/cgit/linux/kernel/git/zwisler/linux.git/log/?h=dax_pmd_clean
This patch (of 2):
Similar to follow_pte(), follow_pte_pmd() allows either a PTE leaf or a huge page PMD leaf to be found and returned.
--- include/linux/mm.h | 2 ++ mm/memory.c | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 7 deletions(-)
include/linux/mm.h +2 -0
@@ -1269,6 +1269,8 @@ int copy_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *vma);void unmap_mapping_range (struct address_space *mapping,loff_t const holebegin,loff_t const holelen,int even_cows); +int follow_pte_pmd (struct mm_struct *mm,unsigned long address, +pte_t **ptepp,pmd_t **pmdpp,spinlock_t **ptlp);int follow_pfn (struct vm_area_struct *vma,unsigned long address,unsigned long *pfn); int follow_phys(struct vm_area_struct *vma, unsigned long address,
mm/memory.c +30 -7
@@ -3780,8 +3780,8 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) }#endif /* __PAGETABLE_PMD_FOLDED */ -static int __follow_pte (struct mm_struct *mm,unsigned long address, -pte_t **ptepp,spinlock_t **ptlp) +static int __follow_pte_pmd (struct mm_struct *mm,unsigned long address, +pte_t **ptepp,pmd_t **pmdpp,spinlock_t **ptlp) {pgd_t *pgd;pud_t *pud;
@@ -3798,11 +3798,20 @@ static int __follow_pte(struct mm_struct *mm, unsigned long address, pmd =pmd_offset (pud, address);VM_BUG_ON (pmd_trans_huge (*pmd)); -if (pmd_none (*pmd) ||unlikely (pmd_bad (*pmd))) - goto out; -/* We cannot handle huge page PFN maps. Luckily they don't exist. */ -if (pmd_huge (*pmd)) +if (pmd_huge (*pmd)) { +if (!pmdpp) + goto out; + + *ptlp =pmd_lock (mm, pmd); +if (pmd_huge (*pmd)) { + *pmdpp = pmd; +return 0 ; + } +spin_unlock (*ptlp); + } + +if (pmd_none (*pmd) ||unlikely (pmd_bad (*pmd))) goto out; ptep =pte_offset_map_lock (mm, pmd, address, ptlp);
@@ -3825,9 +3834,23 @@ static inline int follow_pte(struct mm_struct *mm, unsigned long address,/* (void) is needed to make gcc happy */ (void )__cond_lock (*ptlp, - !(res =__follow_pte (mm, address, ptepp, ptlp))); + !(res =__follow_pte_pmd (mm, address, ptepp,NULL , + ptlp))); +return res; +} + +int follow_pte_pmd (struct mm_struct *mm,unsigned long address, +pte_t **ptepp,pmd_t **pmdpp,spinlock_t **ptlp) +{ +int res; + +/* (void) is needed to make gcc happy */ + (void )__cond_lock (*ptlp, + !(res =__follow_pte_pmd (mm, address, ptepp, pmdpp, + ptlp)));return res; } +EXPORT_SYMBOL (follow_pte_pmd);/** * follow_pfn - look up PFN at a user virtual address
PATCH 2/4 KVM: do not assume PTE is writable after follow_pfn
From: Paolo Bonzini <[email protected]>
commit bd2fae8da794b55bf2ac02632da3a151b10e664c upstream.
In order to convert an HVA to a PFN, KVM usually tries to use the get_user_pages family of functinso. This however is not possible for VM_IO vmas; in that case, KVM instead uses follow_pfn.
In doing this however KVM loses the information on whether the PFN is writable. That is usually not a problem because the main use of VM_IO vmas with KVM is for BARs in PCI device assignment, however it is a bug. To fix it, use follow_pte and check pte_write while under the protection of the PTE lock. The information can be used to fail hva_to_pfn_remapped or passed back to the caller via *writable.
Usage of follow_pfn was introduced in commit add6a0cd1c5b ("KVM: MMU: try to fix up page faults before giving up", 2016-07-05); however, even older version have the same issue, all the way back to commit 2e2e3738af33 ("KVM:
Handle vma regions with no backing page", 2008-07-20), as they also did not check whether the PFN was writable.
[bwh: Backport to 4.9: follow_pte_pmd() does not take start or end
parameters]
--- virt/kvm/kvm_main.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
virt/kvm/kvm_main.c +12 -3
@@ -1519,9 +1519,11 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, kvm_pfn_t *p_pfn) {unsigned long pfn; +pte_t *ptep; +spinlock_t *ptl;int r; - r =follow_pfn (vma, addr, &pfn); + r =follow_pte_pmd (vma->vm_mm, addr, &ptep,NULL , &ptl);if (r) {/* * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
@@ -1536,14 +1538,19 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,if (r)return r; - r =follow_pfn (vma, addr, &pfn); + r =follow_pte_pmd (vma->vm_mm, addr, &ptep,NULL , &ptl);if (r)return r; + } +if (write_fault && !pte_write (*ptep)) { + pfn =KVM_PFN_ERR_RO_FAULT ; + goto out; }if (writable) - *writable = true; + *writable =pte_write (*ptep); + pfn =pte_pfn (*ptep);/* * Get a reference here because callers of *hva_to_pfn* and
@@ -1558,6 +1565,8 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,*/ kvm_get_pfn (pfn); +out: +pte_unmap_unlock (ptep, ptl); *p_pfn = pfn;return 0 ; }
PATCH 3/4 KVM: Use kvm_pfn_t for local PFN variable in hva_to_pfn_remapped()
In reply to Ben HutchingsFrom: Sean Christopherson <[email protected]>
commit a9545779ee9e9e103648f6f2552e73cfe808d0f4 upstream.
Use kvm_pfn_t, a.k.a. u64, for the local 'pfn' variable when retrieving a so called "remapped" hva/pfn pair. In theory, the hva could resolve to a pfn in high memory on a 32-bit kernel.
This bug was inadvertantly exposed by commit bd2fae8da794 ("KVM: do not assume PTE is writable after follow_pfn"), which added an error PFN value to the mix, causing gcc to comlain about overflowing the unsigned long.
arch/x86/kvm/../../../virt/kvm/kvm_main.c: In function ‘hva_to_pfn_remapped’:
include/linux/kvm_host.h:89:30: error: conversion from ‘long long unsigned int’
to ‘long unsigned int’ changes value from ‘9218868437227405314’ to ‘2’ [-Werror=overflow]
89 | #define KVM_PFN_ERR_RO_FAULT (KVM_PFN_ERR_MASK + 2)
| ^virt/kvm/kvm_main.c:1935:9: note: in expansion of macro ‘KVM_PFN_ERR_RO_FAULT’
Message-Id: <[email protected]>
--- virt/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
virt/kvm/kvm_main.c +1 -1
@@ -1518,7 +1518,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, bool write_fault, bool *writable, kvm_pfn_t *p_pfn) { - unsigned long pfn; + kvm_pfn_t pfn;pte_t *ptep;spinlock_t *ptl;int r;
PATCH 4/4 KVM: do not allow mapping valid but non-reference-counted pages
In reply to Ben HutchingsFrom: Nicholas Piggin <[email protected]>
commit f8be156be163a052a067306417cd0ff679068c97 upstream.
It's possible to create a region which maps valid but non-refcounted pages (e.g., tail pages of non-compound higher order allocations). These host pages can then be returned by gfn_to_page, gfn_to_pfn, etc., family of APIs, which take a reference to the page, which takes it from 0 to 1. When the reference is dropped, this will free the page incorrectly.
Fix this by only taking a reference on valid pages if it was non-zero, which indicates it is participating in normal refcounting (and can be released with put_page).
This addresses CVE-2021-22543.
--- virt/kvm/kvm_main.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-)
virt/kvm/kvm_main.c +17 -2
@@ -1513,6 +1513,13 @@ static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)return true; } +static int kvm_try_get_pfn (kvm_pfn_t pfn) +{ +if (kvm_is_reserved_pfn (pfn)) +return 1 ; +return get_page_unless_zero (pfn_to_page (pfn)); +} + static int hva_to_pfn_remapped(struct vm_area_struct *vma, unsigned long addr, bool *async, bool write_fault, bool *writable,
@@ -1562,13 +1569,21 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,* Whoever called remap_pfn_range is also going to call e.g. * unmap_mapping_range before the underlying pages are freed, * causing a call to our MMU notifier. +* +* Certain IO or PFNMAP mappings can be backed with valid +* struct pages, but be allocated without refcounting e.g., +* tail pages of non-compound higher order allocations, which +* would then underflow the refcount when the caller does the +* required put_page. Don't allow those pages here. */ -kvm_get_pfn (pfn); +if (!kvm_try_get_pfn (pfn)) + r = -EFAULT ; out:pte_unmap_unlock (ptep, ptl); *p_pfn = pfn; -return 0 ; + +return r; }/*