hugetlb: Disable huge pmd unshare for uffd-wp
Subsystems (12)
Peter Xu
reviewercover hugetlb: Disable huge pmd unshare for uffd-wp
This series tries to disable huge pmd unshare of hugetlbfs backed memory for uffd-wp. Although uffd-wp of hugetlbfs is still during rfc stage, the idea of this series may be needed for multiple tasks (Axel's uffd minor fault series, and Mike's soft dirty series), so I picked it out from the larger series.
References works:
Uffd shmem+hugetlbfs rfc:
https://lore.kernel.org/lkml/[email protected]/
Uffd minor mode for hugetlbfs:
https://lore.kernel.org/lkml/[email protected]/
Soft dirty for hugetlbfs:
https://lore.kernel.org/lkml/[email protected]/
Please review, thanks.
Peter Xu (4):
hugetlb: Pass vma into huge_pte_alloc() and huge_pmd_share() hugetlb/userfaultfd: Forbid huge pmd sharing when uffd enabled mm/hugetlb: Move flush_hugetlb_tlb_range() into hugetlb.h hugetlb/userfaultfd: Unshare all pmds for hugetlbfs when register wp
arch/arm64/mm/hugetlbpage.c | 7 ++--- arch/ia64/mm/hugetlbpage.c | 3 +- arch/mips/mm/hugetlbpage.c | 4 +-- arch/parisc/mm/hugetlbpage.c | 2 +- arch/powerpc/mm/hugetlbpage.c | 3 +- arch/s390/mm/hugetlbpage.c | 2 +- arch/sh/mm/hugetlbpage.c | 2 +- arch/sparc/mm/hugetlbpage.c | 1 + fs/userfaultfd.c | 57 +++++++++++++++++++++++++++++++++++ include/linux/hugetlb.h | 15 +++++++-- include/linux/userfaultfd_k.h | 9 ++++++ mm/hugetlb.c | 43 +++++++++++++------------- mm/userfaultfd.c | 2 +- 13 files changed, 115 insertions(+), 35 deletions(-)
Signature · 1 line
2.26.2
Peter Xu
reviewerPATCH 1/4 hugetlb: Pass vma into huge_pte_alloc() and huge_pmd_share()
It is a preparation work to be able to behave differently in the per architecture huge_pte_alloc() according to different VMA attributes.
Pass it deeper into huge_pmd_share() so that we can avoid the find_vma() call.
Suggested-by Mike Kravetz <[email protected]>
Reviewed-by Mike Kravetz <[email protected]>
Signed-off-by Peter Xu <[email protected]>
--- arch/arm64/mm/hugetlbpage.c | 4 ++-- arch/ia64/mm/hugetlbpage.c | 3 ++- arch/mips/mm/hugetlbpage.c | 4 ++-- arch/parisc/mm/hugetlbpage.c | 2 +- arch/powerpc/mm/hugetlbpage.c | 3 ++- arch/s390/mm/hugetlbpage.c | 2 +- arch/sh/mm/hugetlbpage.c | 2 +- arch/sparc/mm/hugetlbpage.c | 1 + include/linux/hugetlb.h | 5 +++-- mm/hugetlb.c | 15 ++++++++------- mm/userfaultfd.c | 2 +- 11 files changed, 24 insertions(+), 19 deletions(-)
arch/arm64/mm/hugetlbpage.c +2 -2
@@ -252,7 +252,7 @@ void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,set_pte (ptep, pte); } -pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long sz) {pgd_t *pgdp;
@@ -286,7 +286,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, }else if (sz ==PMD_SIZE ) {if (IS_ENABLED (CONFIG_ARCH_WANT_HUGE_PMD_SHARE ) &&pud_none (READ_ONCE (*pudp))) - ptep =huge_pmd_share (mm, addr, pudp); + ptep =huge_pmd_share (mm, vma, addr, pudp);else ptep = (pte_t *)pmd_alloc (mm, pudp, addr); }else if (sz == (CONT_PMD_SIZE )) {
arch/ia64/mm/hugetlbpage.c +2 -1
@@ -25,7 +25,8 @@ unsigned int hpage_shift = HPAGE_SHIFT_DEFAULT;EXPORT_SYMBOL (hpage_shift);pte_t * -huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) +huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma, +unsigned long addr,unsigned long sz) {unsigned long taddr =htlbpage_to_page (addr);pgd_t *pgd;
arch/mips/mm/hugetlbpage.c +2 -2
@@ -21,8 +21,8 @@#include <asm/tlb.h> #include <asm/tlbflush.h> -pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr, -unsigned long sz) +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma, +unsigned long addr,unsigned long sz) {pgd_t *pgd;p4d_t *p4d;
arch/parisc/mm/hugetlbpage.c +1 -1
@@ -44,7 +44,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr, } -pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long sz) {pgd_t *pgd;
arch/powerpc/mm/hugetlbpage.c +2 -1
@@ -106,7 +106,8 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,* At this point we do the placement change only for BOOK3S 64. This would * possibly work on other subarchs. */ -pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr,unsigned long sz) +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma, +unsigned long addr,unsigned long sz) {pgd_t *pg;p4d_t *p4;
arch/s390/mm/hugetlbpage.c +1 -1
@@ -189,7 +189,7 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm,return pte; } -pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long sz) {pgd_t *pgdp;
arch/sh/mm/hugetlbpage.c +1 -1
@@ -21,7 +21,7 @@#include <asm/tlbflush.h> #include <asm/cacheflush.h> -pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long sz) {pgd_t *pgd;
arch/sparc/mm/hugetlbpage.c +1 -0
@@ -280,6 +280,7 @@ unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&punsigned long pte_leaf_size (pte_t pte) {return 1UL <<tte_to_shift (pte); }pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma, unsigned long addr, unsigned long sz) {pgd_t *pgd;
include/linux/hugetlb.h +3 -2
@@ -152,7 +152,8 @@ void hugetlb_fix_reserve_counts(struct inode *inode);extern struct mutex *hugetlb_fault_mutex_table;u32 hugetlb_fault_mutex_hash (struct address_space *mapping,pgoff_t idx); -pte_t *huge_pmd_share (struct mm_struct *mm,unsigned long addr,pud_t *pud); +pte_t *huge_pmd_share (struct mm_struct *mm,struct vm_area_struct *vma, +unsigned long addr,pud_t *pud);struct address_space *hugetlb_page_mapping_lock_write (struct page *hpage);
@@ -161,7 +162,7 @@ extern struct list_head huge_boot_pages;/* arch callbacks */ -pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long sz);pte_t *huge_pte_offset (struct mm_struct *mm,unsigned long addr,unsigned long sz);
mm/hugetlb.c +8 -7
@@ -3807,7 +3807,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, src_pte =huge_pte_offset (src, addr, sz);if (!src_pte)continue ; - dst_pte =huge_pte_alloc (dst, addr, sz); + dst_pte =huge_pte_alloc (dst, vma, addr, sz);if (!dst_pte) { ret = -ENOMEM ;break ;
@@ -4544,7 +4544,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,*/ mapping = vma->vm_file->f_mapping;i_mmap_lock_read (mapping); - ptep =huge_pte_alloc (mm, haddr,huge_page_size (h)); + ptep =huge_pte_alloc (mm, vma, haddr,huge_page_size (h));if (!ptep) {i_mmap_unlock_read (mapping);return VM_FAULT_OOM ;
@@ -5334,9 +5334,9 @@ void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,* if !vma_shareable check at the beginning of the routine. i_mmap_rwsem is * only required for subsequent processing. */ -pte_t *huge_pmd_share (struct mm_struct *mm,unsigned long addr,pud_t *pud) +pte_t *huge_pmd_share (struct mm_struct *mm,struct vm_area_struct *vma, +unsigned long addr,pud_t *pud) { -struct vm_area_struct *vma =find_vma (mm, addr);struct address_space *mapping = vma->vm_file->f_mapping;pgoff_t idx = ((addr - vma->vm_start) >>PAGE_SHIFT ) + vma->vm_pgoff;
@@ -5414,7 +5414,8 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, }#define want_pmd_share () (1)#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ -pte_t *huge_pmd_share (struct mm_struct *mm,unsigned long addr,pud_t *pud) +pte_t *huge_pmd_share (struct mm_struct *mm,struct vm_area_struct vma, +unsigned long addr,pud_t *pud) {return NULL ; }
@@ -5433,7 +5434,7 @@ void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB -pte_t *huge_pte_alloc (struct mm_struct *mm, +pte_t *huge_pte_alloc (struct mm_struct *mm,struct vm_area_struct *vma,unsigned long addr,unsigned long sz) {pgd_t *pgd;
@@ -5452,7 +5453,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, }else {BUG_ON (sz !=PMD_SIZE );if (want_pmd_share () &&pud_none (*pud)) - pte =huge_pmd_share (mm, addr, pud); + pte =huge_pmd_share (mm, vma, addr, pud);else pte = (pte_t *)pmd_alloc (mm, pud, addr); }
mm/userfaultfd.c +1 -1
@@ -290,7 +290,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,mutex_lock (&hugetlb_fault_mutex_table[hash]); err = -ENOMEM ; - dst_pte =huge_pte_alloc (dst_mm, dst_addr, vma_hpagesize); + dst_pte =huge_pte_alloc (dst_mm, dst_vma, dst_addr, vma_hpagesize);if (!dst_pte) {mutex_unlock (&hugetlb_fault_mutex_table[hash]);i_mmap_unlock_read (mapping);
Signature · 1 line
2.26.2
kernel test robot
build botRe: [PATCH 1/4] hugetlb: Pass vma into huge_pte_alloc() and huge_pmd_share()
Thank you for the patch! Yet something to improve:
Hi Peter,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on arm64/for-next/core]
[also build test ERROR on linux/master linus/master hnaz-linux-mm/master v5.11 next-20210217] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Peter-Xu/hugetlb-Disable-huge-pmd-unshare-for-uffd-wp/20210218-003520 base: https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git for-next/core config: sparc-allyesconfig (attached as .config) compiler: sparc64-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/58ca9ac367f24752915f352fa9ae1ae027a29de3
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Peter-Xu/hugetlb-Disable-huge-pmd-unshare-for-uffd-wp/20210218-003520 git checkout 58ca9ac367f24752915f352fa9ae1ae027a29de3
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=sparc
If you fix the issue, kindly add following tag as appropriate
Reported-by kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
>> arch/sparc/mm/hugetlbpage.c:285:1: error: expected ';', ',' or ')' before '{' token
285 | {
| ^
>> arch/sparc/mm/hugetlbpage.c:269:22: error: 'huge_tte_to_size' defined but not used [-Werror=unused-function] 269 | static unsigned long huge_tte_to_size(pte_t pte)
| ^~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
vim +285 arch/sparc/mm/hugetlbpage.c
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 268
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 @269 static unsigned long huge_tte_to_size(pte_t pte)
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 270 {
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 271 unsigned long size = 1UL << huge_tte_to_shift(pte);
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 272
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 273 if (size == REAL_HPAGE_SIZE)
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 274 size = HPAGE_SIZE;
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 275 return size;
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 276 }
c7d9f77d33a779 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-01 277
e6e4f42eb773c1 arch/sparc/mm/hugetlbpage.c Peter Zijlstra 2020-11-13 278 unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); }
e6e4f42eb773c1 arch/sparc/mm/hugetlbpage.c Peter Zijlstra 2020-11-13 279 unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); }
e6e4f42eb773c1 arch/sparc/mm/hugetlbpage.c Peter Zijlstra 2020-11-13 280 unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); }
e6e4f42eb773c1 arch/sparc/mm/hugetlbpage.c Peter Zijlstra 2020-11-13 281
a5516438959d90 arch/sparc64/mm/hugetlbpage.c Andi Kleen 2008-07-23 282 pte_t *huge_pte_alloc(struct mm_struct *mm,
58ca9ac367f247 arch/sparc/mm/hugetlbpage.c Peter Xu 2021-02-17 283 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
a5516438959d90 arch/sparc64/mm/hugetlbpage.c Andi Kleen 2008-07-23 284 unsigned long addr, unsigned long sz)
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 @285 {
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 286 pgd_t *pgd;
5637bc50483404 arch/sparc/mm/hugetlbpage.c Mike Rapoport 2019-11-24 287 p4d_t *p4d;
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 288 pud_t *pud;
dcd1912d21a025 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-06 289 pmd_t *pmd;
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 290
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 291 pgd = pgd_offset(mm, addr);
5637bc50483404 arch/sparc/mm/hugetlbpage.c Mike Rapoport 2019-11-24 292 p4d = p4d_offset(pgd, addr);
5637bc50483404 arch/sparc/mm/hugetlbpage.c Mike Rapoport 2019-11-24 293 pud = pud_alloc(mm, p4d, addr);
df7b2155bbe755 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-08-11 294 if (!pud)
df7b2155bbe755 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-08-11 295 return NULL;
df7b2155bbe755 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-08-11 296 if (sz >= PUD_SIZE)
4dbe87d5a70215 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-08-11 297 return (pte_t *)pud;
dcd1912d21a025 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-06 298 pmd = pmd_alloc(mm, pud, addr);
dcd1912d21a025 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-06 299 if (!pmd)
dcd1912d21a025 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-02-06 300 return NULL;
59f1183dd368f1 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-03-03 301 if (sz >= PMD_SIZE)
4dbe87d5a70215 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-08-11 302 return (pte_t *)pmd;
4dbe87d5a70215 arch/sparc/mm/hugetlbpage.c Nitin Gupta 2017-08-11 303 return pte_alloc_map(mm, pmd, addr);
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 304 }
^1da177e4c3f41 arch/sparc64/mm/hugetlbpage.c Linus Torvalds 2005-04-16 305 ---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Peter Xu
reviewerPATCH 2/4 hugetlb/userfaultfd: Forbid huge pmd sharing when uffd enabled
In reply to Peter XuHuge pmd sharing could bring problem to userfaultfd. The thing is that userfaultfd is running its logic based on the special bits on page table entries, however the huge pmd sharing could potentially share page table entries for different address ranges. That could cause issues on either:
- When sharing huge pmd page tables for an uffd write protected range, the newly mapped huge pmd range will also be write protected unexpectedly, or,
- When we try to write protect a range of huge pmd shared range, we'll first do huge_pmd_unshare() in hugetlb_change_protection(), however that also means the UFFDIO_WRITEPROTECT could be silently skipped for the shared region, which could lead to data loss.
Since at it, a few other things are done altogether:
- Move want_pmd_share() from mm/hugetlb.c into linux/hugetlb.h, because that's definitely something that arch code would like to use too
- ARM64 currently directly check against CONFIG_ARCH_WANT_HUGE_PMD_SHARE when trying to share huge pmd. Switch to the want_pmd_share() helper.
Since at it, move vma_shareable() from huge_pmd_share() into want_pmd_share().
Signed-off-by Peter Xu <[email protected]>
--- arch/arm64/mm/hugetlbpage.c | 3 +-- include/linux/hugetlb.h | 2 ++ include/linux/userfaultfd_k.h | 9 +++++++++ mm/hugetlb.c | 20 ++++++++++++++------ 4 files changed, 26 insertions(+), 8 deletions(-)
arch/arm64/mm/hugetlbpage.c +1 -2
@@ -284,8 +284,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,*/ ptep =pte_alloc_map (mm, pmdp, addr); }else if (sz ==PMD_SIZE ) { -if (IS_ENABLED (CONFIG_ARCH_WANT_HUGE_PMD_SHARE ) && -pud_none (READ_ONCE (*pudp))) +if (want_pmd_share (vma, addr) &&pud_none (READ_ONCE (*pudp))) ptep =huge_pmd_share (mm, vma, addr, pudp);else ptep = (pte_t *)pmd_alloc (mm, pudp, addr);
include/linux/hugetlb.h +2 -0
@@ -950,4 +950,6 @@ static inline __init void hugetlb_cma_check(void) }#endif +bool want_pmd_share (struct vm_area_struct *vma,unsigned long addr); +#endif /* _LINUX_HUGETLB_H */
include/linux/userfaultfd_k.h +9 -0
@@ -52,6 +52,15 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma,return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx; } +/* +* Never enable huge pmd sharing on uffd-wp registered vmas, because uffd-wp +* protect information is per pgtable entry. +*/ +static inline bool uffd_disable_huge_pmd_share (struct vm_area_struct *vma) +{ +return vma->vm_flags &VM_UFFD_WP ; +} +static inline bool userfaultfd_missing (struct vm_area_struct *vma) {return vma->vm_flags &VM_UFFD_MISSING ;
mm/hugetlb.c +14 -6
@@ -5292,6 +5292,18 @@ static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)return false; } +bool want_pmd_share (struct vm_area_struct *vma,unsigned long addr) +{ +#ifndef CONFIG_ARCH_WANT_HUGE_PMD_SHARE +return false; +#endif +#ifdef CONFIG_USERFAULTFD +if (uffd_disable_huge_pmd_share (vma)) +return false; +#endif +return vma_shareable (vma, addr); +} +/* * Determine if start,end range within vma could be mapped by shared pmd. * If yes, adjust start and end to cover range associated with possible
@@ -5346,9 +5358,6 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,pte_t *pte;spinlock_t *ptl; -if (!vma_shareable (vma, addr)) -return (pte_t *)pmd_alloc (mm, pud, addr); -i_mmap_assert_locked (mapping); vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {if (svma == vma)
@@ -5412,7 +5421,7 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, *addr =ALIGN (*addr,HPAGE_SIZE *PTRS_PER_PTE ) -HPAGE_SIZE ; return 1; } -#define want_pmd_share () (1) +#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ pte_t *huge_pmd_share (struct mm_struct *mm,struct vm_area_struct vma,unsigned long addr,pud_t *pud)
@@ -5430,7 +5439,6 @@ void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { } -#define want_pmd_share () (0)#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
@@ -5452,7 +5460,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, pte = (pte_t *)pud; }else {BUG_ON (sz !=PMD_SIZE ); -if (want_pmd_share () &&pud_none (*pud)) +if (want_pmd_share (vma, addr) &&pud_none (*pud)) pte =huge_pmd_share (mm, vma, addr, pud);else pte = (pte_t *)pmd_alloc (mm, pud, addr);
Signature · 1 line
2.26.2
Peter Xu
reviewerPATCH 3/4 mm/hugetlb: Move flush_hugetlb_tlb_range() into hugetlb.h
In reply to Peter XuPrepare for it to be called outside of mm/hugetlb.c.
Reviewed-by Mike Kravetz <[email protected]>
Signed-off-by Peter Xu <[email protected]>
--- include/linux/hugetlb.h | 8 ++++++++ mm/hugetlb.c | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-)
include/linux/hugetlb.h +8 -0
@@ -952,4 +952,12 @@ static inline __init void hugetlb_cma_check(void)bool want_pmd_share (struct vm_area_struct *vma,unsigned long addr); +#ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE +/* +* ARCHes with special requirements for evicting HUGETLB backing TLB entries can +* implement this. +*/ +#define flush_hugetlb_tlb_range (vma, addr, end) flush_tlb_range(vma, addr, end) +#endif +#endif /* _LINUX_HUGETLB_H */
mm/hugetlb.c +0 -8
@@ -4965,14 +4965,6 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,return i ? i : err; } -#ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE -/* -* ARCHes with special requirements for evicting HUGETLB backing TLB entries can -* implement this. -*/ -#define flush_hugetlb_tlb_range (vma, addr, end) flush_tlb_range(vma, addr, end) -#endif -unsigned long hugetlb_change_protection (struct vm_area_struct *vma,unsigned long address,unsigned long end,pgprot_t newprot) {
Signature · 1 line
2.26.2
Peter Xu
reviewerPATCH 4/4 hugetlb/userfaultfd: Unshare all pmds for hugetlbfs when register wp
In reply to Peter XuHuge pmd sharing for hugetlbfs is racy with userfaultfd-wp because userfaultfd-wp is always based on pgtable entries, so they cannot be shared.
Walk the hugetlb range and unshare all such mappings if there is, right before UFFDIO_REGISTER will succeed and return to userspace.
This will pair with want_pmd_share() in hugetlb code so that huge pmd sharing is completely disabled for userfaultfd-wp registered range.
Signed-off-by Peter Xu <[email protected]>
--- fs/userfaultfd.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+)
fs/userfaultfd.c +57 -0
@@ -15,6 +15,7 @@#include <linux/sched/signal.h> #include <linux/sched/mm.h> #include <linux/mm.h> +#include <linux/mmu_notifier.h> #include <linux/poll.h> #include <linux/slab.h> #include <linux/seq_file.h>
@@ -1190,6 +1191,59 @@ static ssize_t userfaultfd_read(struct file *file, char __user *buf, } } +/* +* This function will unconditionally remove all the shared pmd pgtable entries +* within the specific vma for a hugetlbfs memory range. +*/ +static void hugetlb_unshare_all_pmds (struct vm_area_struct *vma) +{ +#ifdef CONFIG_HUGETLB_PAGE +struct hstate *h =hstate_vma (vma); +unsigned long sz =huge_page_size (h); +struct mm_struct *mm = vma->vm_mm; +struct mmu_notifier_range range; +unsigned long address, start, end; +spinlock_t *ptl; +pte_t *ptep; + +if (!(vma->vm_flags &VM_MAYSHARE )) +return ; + + start =ALIGN (vma->vm_start,PUD_SIZE ); + end =ALIGN_DOWN (vma->vm_end,PUD_SIZE ); + +if (start >= end) +return ; + +/* +* No need to call adjust_range_if_pmd_sharing_possible(), because +* we're going to operate on the whole vma +*/ +mmu_notifier_range_init (&range,MMU_NOTIFY_CLEAR ,0 , vma, mm, + vma->vm_start, vma->vm_end); +mmu_notifier_invalidate_range_start (&range); +i_mmap_lock_write (vma->vm_file->f_mapping); +for (address = start; address < end; address +=PUD_SIZE ) { +unsigned long tmp = address; + + ptep =huge_pte_offset (mm, address, sz); +if (!ptep) +continue ; + ptl =huge_pte_lock (h, mm, ptep); +/* We don't want 'address' to be changed */ +huge_pmd_unshare (mm, vma, &tmp, ptep); +spin_unlock (ptl); + } +flush_hugetlb_tlb_range (vma, vma->vm_start, vma->vm_end); +i_mmap_unlock_write (vma->vm_file->f_mapping); +/* +* No need to call mmu_notifier_invalidate_range(), see +* Documentation/vm/mmu_notifier.rst. +*/ +mmu_notifier_invalidate_range_end (&range); +#endif +} +static void __wake_userfault (struct userfaultfd_ctx *ctx,struct userfaultfd_wake_range *range) {
@@ -1448,6 +1502,9 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx.ctx = ctx; +if (is_vm_hugetlb_page (vma) &&uffd_disable_huge_pmd_share (vma)) +hugetlb_unshare_all_pmds (vma); + skip: prev = vma; start = vma->vm_end;
Signature · 1 line
2.26.2