fixes for large mm_populate() and munlock() operations
Subsystems (4)
v3 cover fixes for large mm_populate() and munlock() operations
These 3 changes are to improve the handling of large mm_populate and munlock operations. They apply on top of mmotm (in particular, they depend on both my prior mm_populate work and Kirill's "thp: avoid dumping huge zero page" change).
- Patch 1 fixes an integer overflow issue when populating 2^32 pages. The nr_pages argument to get_user_pages would overflow, resulting in 0 pages being processed per iteration. I am proposing to simply convert the nr_pages argument to an unsigned long.
- Patch 2 accelerates populating regions with THP pages. get_user_pages() can increment the address by a huge page size in this case instead of a small page size, and avoid repeated mm->page_table_lock acquisitions. This fixes an issue reported by Roman Dubtsov where populating regions via mmap MAP_POPULATE was significantly slower than doing so by touching pages from userspace.
- Patch 3 is a similar acceleration for the munlock case.
Changes between v1 and v2:
- Andrew accepted patch 1 into his -mm tree but suggested the nr_pages argument type should actually be unsigned long; I am sending this as a "fix" for the previous patch 1 to be collapsed over the previous one.
- In patch 2, I am adding a separate follow_page_mask() function so that the callers to the original follow_page() don't have to be modified to ignore the returned page_mask (following another suggestion from Andrew). Also the page_mask argument type was changed to unsigned int.
- In patch 3, I similarly changed the page_mask values to unsigned int.
Changes between v2 and v3:
- In patch 1, updated mm/nommu.c to match the updated gup function prototype and avoid breaking the nommu build.
- In patch 1, removed incorrect VM_BUG_ON in mm/mlock.c
- In patch 3, fixed munlock_vma_page() to return a page mask as expected by munlock_vma_pages_range() instead of a number of pages.
Michel Lespinasse (3):
mm: use long type for page counts in mm_populate() and get_user_pages()
mm: accelerate mm_populate() treatment of THP pages
mm: accelerate munlock() treatment of THP pages
include/linux/hugetlb.h | 6 +++--- include/linux/mm.h | 28 +++++++++++++++++++--------- mm/hugetlb.c | 12 ++++++------ mm/internal.h | 2 +- mm/memory.c | 49 ++++++++++++++++++++++++++++++++----------------- mm/mlock.c | 38 +++++++++++++++++++++++++------------- mm/nommu.c | 21 ++++++++++++--------- 7 files changed, 98 insertions(+), 58 deletions(-)
Signature · 7 lines
1.8.1 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 1/3 mm: use long type for page counts in mm_populate() and get_user_pages()
Use long type for page counts in mm_populate() so as to avoid integer overflow when running the following test code:
int main(void) {
void *p = mmap(NULL, 0x100000000000, PROT_READ,
MAP_PRIVATE | MAP_ANON, -1, 0);
printf("p: %p\n", p);
mlockall(MCL_CURRENT);
printf("done\n");
return 0;
}
Signed-off-by Michel Lespinasse <[email protected]>
--- include/linux/hugetlb.h | 6 +++--- include/linux/mm.h | 15 ++++++++------- mm/hugetlb.c | 12 ++++++------ mm/memory.c | 18 +++++++++--------- mm/mlock.c | 4 ++-- mm/nommu.c | 15 ++++++++------- 6 files changed, 36 insertions(+), 34 deletions(-)
include/linux/hugetlb.h +3 -3
@@ -43,9 +43,9 @@ int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int,#endif int copy_hugetlb_page_range (struct mm_struct *,struct mm_struct *,struct vm_area_struct *); -int follow_hugetlb_page (struct mm_struct *,struct vm_area_struct *, -struct page **,struct vm_area_struct **, -unsigned long *,int *,int ,unsigned int flags); +long follow_hugetlb_page (struct mm_struct *,struct vm_area_struct *, +struct page **,struct vm_area_struct **, +unsigned long *,unsigned long *,long ,unsigned int );void unmap_hugepage_range (struct vm_area_struct *,unsigned long ,unsigned long ,struct page *); void __unmap_hugepage_range_final(struct mmu_gather *tlb,
include/linux/mm.h +8 -7
@@ -1040,13 +1040,14 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *extern int access_remote_vm (struct mm_struct *mm,unsigned long addr,void *buf,int len,int write); -int __get_user_pages (struct task_struct *tsk,struct mm_struct *mm, -unsigned long start,int len,unsigned int foll_flags, -struct page **pages,struct vm_area_struct **vmas, -int *nonblocking); -int get_user_pages (struct task_struct *tsk,struct mm_struct *mm, -unsigned long start,int nr_pages,int write,int force, -struct page **pages,struct vm_area_struct **vmas); +long __get_user_pages (struct task_struct *tsk,struct mm_struct *mm, +unsigned long start,unsigned long nr_pages, +unsigned int foll_flags,struct page **pages, +struct vm_area_struct **vmas,int *nonblocking); +long get_user_pages (struct task_struct *tsk,struct mm_struct *mm, +unsigned long start,unsigned long nr_pages, +int write,int force,struct page **pages, +struct vm_area_struct **vmas);int get_user_pages_fast (unsigned long start,int nr_pages,int write,struct page **pages);struct kvec ;
mm/hugetlb.c +6 -6
@@ -2924,14 +2924,14 @@ follow_huge_pud(struct mm_struct *mm, unsigned long address,return NULL ; } -int follow_hugetlb_page (struct mm_struct *mm,struct vm_area_struct *vma, -struct page **pages,struct vm_area_struct **vmas, -unsigned long *position,int *length,int i, -unsigned int flags) +long follow_hugetlb_page (struct mm_struct *mm,struct vm_area_struct *vma, +struct page **pages,struct vm_area_struct **vmas, +unsigned long *position,unsigned long *nr_pages, +long i,unsigned int flags) {unsigned long pfn_offset;unsigned long vaddr = *position; -int remainder = *length; +unsigned long remainder = *nr_pages;struct hstate *h =hstate_vma (vma);spin_lock (&mm->page_table_lock);
@@ -3001,7 +3001,7 @@ same_page: } }spin_unlock (&mm->page_table_lock); - *length = remainder; + *nr_pages = remainder; *position = vaddr;return i ? i : -EFAULT ;
mm/memory.c +9 -9
@@ -1673,15 +1673,15 @@ static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long add* instead of __get_user_pages. __get_user_pages should be used only if * you need some special @gup_flags. */ -int __get_user_pages (struct task_struct *tsk,struct mm_struct *mm, -unsigned long start,int nr_pages,unsigned int gup_flags, -struct page **pages,struct vm_area_struct **vmas, -int *nonblocking) +long __get_user_pages (struct task_struct *tsk,struct mm_struct *mm, +unsigned long start,unsigned long nr_pages, +unsigned int gup_flags,struct page **pages, +struct vm_area_struct **vmas,int *nonblocking) { -int i; +long i;unsigned long vm_flags; -if (nr_pages <=0 ) +if (!nr_pages)return 0 ;VM_BUG_ON (!!pages != !!(gup_flags &FOLL_GET ));
@@ -1977,9 +1977,9 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,* * See also get_user_pages_fast, for performance critical applications. */ -int get_user_pages (struct task_struct *tsk,struct mm_struct *mm, -unsigned long start,int nr_pages,int write,int force, -struct page **pages,struct vm_area_struct **vmas) +long get_user_pages (struct task_struct *tsk,struct mm_struct *mm, +unsigned long start,unsigned long nr_pages,int write, +int force,struct page **pages,struct vm_area_struct **vmas) {int flags =FOLL_TOUCH ;
mm/mlock.c +2 -2
@@ -160,7 +160,7 @@ long __mlock_vma_pages_range(struct vm_area_struct *vma, {struct mm_struct *mm = vma->vm_mm;unsigned long addr = start; -int nr_pages = (end - start) /PAGE_SIZE ; +unsigned long nr_pages = (end - start) /PAGE_SIZE ;int gup_flags;VM_BUG_ON (start & ~PAGE_MASK );
@@ -378,7 +378,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)unsigned long end, nstart, nend;struct vm_area_struct *vma =NULL ;int locked =0 ; -int ret =0 ; +long ret =0 ;VM_BUG_ON (start & ~PAGE_MASK );VM_BUG_ON (len !=PAGE_ALIGN (len));
mm/nommu.c +8 -7
@@ -139,10 +139,10 @@ unsigned int kobjsize(const void *objp)return PAGE_SIZE <<compound_order (page); } -int __get_user_pages (struct task_struct *tsk,struct mm_struct *mm, -unsigned long start,int nr_pages,unsigned int foll_flags, -struct page **pages,struct vm_area_struct **vmas, -int *retry) +long __get_user_pages (struct task_struct *tsk,struct mm_struct *mm, +unsigned long start,unsigned long nr_pages, +unsigned int foll_flags,struct page **pages, +struct vm_area_struct **vmas,int *nonblocking) {struct vm_area_struct *vma;unsigned long vm_flags;
@@ -189,9 +189,10 @@ finish_or_fault:* slab page or a secondary page from a compound page * - don't permit access to VMAs that don't support it, such as I/O mappings */ -int get_user_pages (struct task_struct *tsk,struct mm_struct *mm, -unsigned long start,int nr_pages,int write,int force, -struct page **pages,struct vm_area_struct **vmas) +long get_user_pages (struct task_struct *tsk,struct mm_struct *mm, +unsigned long start,unsigned long nr_pages, +int write,int force,struct page **pages, +struct vm_area_struct **vmas) {int flags =0 ;
Signature · 7 lines
1.8.1 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 2/3 mm: accelerate mm_populate() treatment of THP pages
In reply to Michel LespinasseThis change adds a follow_page_mask function which is equivalent to follow_page, but with an extra page_mask argument.
follow_page_mask sets *page_mask to HPAGE_PMD_NR - 1 when it encounters a THP page, and to 0 in other cases.
__get_user_pages() makes use of this in order to accelerate populating THP ranges - that is, when both the pages and vmas arrays are NULL, we don't need to iterate HPAGE_PMD_NR times to cover a single THP page (and we also avoid taking mm->page_table_lock that many times).
Signed-off-by Michel Lespinasse <[email protected]>
--- include/linux/mm.h | 13 +++++++++++-- mm/memory.c | 31 +++++++++++++++++++++++-------- mm/nommu.c | 6 ++++-- 3 files changed, 38 insertions(+), 12 deletions(-)
include/linux/mm.h +11 -2
@@ -1636,8 +1636,17 @@ int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,int vm_insert_mixed (struct vm_area_struct *vma,unsigned long addr,unsigned long pfn); -struct page *follow_page (struct vm_area_struct *,unsigned long address, -unsigned int foll_flags); +struct page *follow_page_mask (struct vm_area_struct *vma, +unsigned long address,unsigned int foll_flags, +unsigned int *page_mask); + +static inline struct page *follow_page (struct vm_area_struct *vma, +unsigned long address,unsigned int foll_flags) +{ +unsigned int unused_page_mask; +return follow_page_mask (vma, address, foll_flags, &unused_page_mask); +} +#define FOLL_WRITE 0x01/* check pte is writable */ #define FOLL_TOUCH 0x02/* mark page accessed */ #define FOLL_GET 0x04/* do get_page on page */
mm/memory.c +23 -8
@@ -1458,10 +1458,11 @@ int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,EXPORT_SYMBOL_GPL (zap_vma_ptes);/** -* follow_page - look up a page descriptor from a user-virtual address +* follow_page_mask - look up a page descriptor from a user-virtual address * @vma: vm_area_struct mapping @address * @address: virtual address to look up * @flags: flags modifying lookup behaviour +* @page_mask: on output, *page_mask is set according to the size of the page * * @flags can have FOLL_ flags set, defined in <linux/mm.h> *
@@ -1469,8 +1470,9 @@ EXPORT_SYMBOL_GPL(zap_vma_ptes);* an error pointer if there is a mapping to something not represented * by a page descriptor (see also vm_normal_page()). */ -struct page *follow_page (struct vm_area_struct *vma,unsigned long address, -unsigned int flags) +struct page *follow_page_mask (struct vm_area_struct *vma, +unsigned long address,unsigned int flags, +unsigned int *page_mask) {pgd_t *pgd;pud_t *pud;
@@ -1480,6 +1482,8 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,struct page *page;struct mm_struct *mm = vma->vm_mm; + *page_mask =0 ; + page =follow_huge_addr (mm, address, flags &FOLL_WRITE );if (!IS_ERR (page)) {BUG_ON (flags &FOLL_GET );
@@ -1526,6 +1530,7 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, page =follow_trans_huge_pmd (vma, address, pmd, flags);spin_unlock (&mm->page_table_lock); + *page_mask =HPAGE_PMD_NR -1 ; goto out; } }else
@@ -1680,6 +1685,7 @@ long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, {long i;unsigned long vm_flags; +unsigned int page_mask;if (!nr_pages)return 0 ;
@@ -1757,6 +1763,7 @@ long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,get_page (page); }pte_unmap (pte); + page_mask =0 ; goto next_page; }
@@ -1774,6 +1781,7 @@ long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, do { struct page *page; unsigned int foll_flags = gup_flags; + unsigned int page_increm; /* * If we have a pending SIGKILL, don't keep faulting
@@ -1783,7 +1791,8 @@ long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,return i ? i : -ERESTARTSYS ;cond_resched (); -while (!(page =follow_page (vma, start, foll_flags))) { +while (!(page =follow_page_mask (vma, start, + foll_flags, &page_mask))) {int ret;unsigned int fault_flags =0 ;
@@ -1857,13 +1866,19 @@ long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,flush_anon_page (vma, page, start);flush_dcache_page (page); + page_mask =0 ; } next_page: -if (vmas) +if (vmas) { vmas[i] = vma; - i++; - start +=PAGE_SIZE ; - nr_pages--; + page_mask =0 ; + } + page_increm =1 + (~(start >>PAGE_SHIFT ) & page_mask); +if (page_increm > nr_pages) + page_increm = nr_pages; + i += page_increm; + start += page_increm *PAGE_SIZE ; + nr_pages -= page_increm; }while (nr_pages && start < vma->vm_end); }while (nr_pages);return i;
mm/nommu.c +4 -2
@@ -1818,9 +1818,11 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,return ret; } -struct page *follow_page (struct vm_area_struct *vma,unsigned long address, -unsigned int foll_flags) +struct page *follow_page_mask (struct vm_area_struct *vma, +unsigned long address,unsigned int flags, +unsigned int *page_mask) { + *page_mask =0 ;return NULL ; }
Signature · 7 lines
1.8.1 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 3/3 mm: accelerate munlock() treatment of THP pages
In reply to Michel Lespinassemunlock_vma_pages_range() was always incrementing addresses by PAGE_SIZE at a time. When munlocking THP pages (or the huge zero page), this resulted in taking the mm->page_table_lock 512 times in a row.
We can do better by making use of the page_mask returned by follow_page_mask (for the huge zero page case), or the size of the page munlock_vma_page() operated on (for the true THP page case).
Note - I am sending this as RFC only for now as I can't currently put my finger on what if anything prevents split_huge_page() from operating concurrently on the same page as munlock_vma_page(), which would mess up our NR_MLOCK statistics. Is this a latent bug or is there a subtle point I missed here ?
Signed-off-by Michel Lespinasse <[email protected]>
--- mm/internal.h | 2 +- mm/mlock.c | 34 +++++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 12 deletions(-)
mm/internal.h +1 -1
@@ -195,7 +195,7 @@ static inline int mlocked_vma_newpage(struct vm_area_struct *vma,* must be called with vma's mmap_sem held for read or write, and page locked. */ extern void mlock_vma_page (struct page *page); -extern void munlock_vma_page (struct page *page); +extern unsigned int munlock_vma_page (struct page *page);/* * Clear the page's PageMlocked(). This can be useful in a situation where
mm/mlock.c +23 -11
@@ -102,13 +102,16 @@ void mlock_vma_page(struct page *page)* can't isolate the page, we leave it for putback_lru_page() and vmscan * [page_referenced()/try_to_unmap()] to deal with. */ -void munlock_vma_page (struct page *page) +unsigned int munlock_vma_page (struct page *page) { +unsigned int page_mask =0 ; +BUG_ON (!PageLocked (page));if (TestClearPageMlocked (page)) { -mod_zone_page_state (page_zone (page),NR_MLOCK , - -hpage_nr_pages (page)); +unsigned int nr_pages =hpage_nr_pages (page); +mod_zone_page_state (page_zone (page),NR_MLOCK , -nr_pages); + page_mask = nr_pages -1 ;if (!isolate_lru_page (page)) {int ret =SWAP_AGAIN ;
@@ -141,6 +144,8 @@ void munlock_vma_page(struct page *page)count_vm_event (UNEVICTABLE_PGMUNLOCKED ); } } + +return page_mask; }/**
@@ -159,7 +164,6 @@ long __mlock_vma_pages_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *nonblocking) { struct mm_struct *mm = vma->vm_mm; -unsigned long addr = start;unsigned long nr_pages = (end - start) /PAGE_SIZE ;int gup_flags;
@@ -185,7 +189,7 @@ long __mlock_vma_pages_range(struct vm_area_struct *vma,if (vma->vm_flags & (VM_READ |VM_WRITE |VM_EXEC )) gup_flags |=FOLL_FORCE ; -return __get_user_pages (current, mm, addr, nr_pages, gup_flags, +return __get_user_pages (current, mm, start, nr_pages, gup_flags,NULL ,NULL , nonblocking); }
@@ -222,13 +226,12 @@ static int __mlock_posix_error_return(long retval)void munlock_vma_pages_range (struct vm_area_struct *vma,unsigned long start,unsigned long end) { -unsigned long addr; - -lru_add_drain (); vma->vm_flags &= ~VM_LOCKED ; -for (addr = start; addr < end; addr +=PAGE_SIZE ) { +while (start < end) {struct page *page; +unsigned int page_mask, page_increm; +/* * Although FOLL_DUMP is intended for get_dump_page(), * it just so happens that its special treatment of the
@@ -236,13 +239,22 @@ void munlock_vma_pages_range(struct vm_area_struct *vma,* suits munlock very well (and if somehow an abnormal page * has sneaked into the range, we won't oops here: great). */ - page =follow_page (vma, addr,FOLL_GET |FOLL_DUMP ); + page =follow_page_mask (vma, start,FOLL_GET |FOLL_DUMP , + &page_mask);if (page && !IS_ERR (page)) {lock_page (page); -munlock_vma_page (page); +lru_add_drain (); +/* +* Any THP page found by follow_page_mask() may have +* gotten split before reaching munlock_vma_page(), +* so we need to recompute the page_mask here. +*/ + page_mask =munlock_vma_page (page);unlock_page (page);put_page (page); } + page_increm =1 + (~(start >>PAGE_SHIFT ) & page_mask); + start += page_increm *PAGE_SIZE ;cond_resched (); } }
Signature · 7 lines
1.8.1 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>