Consolidate lots of hugepage code
Subsystems (3)
RFC 1/1 Consolidate lots of hugepage code
wA lot of the code in arch/*/mm/hugetlbpage.c is quite similar. This patch attempts to consolidate a lot of the code across the arch's, putting the combined version in mm/hugetlb.c. There are a couple of uglyish hacks in order to cover all the hugepage archs, but the result is a very large reduction in the total amount of code. It also means things like hugepage lazy allocation could be implemented in one place, instead of six.
As yet this is entirely untested, except on ppc64. Comments? Objections? Testing acks?
Notes:
- this patch changes the meaning of set_huge_pte() to be more analagous to set_pte()
- does SH4 need special huge_ptep_get_and_clear()??
mm/hugetlb.c +177 -1
@@ -7,9 +7,13 @@#include <linux/init.h> #include <linux/module.h> #include <linux/mm.h> -#include <linux/hugetlb.h> #include <linux/sysctl.h> #include <linux/highmem.h> +#include <linux/pagemap.h> +#include <asm/page.h> +#include <asm/pgtable.h> + +#include <linux/hugetlb.h> const unsigned long hugetlb_zero =0 , hugetlb_infinity = ~0UL ;static unsigned long nr_huge_pages, free_huge_pages;
@@ -248,6 +252,75 @@ .nopage = hugetlb_nopage, }; +pte_t *huge_pte_alloc (struct mm_struct *mm,unsigned long addr); +pte_t *huge_pte_offset (struct mm_struct *mm,unsigned long addr); + +pte_t make_huge_pte (struct vm_area_struct *vma,struct page *page) +{ +pte_t entry; + +if (vma->vm_flags &VM_WRITE ) { + entry = +pte_mkwrite (pte_mkdirty (mk_pte (page, vma->vm_page_prot))); + }else { + entry =pte_wrprotect (mk_pte (page, vma->vm_page_prot)); + } + entry =pte_mkyoung (entry); + entry =pte_mkhuge (entry); + +return entry; +} + +int copy_hugetlb_page_range (struct mm_struct *dst,struct mm_struct *src, +struct vm_area_struct *vma) +{ +pte_t *src_pte, *dst_pte, entry; +struct page *ptepage; +unsigned long addr = vma->vm_start; +unsigned long end = vma->vm_end; + +while (addr < end) { + dst_pte =huge_pte_alloc (dst, addr); +if (!dst_pte) + goto nomem; + src_pte =huge_pte_offset (src, addr); +BUG_ON (!src_pte ||pte_none (*src_pte));/* prefaulted */ + entry = *src_pte; + ptepage =pte_page (entry); +get_page (ptepage); +set_huge_pte (dst_pte, entry); + dst->rss += (HPAGE_SIZE /PAGE_SIZE ); + addr +=HPAGE_SIZE ; + } +return 0 ; + +nomem: +return -ENOMEM ; +} + +void unmap_hugepage_range (struct vm_area_struct *vma,unsigned long start, +unsigned long end) +{ +struct mm_struct *mm = vma->vm_mm; +unsigned long address; +pte_t pte; +struct page *page; + +WARN_ON (!is_vm_hugetlb_page (vma)); +BUG_ON (start & ~HPAGE_MASK ); +BUG_ON (end & ~HPAGE_MASK ); + +for (address = start; address < end; address +=HPAGE_SIZE ) { + pte =huge_ptep_get_and_clear (huge_pte_offset (mm, address)); +if (pte_none (pte)) +continue ; + page =pte_page (pte); +put_page (page); + } + mm->rss -= (end - start) >>PAGE_SHIFT ; +flush_tlb_range (vma, start, end); +} +void zap_hugepage_range (struct vm_area_struct *vma,unsigned long start,unsigned long length) {
@@ -257,3 +330,106 @@unmap_hugepage_range (vma, start, start + length);spin_unlock (&mm->page_table_lock); } + +int hugetlb_prefault (struct address_space *mapping,struct vm_area_struct *vma) +{ +struct mm_struct *mm = current->mm; +unsigned long addr; +int ret =0 ; + +WARN_ON (!is_vm_hugetlb_page (vma)); +BUG_ON (vma->vm_start & ~HPAGE_MASK ); +BUG_ON (vma->vm_end & ~HPAGE_MASK ); + +spin_lock (&mm->page_table_lock); +for (addr = vma->vm_start; addr < vma->vm_end; addr +=HPAGE_SIZE ) { +unsigned long idx; +pte_t *pte =huge_pte_alloc (mm, addr); +struct page *page; + +if (!pte) { + ret = -ENOMEM ; + goto out; + } +if (!pte_none (*pte)) +hugetlb_clean_stale_pgtable (pte); + + idx = ((addr - vma->vm_start) >>HPAGE_SHIFT ) + + (vma->vm_pgoff >> (HPAGE_SHIFT -PAGE_SHIFT )); + page =find_get_page (mapping, idx); +if (!page) { +/* charge the fs quota first */ +if (hugetlb_get_quota (mapping)) { + ret = -ENOMEM ; + goto out; + } + page =alloc_huge_page (); +if (!page) { +hugetlb_put_quota (mapping); + ret = -ENOMEM ; + goto out; + } + ret =add_to_page_cache (page, mapping, idx,GFP_ATOMIC ); +if (! ret) { +unlock_page (page); + }else { +hugetlb_put_quota (mapping); +free_huge_page (page); + goto out; + } + } + mm->rss += (HPAGE_SIZE /PAGE_SIZE ); +set_huge_pte (pte,make_huge_pte (vma, page)); + } +out: +spin_unlock (&mm->page_table_lock); +return ret; +} + +int follow_hugetlb_page (struct mm_struct *mm,struct vm_area_struct *vma, +struct page **pages,struct vm_area_struct **vmas, +unsigned long *position,int *length,int i) +{ +unsigned long vpfn, vaddr = *position; +int remainder = *length; + +BUG_ON (!is_vm_hugetlb_page (vma)); + + vpfn = vaddr/PAGE_SIZE ; +while (vaddr < vma->vm_end && remainder) { + +if (pages) { +pte_t *pte; +struct page *page; + +/* Some archs (sparc64, sh*) have multiple +* pte_ts to each hugepage. We have to make +* sure we get the first, for the page +* indexing below to work. */ + pte =huge_pte_offset (mm, vaddr &HPAGE_MASK ); + +/* hugetlb should be locked, and hence, prefaulted */ +WARN_ON (!pte ||pte_none (*pte)); + + page = &pte_page (*pte)[vpfn % (HPAGE_SIZE /PAGE_SIZE )]; + +WARN_ON (!PageCompound (page)); + +get_page (page); + pages[i] = page; + } + +if (vmas) + vmas[i] = vma; + + vaddr +=PAGE_SIZE ; + ++vpfn; + --remainder; + ++i; + } + + *length = remainder; + *position = vaddr; + +return i; +}
arch/ppc64/mm/hugetlbpage.c +2 -177
215 lines not on this page · raw message
arch/ia64/mm/hugetlbpage.c +2 -158
190 lines not on this page · raw message
arch/i386/mm/hugetlbpage.c +8 -166
207 lines not on this page · raw message
arch/sh64/mm/hugetlbpage.c +17 -171
223 lines not on this page · raw message
arch/sh/mm/hugetlbpage.c +17 -171
223 lines not on this page · raw message
arch/sparc64/mm/hugetlbpage.c +17 -171
223 lines not on this page · raw message
include/asm-sh/pgtable.h +1 -0
@@ -194,6 +194,7 @@static inline pte_t pte_mkdirty (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_DIRTY));return pte; }static inline pte_t pte_mkyoung (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_ACCESSED));return pte; }static inline pte_t pte_mkwrite (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_RW));return pte; } +static inline pte_t pte_mkhuge (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_SZHUGE));return pte; }/* * Macro and implementation to make a page protection as uncachable.
include/asm-ia64/pgtable.h +1 -0
@@ -281,6 +281,7 @@#define pte_mkyoung (pte) (__pte(pte_val(pte) | _PAGE_A))#define pte_mkclean (pte) (__pte(pte_val(pte) & ~_PAGE_D))#define pte_mkdirty (pte) (__pte(pte_val(pte) | _PAGE_D)) +#define pte_mkhuge (entry) (__pte(pte_val(pte) | _PAGE_P))/* * Macro to a page protection value as "uncacheable". Note that "protection" is really a
include/asm-i386/pgtable.h +1 -1
@@ -236,6 +236,7 @@static inline pte_t pte_mkdirty (pte_t pte) { (pte).pte_low |= _PAGE_DIRTY;return pte; }static inline pte_t pte_mkyoung (pte_t pte) { (pte).pte_low |= _PAGE_ACCESSED;return pte; }static inline pte_t pte_mkwrite (pte_t pte) { (pte).pte_low |= _PAGE_RW;return pte; } +static inline pte_t pte_mkhuge (pte_t pte) { (pte).pte_low |= _PAGE_PRESENT | _PAGE_PSE;return pte; }#ifdef CONFIG_X86_PAE # include <asm/pgtable-3level.h>
@@ -273,7 +274,6 @@*/ #define mk_pte (page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) -#define mk_pte_huge (entry) ((entry).pte_low |= _PAGE_PRESENT | _PAGE_PSE)static inline pte_t pte_modify (pte_t pte,pgprot_t newprot) {
include/asm-sparc64/page.h +1 -0
@@ -93,6 +93,7 @@#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT)#define HPAGE_MASK (~(HPAGE_SIZE - 1UL))#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) +#define ARCH_HAS_SETCLEAR_HUGE_PTE #endif #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \
include/asm-sparc64/pgtable.h +1 -0
@@ -302,6 +302,7 @@#define pte_mkyoung (pte) (__pte(pte_val(pte) | _PAGE_ACCESSED | _PAGE_R))#define pte_mkwrite (pte) (__pte(pte_val(pte) | _PAGE_WRITE))#define pte_mkdirty (pte) (__pte(pte_val(pte) | _PAGE_MODIFIED | _PAGE_W)) +#define pte_mkhuge (pte) (__pte(pte_val(pte) | _PAGE_SZHUGE))/* to find an entry in a page-table-directory. */ #define pgd_index (address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD))
include/asm-sh/page.h +1 -0
@@ -31,6 +31,7 @@#define HPAGE_SIZE (1UL << HPAGE_SHIFT)#define HPAGE_MASK (~(HPAGE_SIZE-1))#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT-PAGE_SHIFT) +#define ARCH_HAS_SETCLEAR_HUGE_PTE #endif #ifdef __KERNEL__
include/asm-sh64/pgtable.h +2 -0
@@ -429,6 +429,8 @@extern inline pte_t pte_mkexec (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_EXECUTE));return pte; }extern inline pte_t pte_mkdirty (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_DIRTY));return pte; }extern inline pte_t pte_mkyoung (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_ACCESSED));return pte; } +extern inline pte_t pte_mkhuge (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_SZHUGE));return pte; } +/* * Conversion functions: convert a page and protection to a page entry.
include/asm-sh64/page.h +1 -0
@@ -41,6 +41,7 @@#define HPAGE_SIZE (1UL << HPAGE_SHIFT)#define HPAGE_MASK (~(HPAGE_SIZE-1))#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT-PAGE_SHIFT) +#define ARCH_HAS_SETCLEAR_HUGE_PTE #endif #ifdef __KERNEL__
include/linux/hugetlb.h +14 -0
@@ -47,6 +47,20 @@int prepare_hugepage_range (unsigned long addr,unsigned long len);#endif +#ifndef ARCH_HAS_SETCLEAR_HUGE_PTE +#define set_huge_pte (ptep, pte) set_pte(ptep, pte) +#define huge_ptep_get_and_clear (ptep) ptep_get_and_clear(ptep) +#else +void set_huge_pte (pte_t *ptep,pte_t pte); +pte_t huge_ptep_get_and_clear (pte_t *ptep); +#endif + +#ifndef ARCH_HAS_HUGETLB_CLEAN_STALE_PGTABLE +#define hugetlb_clean_stale_pgtable (pte) BUG() +#else +void hugetlb_clean_stale_pgtable (pte_t *pte); +#endif +#else /* !CONFIG_HUGETLB_PAGE */ static inline int is_vm_hugetlb_page (struct vm_area_struct *vma)
include/asm-i386/page.h +1 -0
@@ -64,6 +64,7 @@#define HPAGE_MASK (~(HPAGE_SIZE - 1))#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA +#define ARCH_HAS_HUGETLB_CLEAN_STALE_PGTABLE #endif
Signature · 4 lines
David Gibson | For every complex problem there is a david AT gibson.dropbear.id.au | solution which is simple, neat and | wrong. http://www.ozlabs.org/people/dgibson
Quoted 5 lines · David Gibson
wA lot of the code in
arch/*/mm/hugetlbpage.cis quite similar. This patch attempts to consolidate a lot of the code across the arch's, putting the combined version inmm/hugetlb.c. There are a couple of uglyish hacks in order to cover all the hugepage archs, but the result is a very large reduction in the total amount of code. It also means
Quoted 8 lines
things like hugepage lazy allocation could be implemented in one place, instead of six.
As yet this is entirely untested, except on ppc64. Comments? Objections? Testing acks?
Notes:
- this patch changes the meaning ofset_huge_pte()to be more analagous toset_pte()
- does SH4 need specialhuge_ptep_get_and_clear()??
Further consolidation is premature given that outstanding hugetlb bugs have the implication that architectures' needs are not being served by the current arch/core split. I have at least two relatively major hugetlb bugs outstanding, the lack of a flush_dcache_page() analogue first, and another (soon to be a reported to affected distros) less well-understood. Unless they're directly toward the end of restoring hugetlb to a sound state, they're counterproductive to merge before patches doing so.
-- wli
Hi,
Quoted 7 lines
Further consolidation is premature given that outstanding hugetlb bugs have the implication that architectures' needs are not being served by the current
arch/coresplit. I have at least two relatively major hugetlb bugs outstanding, the lack of aflush_dcache_page()analogue first, and another (soon to be a reported to affected distros) less well-understood. Unless they're directly toward the end of restoring hugetlb to a sound state, they're counterproductive to merge before patches doing so.
Could you point me at a summary of these 2 issues?
Anton
Quoted 7 lines · At some point in the past, I wrote:
Quoted 7 lines
Further consolidation is premature given that outstanding hugetlb bugs have the implication that architectures' needs are not being served by the current
arch/coresplit. I have at least two relatively major hugetlb bugs outstanding, the lack of aflush_dcache_page()analogue first, and another (soon to be a reported to affected distros) less well-understood. Unless they're directly toward the end of restoring hugetlb to a sound state, they're counterproductive to merge before patches doing so.
Quoted 1 line · Anton Blanchard
Could you point me at a summary of these 2 issues?
It's all pretty obvious. The first is checking page size vs. cache size and whether it's VI or does anything unusual; thus far things look hopeful that flush_dcache_page() analogues are unnecessary. More information about Super-H is needed to wrap up what will probably be no more than an audit. The second is a triplefault on x86-64 under some condition involving a long-running database regression test. There has obviously been considerably less progress there in no small part due to the amount of time required to reproduce the issue.
-- wli
Hi,
Quoted 5 lines
It's all pretty obvious. The first is checking page size vs. cache size and whether it's VI or does anything unusual; thus far things look hopeful that
flush_dcache_page()analogues are unnecessary. More information about Super-H is needed to wrap up what will probably be no more than an audit.
Good to hear.
Quoted 4 lines
The second is a triplefault on x86-64 under some
condition involving a long-running database regression test. There has obviously been considerably less progress there in no small part due to the amount of time required to reproduce the issue.
OK. We have not seen a similar issue on ppc64 even with extensive testing (although with HPC apps). The question is how long we should hold off on further hugetlb development waiting for this one bug report on a single architecture to be chased.
Anton
Quoted 4 lines · At some point in the past, I wrote:
Quoted 4 lines
The second is a triplefault on x86-64 under some
condition involving a long-running database regression test. There has obviously been considerably less progress there in no small part due to the amount of time required to reproduce the issue.
Quoted 4 lines · Anton Blanchard
OK. We have not seen a similar issue on ppc64 even with extensive testing (although with HPC apps). The question is how long we should hold off on further hugetlb development waiting for this one bug report on a single architecture to be chased.
Until it's fixed. Until then I'm considering it a byproduct of that same development. And with your report, that makes it two architectures, not one.
The concepts of the features etc. are all generally okay, though very buzzword-centric. In general the audits and sweeps have been lacking thoroughness in the architecture-specific areas. I expect that particular issue to have been the cause of these two bugreports.
-- wli
Quoted 10 lines · Anton Blanchard
Quoted 4 lines
OK. We have not seen a similar issue on ppc64 even with extensive testing (although with HPC apps). The question is how long we should hold off on further hugetlb development waiting for this one bug report on a single architecture to be chased.
Until it's fixed. Until then I'm considering it a byproduct of that same development. And with your report, that makes it two architectures, not one.
We _arent_ seeing it on ppc64. Can we at least have a complete bug report if we are to halt all hugetlb development? At the moment we dont have much information to go on at all.
Anton
Quoted 3 lines · At some point in the past, I wrote:
Quoted 3 lines
Until it's fixed. Until then I'm considering it a byproduct of that same development. And with your report, that makes it two architectures, not one.
Quoted 3 lines · Anton Blanchard
We _arent_ seeing it on ppc64. Can we at least have a complete bug report if we are to halt all hugetlb development? At the moment we dont have much information to go on at all.
Sorry, I don't get complete bugreports myself. If you care to try to actually fix something (it's doubtful you yourself are the culprit) I'm still trying to reproduce it myself with long-running database tests. It's reliably reproducible on the reporters' machines.
The particular bug is only one piece of evidence. Just asking basic questions about what was done for architecture code reveals that all this "development" is not paying proper attention to architecture code. I merely insist that development toward the end of stabilization occur prior to that for large feature work.
And frankly, I'm rather unimpressed with the gravity of the proposed featurework, particularly in comparison to the stability requirements of users on typical production systems.
Nor am I impressed with the quality. The patch presentations have been messy, the audits (as mentioned above) incomplete, the benefits not clearly demonstrated, and the code itself not so pretty. Just respinning the patches so they're properly incremental and the code somewhat cleaner (e.g. some recent one nested tabs 5 deep or so) would already remedy a large number of the issues with the featurework. Once arranged that way the audits' incompleteness can be dealt with by those with the fortitude to thoroughly audit and/or prior architecture knowledge to correct the patches for arches they don't deal with properly.
-- wli
Hi wli,
Any progress on this? If not Id like to suggest we get Davids patch into -mm.
Anton
Quoted 27 lines
Sorry, I don't get complete bugreports myself. If you care to try to actually fix something (it's doubtful you yourself are the culprit) I'm still trying to reproduce it myself with long-running database tests. It's reliably reproducible on the reporters' machines.
The particular bug is only one piece of evidence. Just asking basic questions about what was done for architecture code reveals that all this "development" is not paying proper attention to architecture code. I merely insist that development toward the end of stabilization occur prior to that for large feature work.
And frankly, I'm rather unimpressed with the gravity of the proposed featurework, particularly in comparison to the stability requirements of users on typical production systems.
Nor am I impressed with the quality. The patch presentations have been messy, the audits (as mentioned above) incomplete, the benefits not clearly demonstrated, and the code itself not so pretty. Just respinning the patches so they're properly incremental and the code somewhat cleaner (e.g. some recent one nested tabs 5 deep or so) would already remedy a large number of the issues with the featurework. Once arranged that way the audits' incompleteness can be dealt with by those with the fortitude to thoroughly audit and/or prior architecture knowledge to correct the patches for arches they don't deal with properly.
-- wli