sparc64: Add 16GB hugepage support
cover sparc64: Add 16GB hugepage support
SPARC architecture supports 16G hugepages but the kernel did not support these. This patch series adds support for it and also cleanes up some page walk/alloc functions.
Patch 1/4: Core changes needed to add 16G hugepage support: To map a
single 16G hugepage, two PUD entries are used. Each PUD entry maps 8G portion of a 16G page. This page table encoding scheme is same as that used for hugepages at PMD level (8M, 256M and 2G pages) where each PMD entry points successively to 8M regions within a page. No page table entries below the PUD level are allocated for 16G hugepage since those are not required.
TSB entries for a 16G page are created at every 4M boundary since the HUGE_TSB is used for these pages which is configured with page size of 4M. When walking page tables (on a TSB miss), bits [32:22] are transferred from vaddr to PUD to resolve addresses at 4M boundary. The resolved address mapping is then stored in HUGE_TSB.
Patch 2/4: get_user_pages() etc. are used for direct IO. These
functions were not aware of hugepages at the PUD level and would try to continue walking page tables beyond the PUD level. Since 16G hugepages have page tables allocated till PUD level only, these accesses would result in invalid access. This patch adds the case for PUD huge pages to these functions.
Patch 3/4: Fixes a bug in gup_huge_pmd() which caused wrong page
within a hugepage to be considered as the head page. gup_huge_pud() already has this fix.
Patch 4/4: Patch 1 added the case of PUD entry being huge in page
table walk and alloc functions. This new case further increased nesting in these functions and made them harder to follow. This patch flattens these functions for better readability.
Changelog v4 vs v3:
- Added cover letter (patch 0/4) for patch series.
Changelog v3 vs v2:
- Fixed email headers so the subject shows up correctly.
Changelog v2 vs v1:
- Remove redundant brgez,pn (Bob Picco)
- Remove unncessary label rename from 700 to 701 (Rob Gardner)
- Add patch description (Paul)
- Add 16G case to get_user_pages()
Nitin Gupta (4):
sparc64: Add 16GB hugepage support
sparc64: Support huge PUD case in get_user_pages
sparc64: Fix gup_huge_pmd
sparc64: Cleanup hugepage table walk functions
arch/sparc/include/asm/page_64.h | 3 +- arch/sparc/include/asm/pgtable_64.h | 20 ++++++- arch/sparc/include/asm/tsb.h | 30 +++++++++++ arch/sparc/kernel/tsb.S | 2 +- arch/sparc/mm/gup.c | 49 ++++++++++++++++- arch/sparc/mm/hugetlbpage.c | 102 +++++++++++++++++++++--------------- arch/sparc/mm/init_64.c | 41 ++++++++++++--- 7 files changed, 193 insertions(+), 54 deletions(-)
Signature · 1 line
2.9.2
PATCH 1/4 sparc64: Add 16GB hugepage support
Adds support for 16GB hugepage size. To use this page size use kernel parameters as:
default_hugepageszG hugepageszG hugepages
Testing:
Tested with the stream benchmark which allocates 48G of arrays backed by 16G hugepages and does RW operation on them in parallel.
Orabug: 25362942
--- arch/sparc/include/asm/page_64.h | 3 +- arch/sparc/include/asm/pgtable_64.h | 5 +++ arch/sparc/include/asm/tsb.h | 30 +++++++++++++++ arch/sparc/kernel/tsb.S | 2 +- arch/sparc/mm/hugetlbpage.c | 74 ++++++++++++++++++++++++++----------- arch/sparc/mm/init_64.c | 41 ++++++++++++++++---- 6 files changed, 125 insertions(+), 30 deletions(-)
arch/sparc/include/asm/page_64.h +2 -1
@@ -17,6 +17,7 @@#define HPAGE_SHIFT 23#define REAL_HPAGE_SHIFT 22 +#define HPAGE_16GB_SHIFT 34#define HPAGE_2GB_SHIFT 31#define HPAGE_256MB_SHIFT 28#define HPAGE_64K_SHIFT 16
@@ -28,7 +29,7 @@#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA #define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) -#define HUGE_MAX_HSTATE 4 +#define HUGE_MAX_HSTATE 5#endif #ifndef __ASSEMBLY__
arch/sparc/include/asm/pgtable_64.h +5 -0
@@ -414,6 +414,11 @@ static inline bool is_hugetlb_pmd(pmd_t pmd)return !!(pmd_val (pmd) & _PAGE_PMD_HUGE); } +static inline bool is_hugetlb_pud (pud_t pud) +{ +return !!(pud_val (pud) & _PAGE_PUD_HUGE); +} +#ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline pmd_t pmd_mkhuge (pmd_t pmd) {
arch/sparc/include/asm/tsb.h +30 -0
@@ -195,6 +195,35 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;nop ; \ 699: +/* PUD has been loaded into REG1, interpret the value, seeing +* if it is a HUGE PUD or a normal one. If it is not valid +* then jump to FAIL_LABEL. If it is a HUGE PUD, and it +* translates to a valid PTE, branch to PTE_LABEL. +* +* We have to propagate bits [32:22] from the virtual address +* to resolve at 4M granularity. +*/ +#if defined(CONFIG_HUGETLB_PAGE ) || defined(CONFIG_TRANSPARENT_HUGEPAGE ) +#define USER_PGTABLE_CHECK_PUD_HUGE (VADDR ,REG1 ,REG2 ,FAIL_LABEL ,PTE_LABEL ) \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PUD_HUGE), REG2; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pt %xcc, 700f; \ + sethi %hi(0x1ffc0000), REG2; \ + sllx REG2, 1, REG2; \ + brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ +700: +#else +#define USER_PGTABLE_CHECK_PUD_HUGE (VADDR ,REG1 ,REG2 ,FAIL_LABEL ,PTE_LABEL ) \ + brz,pn REG1, FAIL_LABEL; \ + nop; +#endif +/* PMD has been loaded into REG1, interpret the value, seeing * if it is a HUGE PMD or a normal one. If it is not valid * then jump to FAIL_LABEL. If it is a HUGE PMD, and it
@@ -242,6 +271,7 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; srlx REG2, 64 - PAGE_SHIFT, REG2; \ andn REG2, 0x7, REG2; \ ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ +USER_PGTABLE_CHECK_PUD_HUGE (VADDR ,REG1 ,REG2 ,FAIL_LABEL ,800f ) \ brz,pn REG1, FAIL_LABEL; \ sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \
arch/sparc/kernel/tsb.S +1 -1
@@ -117,7 +117,7 @@ tsb_miss_page_table_walk_sun4v_fastpath: /* Valid PTE is now in %g5. */ #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - sethi %uhi(_PAGE_PMD_HUGE), %g7 + sethi %uhi(_PAGE_PMD_HUGE | _PAGE_PUD_HUGE), %g7 sllx %g7, 32, %g7 andcc %g5, %g7, %g0
arch/sparc/mm/hugetlbpage.c +53 -21
141 lines not on this page · raw message
arch/sparc/mm/init_64.c +34 -7
61 lines not on this page · raw message
Signature · 1 line
2.9.2
PATCH 2/4 sparc64: Support huge PUD case in get_user_pages
In reply to Nitin Guptaget_user_pages() is used to do direct IO. It already handles the case where the address range is backed by PMD huge pages. This patch now adds the case where the range could be backed by PUD huge pages.
--- arch/sparc/include/asm/pgtable_64.h | 15 ++++++++++-- arch/sparc/mm/gup.c | 47 ++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 3 deletions(-)
arch/sparc/include/asm/pgtable_64.h +13 -2
@@ -692,6 +692,8 @@ static inline unsigned long pmd_write(pmd_t pmd)return pte_write (pte); } +#define pud_write (pud) pte_write(__pte(pud_val(pud))) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline unsigned long pmd_dirty (pmd_t pmd) {
@@ -828,9 +830,18 @@ static inline unsigned long __pmd_page(pmd_t pmd)return ((unsigned long )__va (pfn <<PAGE_SHIFT )); } + +static inline unsigned long pud_page_vaddr (pud_t pud) +{ +pte_t pte =__pte (pud_val (pud)); +unsigned long pfn; + + pfn =pte_pfn (pte); + +return ((unsigned long )__va (pfn <<PAGE_SHIFT )); +} +#define pmd_page (pmd) virt_to_page((void *)__pmd_page(pmd)) -#define pud_page_vaddr (pud) \ - ((unsigned long) __va(pud_val(pud)))#define pud_page (pud) virt_to_page((void *)pud_page_vaddr(pud))#define pmd_clear (pmdp) (pmd_val(*(pmdp)) = 0UL)#define pud_present (pud) (pud_val(pud) != 0U)
arch/sparc/mm/gup.c +46 -1
59 lines not on this page · raw message
Signature · 1 line
2.9.2
PATCH 3/4 sparc64: Fix gup_huge_pmd
In reply to Nitin GuptaThe function assumes that each PMD points to head of a huge page. This is not correct as a PMD can point to start of any 8M region with a, say 256M, hugepage. The fix ensures that it points to the correct head of any PMD huge page.
--- arch/sparc/mm/gup.c | 2 ++ 1 file changed, 2 insertions(+)
arch/sparc/mm/gup.c +2 -0
@@ -80,6 +80,8 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, refs =0 ; head =pmd_page (pmd); page = head + ((addr & ~PMD_MASK ) >>PAGE_SHIFT ); +if (PageTail (head)) + head =compound_head (head); do { VM_BUG_ON(compound_head(page) != head); pages[*nr] = page;
Signature · 1 line
2.9.2
PATCH 4/4 sparc64: Cleanup hugepage table walk functions
In reply to Nitin GuptaFlatten out nested code structure in huge_pte_offset() and huge_pte_alloc().
--- arch/sparc/mm/hugetlbpage.c | 54 +++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 34 deletions(-)
arch/sparc/mm/hugetlbpage.c +20 -34
@@ -266,27 +266,19 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,pgd_t *pgd;pud_t *pud;pmd_t *pmd; -pte_t *pte =NULL ; pgd =pgd_offset (mm, addr); pud =pud_alloc (mm, pgd, addr);if (!pud)return NULL ; -if (sz >=PUD_SIZE ) - pte = (pte_t *)pud; -else { - pmd =pmd_alloc (mm, pud, addr); -if (!pmd) -return NULL ; - -if (sz >=PMD_SIZE ) - pte = (pte_t *)pmd; -else - pte =pte_alloc_map (mm, pmd, addr); - } - -return pte; +return (pte_t *)pud; + pmd =pmd_alloc (mm, pud, addr); +if (!pmd) +return NULL ; +if (sz >=PMD_SIZE ) +return (pte_t *)pmd; +return pte_alloc_map (mm, pmd, addr); }pte_t *huge_pte_offset (struct mm_struct *mm,unsigned long addr)
@@ -294,27 +286,21 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)pgd_t *pgd;pud_t *pud;pmd_t *pmd; -pte_t *pte =NULL ; pgd =pgd_offset (mm, addr); -if (!pgd_none (*pgd)) { - pud =pud_offset (pgd, addr); -if (!pud_none (*pud)) { -if (is_hugetlb_pud (*pud)) - pte = (pte_t *)pud; -else { - pmd =pmd_offset (pud, addr); -if (!pmd_none (*pmd)) { -if (is_hugetlb_pmd (*pmd)) - pte = (pte_t *)pmd; -else - pte =pte_offset_map (pmd, addr); - } - } - } - } - -return pte; +if (pgd_none (*pgd)) +return NULL ; + pud =pud_offset (pgd, addr); +if (pud_none (*pud)) +return NULL ; +if (is_hugetlb_pud (*pud)) +return (pte_t *)pud; + pmd =pmd_offset (pud, addr); +if (pmd_none (*pmd)) +return NULL ; +if (is_hugetlb_pmd (*pmd)) +return (pte_t *)pmd; +return pte_offset_map (pmd, addr); } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
Signature · 1 line
2.9.2
David Miller
maintainerFrom: Nitin Gupta <[email protected]>
Date: Tue, 20 Jun 2017 18:05:53 -0700
Quoted 8 lines
Patch 1/4: Core changes needed to add 16G hugepage support: To map a
single 16G hugepage, two PUD entries are used. Each PUD entry maps 8G portion of a 16G page. This page table encoding scheme is same as that used for hugepages at PMD level (8M, 256M and 2G pages) where each PMD entry points successively to 8M regions within a page. No page table entries below the PUD level are allocated for 16G hugepage since those are not required.
Quoted 5 lines
TSB entries for a 16G page are created at every 4M boundary since the
HUGE_TSBis used for these pages which is configured with page size of 4M. When walking page tables (on a TSB miss), bits [32:22] are transferred from vaddr to PUD to resolve addresses at 4M boundary. The resolved address mapping is then stored inHUGE_TSB.
Ok this is a new feature.
Quoted 10 lines
Patch 2/4:
get_user_pages()etc. are used for direct IO. These
functions were not aware of hugepages at the PUD level and would try to continue walking page tables beyond the PUD level. Since 16G hugepages have page tables allocated till PUD level only, these accesses would result in invalid access. This patch adds the case for PUD huge pages to these functions.Patch 3/4: Fixes a bug in
gup_huge_pmd()which caused wrong page
within a hugepage to be considered as the head page.gup_huge_pud()already has this fix.
These are bug fixes that appear to be relevant for mainline right?
Please submit these two separately.
Quoted 4 lines
Patch 4/4: Patch 1 added the case of PUD entry being huge in page
table walk and alloc functions. This new case further increased nesting in these functions and made them harder to follow. This patch flattens these functions for better readability.
Ok, this is a cleanup and can go along with the new feature patch #1.
So submit the two bug fixes against 'sparc' GIT, then when I next merge 'sparc' into 'sparc-next', you can submit #1 and #4.
THanks.
Quoted 26 lines · David Miller
From: Nitin Gupta <[email protected]>
Date: Tue, 20 Jun 2017 18:05:53 -0700Quoted 8 lines
Patch 1/4: Core changes needed to add 16G hugepage support: To map a
single 16G hugepage, two PUD entries are used. Each PUD entry maps 8G portion of a 16G page. This page table encoding scheme is same as that used for hugepages at PMD level (8M, 256M and 2G pages) where each PMD entry points successively to 8M regions within a page. No page table entries below the PUD level are allocated for 16G hugepage since those are not required.Quoted 5 lines
TSB entries for a 16G page are created at every 4M boundary since the
HUGE_TSBis used for these pages which is configured with page size of 4M. When walking page tables (on a TSB miss), bits [32:22] are transferred from vaddr to PUD to resolve addresses at 4M boundary. The resolved address mapping is then stored inHUGE_TSB.Ok this is a new feature.
Quoted 7 lines
Patch 2/4:
get_user_pages()etc. are used for direct IO. These
functions were not aware of hugepages at the PUD level and would try to continue walking page tables beyond the PUD level. Since 16G hugepages have page tables allocated till PUD level only, these accesses would result in invalid access. This patch adds the case for PUD huge pages to these functions.
Quoted 7 lines
Quoted 3 lines
Patch 3/4: Fixes a bug in
gup_huge_pmd()which caused wrong page
within a hugepage to be considered as the head page.gup_huge_pud()already has this fix.These are bug fixes that appear to be relevant for mainline right?
Please submit these two separately.
Yes, these two bugfixes are relevant for mainline. I think Patch 2 should go along with feature patches since without Patch 1, which adds 16G hugepage support, Patch 2 does not make any sense. Patch 2 is more like an extension of Patch 1, and adds direct IO support over 16G pages.
Quoted 11 lines
Quoted 5 lines
Patch 4/4: Patch 1 added the case of PUD entry being huge in page
table walk and alloc functions. This new case further increased nesting in these functions and made them harder to follow. This patch flattens these functions for better readability.Ok, this is a cleanup and can go along with the new feature patch #1.
So submit the two bug fixes against 'sparc' GIT, then when I next merge 'sparc' into 'sparc-next', you can submit #1 and #4.
Ok, please let me know if you agree with above and I should send only Patch 3 for now. When this fix makes it into sparc-next, I will send feature patches against the sparc tree.
Thanks,
Nitin
David Miller
maintainerFrom: Nitin Gupta <[email protected]>
Date: Tue, 20 Jun 2017 19:15:55 -0700
Quoted 33 lines · David Miller
Quoted 32 lines
From: Nitin Gupta <[email protected]>
Date: Tue, 20 Jun 2017 18:05:53 -0700Quoted 8 lines
Patch 1/4: Core changes needed to add 16G hugepage support: To map a
single 16G hugepage, two PUD entries are used. Each PUD entry maps 8G portion of a 16G page. This page table encoding scheme is same as that used for hugepages at PMD level (8M, 256M and 2G pages) where each PMD entry points successively to 8M regions within a page. No page table entries below the PUD level are allocated for 16G hugepage since those are not required.Quoted 5 lines
TSB entries for a 16G page are created at every 4M boundary since the
HUGE_TSBis used for these pages which is configured with page size of 4M. When walking page tables (on a TSB miss), bits [32:22] are transferred from vaddr to PUD to resolve addresses at 4M boundary. The resolved address mapping is then stored inHUGE_TSB.Ok this is a new feature.
Quoted 10 lines
Patch 2/4:
get_user_pages()etc. are used for direct IO. These
functions were not aware of hugepages at the PUD level and would try to continue walking page tables beyond the PUD level. Since 16G hugepages have page tables allocated till PUD level only, these accesses would result in invalid access. This patch adds the case for PUD huge pages to these functions.Patch 3/4: Fixes a bug in
gup_huge_pmd()which caused wrong page
within a hugepage to be considered as the head page.gup_huge_pud()already has this fix.These are bug fixes that appear to be relevant for mainline right?
Quoted 7 lines
Quoted 1 line
Please submit these two separately.
Yes, these two bugfixes are relevant for mainline. I think Patch 2 should go along with feature patches since without Patch 1, which adds 16G hugepage support, Patch 2 does not make any sense. Patch 2 is more like an extension of Patch 1, and adds direct IO support over 16G pages.
Ok, so PUD huge pages don't happen without patch #1 and therefore patch #2 doesn't apply without 16GB page support.
But this makes me think that you must combine patch #1 and #2, because #1 adds a regression in that the page table walked in get_user_pages() is broken by it.
Or, reverse them, do patch #2 to teach get_user_pages() about pud huge pages. Then you can safely add #1.
Quoted 7 lines
Quoted 7 lines
Quoted 4 lines
Patch 4/4: Patch 1 added the case of PUD entry being huge in page
table walk and alloc functions. This new case further increased nesting in these functions and made them harder to follow. This patch flattens these functions for better readability.Ok, this is a cleanup and can go along with the new feature patch #1.
Quoted 7 lines
Quoted 3 lines
So submit the two bug fixes against 'sparc' GIT, then when I next merge 'sparc' into 'sparc-next', you can submit #1 and #4.
Ok, please let me know if you agree with above and I should send only Patch 3 for now. When this fix makes it into sparc-next, I will send feature patches against the sparc tree.
I do agree. #3 for sparc and #1+#2 & #3 for sparc-next.