convert s390 to generic mmu_gather
Subsystems (5)
cover convert s390 to generic mmu_gather
Hi Peter,
as an add-on for the TLB flushing changes two patches to make s390 use the generic mmu_gather code as well. I let it run for a few hours with several TLB stress tests, no fallout so far. It certainly can use more testing but it looks ok to me.
Martin Schwidefsky (2):
asm-generic/tlb: introduce HAVE_MMU_GATHER_NO_GATHER s390/tlb: convert to generic mmu_gather
arch/Kconfig | 3 + arch/s390/Kconfig | 3 + arch/s390/include/asm/tlb.h | 130 ++++++++++++++------------------------------ arch/s390/mm/pgalloc.c | 63 +-------------------- include/asm-generic/tlb.h | 9 ++- mm/mmu_gather.c | 114 ++++++++++++++++++++++---------------- 6 files changed, 121 insertions(+), 201 deletions(-)
Signature · 1 line
2.16.4
PATCH 1/2 asm-generic/tlb: introduce HAVE_MMU_GATHER_NO_GATHER
Add the Kconfig option HAVE_MMU_GATHER_NO_GATHER to the generic mmu_gather code. If the option is set the mmu_gather will not track individual pages for delayed page free anymore. A platform that enables the option needs to provide its own implementation of the __tlb_remove_page_size function to free pages.
--- arch/Kconfig | 3 ++ include/asm-generic/tlb.h | 9 +++- mm/mmu_gather.c | 114 +++++++++++++++++++++++++++------------------- 3 files changed, 77 insertions(+), 49 deletions(-)
arch/Kconfig +3 -0
@@ -368,6 +368,9 @@ config HAVE_RCU_TABLE_INVALIDATE config HAVE_MMU_GATHER_PAGE_SIZE bool +config HAVE_MMU_GATHER_NO_GATHER + bool + config ARCH_HAVE_NMI_SAFE_CMPXCHG bool
include/asm-generic/tlb.h +7 -2
@@ -179,6 +179,7 @@ extern void tlb_remove_table(struct mmu_gather *tlb, void *table);#endif +#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER /* * If we can't allocate a page to make a big batch of page pointers * to work on, then just handle a few from the on-stack structure.
@@ -203,6 +204,10 @@ struct mmu_gather_batch {*/ #define MAX_GATHER_BATCH_COUNT (10000UL/MAX_GATHER_BATCH) +extern bool __tlb_remove_page_size (struct mmu_gather *tlb,struct page *page, +int page_size); +#endif +/* * struct mmu_gather is an opaque type used by the mm code for passing around * any data needed by arch specific code for tlb_remove_page.
@@ -249,6 +254,7 @@ struct mmu_gather {unsigned int batch_count; +#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER struct mmu_gather_batch *active;struct mmu_gather_batch local;struct page *__pages[MMU_GATHER_BUNDLE ];
@@ -256,6 +262,7 @@ struct mmu_gather {#ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE unsigned int page_size;#endif +#endif }; void arch_tlb_gather_mmu(struct mmu_gather *tlb,
@@ -264,8 +271,6 @@ void tlb_flush_mmu(struct mmu_gather *tlb);void arch_tlb_finish_mmu (struct mmu_gather *tlb,unsigned long start,unsigned long end,bool force);void tlb_flush_mmu_free (struct mmu_gather *tlb); -extern bool __tlb_remove_page_size (struct mmu_gather *tlb,struct page *page, -int page_size); static inline void __tlb_adjust_range(struct mmu_gather *tlb, unsigned long address,
mm/mmu_gather.c +67 -47
160 lines not on this page · raw message
Signature · 1 line
2.16.4
PATCH 2/2 s390/tlb: convert to generic mmu_gather
In reply to Martin Schwidefsky--- arch/s390/Kconfig | 3 + arch/s390/include/asm/tlb.h | 130 ++++++++++++++------------------------------ arch/s390/mm/pgalloc.c | 63 +-------------------- 3 files changed, 44 insertions(+), 152 deletions(-)
arch/s390/Kconfig +3 -0
@@ -157,10 +157,13 @@ config S390 select HAVE_MEMBLOCK select HAVE_MEMBLOCK_NODE_MAP select HAVE_MEMBLOCK_PHYS_MAP + select HAVE_MMU_GATHER_NO_GATHER select HAVE_MOD_ARCH_SPECIFIC select HAVE_NOP_MCOUNT select HAVE_OPROFILE select HAVE_PERF_EVENTS + select HAVE_RCU_TABLE_FREE + select HAVE_RCU_TABLE_INVALIDATE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_SYSCALL_TRACEPOINTS
arch/s390/include/asm/tlb.h +40 -90
178 lines not on this page · raw message
arch/s390/mm/pgalloc.c +1 -62
75 lines not on this page · raw message
Signature · 1 line
2.16.4
Peter Zijlstra
maintainerRe: [PATCH 2/2] s390/tlb: convert to generic mmu_gather
Quoted 11 lines · Martin Schwidefsky
arch/s390/include/asm/tlb.h +5
-0+#define pte_free_tlb pte_free_tlb +#define pmd_free_tlb pmd_free_tlb +#define p4d_free_tlb p4d_free_tlb +#define pud_free_tlb pud_free_tlb@@ -121,9 +62,18 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb,* page table from the tlb. */ static inline void pte_free_tlb (struct mmu_gather *tlb,pgtable_t pte, +unsigned long address) {
arch/s390/include/asm/tlb.h static inline void tlb_remove_page_size(struct mmu_gather *tlb, · line 76
Quoted 10 lines
arch/s390/include/asm/tlb.h +10
-0+__tlb_adjust_range (tlb, address,PAGE_SIZE ); + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_ptes =1 ; +/* +* page_table_free_rcu takes care of the allocation bit masks +* of the 2K table fragments in the 4K page table page, +* then calls tlb_remove_table. +*/ +page_table_free_rcu (tlb, (unsigned long *) pte, address);
(whitespace damage, fixed)
Also, could you perhaps explain the need for that page_table_alloc/page_table_free code? That is, I get the comment about using 2K page-table fragments out of 4k physical page, but why this custom allocator instead of kmem_cache? It feels like there's a little extra complication, but it's not immediately obvious what.
Review static inline void tlb_remove_page_size(struct mmu_gather *tlb, · line 77
Quoted 1 line
(quoted diff) +0
-0}
We _could_ use __pte_free_tlb() here I suppose, but...
Quoted 24 lines
arch/s390/include/asm/tlb.h +8
-0/* @@ -139,6 +89,10 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,if (tlb->mm->context.asce_limit <= _REGION3_SIZE)return ;pgtable_pmd_page_dtor (virt_to_page (pmd)); +__tlb_adjust_range (tlb, address,PAGE_SIZE ); + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_puds =1 ;tlb_remove_table (tlb, pmd); }@@ -154,6 +108,10 @@ static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, {if (tlb->mm->context.asce_limit <= _REGION1_SIZE)return ; +__tlb_adjust_range (tlb, address,PAGE_SIZE ); + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_p4ds =1 ;tlb_remove_table (tlb, p4d); }
arch/s390/include/asm/tlb.h static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, · line 134
Quoted 8 lines
arch/s390/include/asm/tlb.h +3
-0@@ -169,19 +127,11 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, {if (tlb->mm->context.asce_limit <= _REGION2_SIZE)return ; + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_puds =1 ;tlb_remove_table (tlb, pud); }
It's that ASCE limit that makes it impossible to use the generic helpers, right?
Re: [PATCH 2/2] s390/tlb: convert to generic mmu_gather
Quoted 25 lines · On Wed, 19 Sep 2018 14:38:49 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 11 lines · Martin Schwidefsky
arch/s390/include/asm/tlb.h +5
-0+#define pte_free_tlb pte_free_tlb +#define pmd_free_tlb pmd_free_tlb +#define p4d_free_tlb p4d_free_tlb +#define pud_free_tlb pud_free_tlb@@ -121,9 +62,18 @@ static inline void tlb_remove_page_size(struct mmu_gather *tlb,* page table from the tlb. */ static inline void pte_free_tlb (struct mmu_gather *tlb,pgtable_t pte, +unsigned long address) {arch/s390/include/asm/tlb.h static inline void tlb_remove_page_size(struct mmu_gather *tlb, · line 76
Quoted 10 lines
arch/s390/include/asm/tlb.h +10
-0+__tlb_adjust_range (tlb, address,PAGE_SIZE ); + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_ptes =1 ; +/* +* page_table_free_rcu takes care of the allocation bit masks +* of the 2K table fragments in the 4K page table page, +* then calls tlb_remove_table. +*/ +page_table_free_rcu (tlb, (unsigned long *) pte, address);(whitespace damage, fixed)
Quoted 5 lines
Also, could you perhaps explain the need for that page_table_alloc/page_table_free code? That is, I get the comment about using 2K page-table fragments out of 4k physical page, but why this custom allocator instead of
kmem_cache? It feels like there's a little extra complication, but it's not immediately obvious what.
The kmem_cache code uses the fields of struct page for its tracking. pgtable_page_ctor uses the same fields, e.g. for the ptl. Last time I tried to convert the page_table_alloc/page_table_free to kmem_cache it just crashed. Plus the split of 4K pages into 2 2K fragments is done on a per mm basis, that should help a little bit with fragmentation.
Quoted 27 lines
Review static inline void tlb_remove_page_size(struct mmu_gather *tlb, · line 77
Quoted 1 line
(quoted diff) +0
-0}We _could_ use
__pte_free_tlb()here I suppose, but...Quoted 23 lines
arch/s390/include/asm/tlb.h +8
-0/* @@ -139,6 +89,10 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,if (tlb->mm->context.asce_limit <= _REGION3_SIZE)return ;pgtable_pmd_page_dtor (virt_to_page (pmd)); +__tlb_adjust_range (tlb, address,PAGE_SIZE ); + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_puds =1 ;tlb_remove_table (tlb, pmd); }@@ -154,6 +108,10 @@ static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, {if (tlb->mm->context.asce_limit <= _REGION1_SIZE)return ; +__tlb_adjust_range (tlb, address,PAGE_SIZE ); + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_p4ds =1 ;tlb_remove_table (tlb, p4d); }
Quoted 12 lines
arch/s390/include/asm/tlb.h static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, · line 134
Quoted 9 lines
arch/s390/include/asm/tlb.h +3
-0@@ -169,19 +127,11 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, {if (tlb->mm->context.asce_limit <= _REGION2_SIZE)return ; + tlb->mm->context.flush_mm =1 ; + tlb->freed_tables =1 ; + tlb->cleared_puds =1 ;tlb_remove_table (tlb, pud); }It's that ASCE limit that makes it impossible to use the generic helpers, right?
There are two problems, one of them is related to the ASCE limit:
1) s390 supports 4 different page table layouts. 2-levels (2^31 bytes) for 31-bit compat, 3-levels (2^42 bytes) as the default for 64-bit, 4-levels (2^53) if 4 tera-bytes are not enough and 5-levels (2^64) for the bragging rights.
The pxd_free_tlb() turn into nops if the number of page table levels require it.
2) The mm->context.flush_mm indication.
That goes back to this beauty in the architecture:
* "A valid table entry must not be changed while it is attached
* to any CPU and may be used for translation by that CPU except to
* (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY,
* or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page
* table entry, or (3) make a change by means of a COMPARE AND SWAP
* AND PURGE instruction that purges the TLB."
If one CPU is doing a mmu_gather page table operation on the only active thread in the system the individual page table updates are done in a lazy fashion with simple stores. If a second CPU picks up another thread for execution, the attach_count is increased and the page table updates are done with IPTE/IDTE from now on. But there might by TLBs of around that are not flushed yet. We may *not* let the second CPU see these TLBs, otherwise the CPU may start an instruction, then loose the TLB without being able to recreate it. Due to that the CPU can end up with a half finished instruction it can not roll back nor complete, ending in a check-stop. The simplest example is MVC with a length of e.g. 256 bytes. The instruction has to complete with all 256 bytes moved, or no bytes may have at all.
That is where the mm->context.flush_mm indication comes into play, if the second CPU finds the bit set at the time it attaches a thread, it will to an IDTE for flush all TLBs for the mm.
Signature · 4 lines
blue skies, Martin. "Reality continues to ruin my life." - Calvin.
Peter Zijlstra
maintainerRe: [PATCH 2/2] s390/tlb: convert to generic mmu_gather
Quoted 15 lines · Martin Schwidefsky
Quoted 11 lines · On Wed, 19 Sep 2018 14:38:49 +0200 Peter Zijlstra <[email protected]> wrote:
Quoted 1 line · Martin Schwidefsky
+
page_table_free_rcu(tlb, (unsigned long *) pte, address);(whitespace damage, fixed)
Also, could you perhaps explain the need for that page_table_alloc/page_table_free code? That is, I get the comment about using 2K page-table fragments out of 4k physical page, but why this custom allocator instead of
kmem_cache? It feels like there's a little extra complication, but it's not immediately obvious what.
Quoted 5 lines
The
kmem_cachecode uses the fields of struct page for its tracking.pgtable_page_ctoruses the same fields, e.g. for the ptl. Last time I tried to convert the page_table_alloc/page_table_free tokmem_cacheit just crashed. Plus the split of 4K pages into 2 2K fragments is done on a per mm basis, that should help a little bit with fragmentation.
Fair enough, thanks for the information.
Quoted 9 lines
Quoted 2 lines
It's that ASCE limit that makes it impossible to use the generic helpers, right?
There are two problems, one of them is related to the ASCE limit:
1) s390 supports 4 different page table layouts. 2-levels (2^31 bytes) for 31-bit compat, 3-levels (2^42 bytes) as the default for 64-bit, 4-levels (2^53) if 4 tera-bytes are not enough and 5-levels (2^64) for the bragging rights.
Thepxd_free_tlb()turn into nops if the number of page table levels require it.
Shiny, I think we (x86) have to choose at boot time which paging mode we want and have to stick to it.
Quoted 10 lines
2) The
mm->context.flush_mmindication.
That goes back to this beauty in the architecture:* "A valid table entry must not be changed while it is attached
* to any CPU and may be used for translation by that CPU except to
* (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY,
* or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page
* table entry, or (3) make a change by means of a COMPARE AND SWAP
* AND PURGE instruction that purges the TLB."
Quoted 14 lines
If one CPU is doing a
mmu_gatherpage table operation on the only active thread in the system the individual page table updates are done in a lazy fashion with simple stores. If a second CPU picks up another thread for execution, theattach_countis increased and the page table updates are done with IPTE/IDTE from now on. But there might by TLBs of around that are not flushed yet. We may *not* let the second CPU see these TLBs, otherwise the CPU may start an instruction, then loose the TLB without being able to recreate it. Due to that the CPU can end up with a half finished instruction it can not roll back nor complete, ending in a check-stop. The simplest example is MVC with a length of e.g. 256 bytes. The instruction has to complete with all 256 bytes moved, or no bytes may have at all.
That is where themm->context.flush_mmindication comes into play, if the second CPU finds the bit set at the time it attaches a thread, it will to an IDTE for flush all TLBs for the mm.
Oh man.. what fun. Still, this bit could easily be set in the __*_free_tlb() functions afaict. Still 1) above is enough.
Thanks!