folded page table walkers
Subsystems (3)
PATCH 1/2 folded page table walkers
(Note these are all obviously just RFCs, until after 2.6.11, at which time I'll send them off to Andrew if they've proven OK).
Anyway, here is the patch to fold the page table walkers nicely for 2 and 3 level implementations... now we are getting some good reasons why people should convert to the -nop?d.h headers :)
We cut 35% off lmbench fork time with 2 level UP i386. This patch gets most of the improvement.
plain +ptwalk +folded +funit-at-a-time for mm/mm/memory.o text size 10935 10679 9799 9519
lmb fork 335 327 223 218
lmb exec 1417 1445 1228 1198 lmb shell 5483 5507 5195 5087
---
linux-2.6-npiggin/include/asm-generic/pgtable-nopmd.h | 9 +++++++++ linux-2.6-npiggin/include/asm-generic/pgtable-nopud.h | 8 ++++++++ linux-2.6-npiggin/include/asm-generic/pgtable.h | 4 ++++ 3 files changed, 21 insertions(+)
include/asm-generic/pgtable.h renamed from include/asm-generic/pgtable.h~vm-folded-walkers +4 -0
@@ -175,6 +175,7 @@ static inline void ptep_mkdirty(pte_t *p* * see for_each_pgd */ +#ifndef for_each_pud#define for_each_pud (pgd, start, end, pud, pud_start, pud_end) \ for ( pud = pud_offset(pgd, start), \ pud_start = start; \
@@ -183,12 +184,14 @@ static inline void ptep_mkdirty(pte_t *p pud <=pud_offset (pgd, end-1 ); \ pud_start = pud_end, \ pud++ ) +#endif /* * for_each_pmd - iterate through pmd entries in a given pud * * see for_each_pgd */ +#ifndef for_each_pmd#define for_each_pmd (pud, start, end, pmd, pmd_start, pmd_end) \ for ( pmd = pmd_offset(pud, start), \ pmd_start = start; \
@@ -197,6 +200,7 @@ static inline void ptep_mkdirty(pte_t *p pmd <=pmd_offset (pud, end-1 ); \ pmd_start = pmd_end, \ pmd++ ) +#endif /* * for_each_pte_map - iterate through pte entries in a given pmd
include/asm-generic/pgtable-nopud.h renamed from include/asm-generic/pgtable-nopud.h~vm-folded-walkers +8 -0
@@ -52,5 +52,13 @@ static inline pud_t * pud_offset(pgd_t *#define pud_free (x) do { } while (0)#define __pud_free_tlb (tlb, x) do { } while (0) +#undef for_each_pud +#define for_each_pud (pgd, start, end, pud, pud_start, pud_end) \ + for ( pud = (pud_t *)pgd, \ + pud_start = start, \ + pud_end = end; \ + pud_start == start; \ + pud_start++ ) +#endif /* __ASSEMBLY__ */ #endif /* _PGTABLE_NOPUD_H */
include/asm-generic/pgtable-nopmd.h renamed from include/asm-generic/pgtable-nopmd.h~vm-folded-walkers +9 -0
@@ -55,6 +55,15 @@ static inline pmd_t * pmd_offset(pud_t *#define pmd_free (x) do { } while (0)#define __pmd_free_tlb (tlb, x) do { } while (0) +#undef for_each_pmd +#define for_each_pmd (pud, start, end, pmd, pmd_start, pmd_end) \ + for ( pmd = (pmd_t *)pud, \ + pmd_start = start, \ + pmd_end = end; \ + pmd_start == start; \ + pmd_start++ ) + +#endif /* __ASSEMBLY__ */ #endif /* _PGTABLE_NOPMD_H */
_
PATCH 2/2 trim unused functions
And lastly, redo this vital optimisation that David had to remove earlier. Saves a few bytes.
---
linux-2.6-npiggin/include/asm-generic/pgtable-nopmd.h | 2 ++ linux-2.6-npiggin/include/asm-generic/pgtable-nopud.h | 2 ++ linux-2.6-npiggin/mm/memory.c | 8 +++++++- 3 files changed, 11 insertions(+), 1 deletion(-)
include/asm-generic/pgtable-nopud.h renamed from include/asm-generic/pgtable-nopud.h~vm-trim-unused +2 -0
@@ -3,6 +3,8 @@#ifndef __ASSEMBLY__ +#define __PAGETABLE_PUD_FOLDED +/* * Having the pud type consist of a pgd gets the size right, and allows * us to conceptually access the pgd entry that this pud is folded into
include/asm-generic/pgtable-nopmd.h renamed from include/asm-generic/pgtable-nopmd.h~vm-trim-unused +2 -0
@@ -5,6 +5,8 @@#include <asm-generic/pgtable-nopud.h> +#define __PAGETABLE_PMD_FOLDED +/* * Having the pmd type consist of a pud gets the size right, and allows * us to conceptually access the pud entry that this pmd is folded into
mm/memory.c renamed from mm/memory.c~vm-trim-unused +7 -1
@@ -2006,6 +2006,8 @@ int handle_mm_fault(struct mm_struct *mm }#ifndef __ARCH_HAS_4LEVEL_HACK + +#ifndef __PAGETABLE_PUD_FOLDED/* * Allocate page upper directory. *
@@ -2037,7 +2039,9 @@ pud_t fastcall *__pud_alloc(struct mm_st out: return pud_offset(pgd, address); } +#endif /* __PAGETABLE_PUD_FOLDED */ +#ifndef __PAGETABLE_PMD_FOLDED/* * Allocate page middle directory. *
@@ -2069,7 +2073,9 @@ pmd_t fastcall *__pmd_alloc(struct mm_st out:return pmd_offset (pud, address); } -#else +#endif /* __PAGETABLE_PMD_FOLDED */ + +#else /* __ARCH_HAS_4LEVEL_HACK */ pmd_t fastcall *__pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) {pmd_t *new;
_