Implement page table accounting for powerpc

PATCH In review Balbir Singh 2 participants 5 messages May 2, 2017 – Jun 8, 2017

Subsystems (2)

Memory management PowerPC

lore

Share & tools

lore

Balbir Singh

author

2017-05-02 05:17 UTC #0 lore raw

v3 cover Implement page table accounting for powerpc

(3e79ec7 arch: x86: charge page tables to kmemcg) added support for page table accounting). This patch is the second iteration to add support, in the earlier iteration only book3s 64 bit was supported. This iteration adds support for booke/3s/32 and 64 bit.

There is some ugliness in this patchset, pgalloc.h is included from book3s_64_mmu_radix.c to reuse the pte/pmd/pud and pgd management routines. We use #ifdef MODULE to provide a version that provides full accounting. The alternatives are discussed in patch 1 below

Changelog v3:
- Fixed a build failure with 32 bit powerpc
- Optimizations for pte_alloc_one()
Changelog v2:
- Added support for 32 bit and booke
- Added hugepte alloc accounting

Balbir Singh (3):
powerpc/mm/book(e)(3s)/64: Add page table accounting powerpc/mm/book(e)(3s)/32: Add page table accounting powerpc/mm/hugetlb: Add support for page accounting

 arch/powerpc/include/asm/book3s/32/pgalloc.h |  3 ++-
 arch/powerpc/include/asm/book3s/64/pgalloc.h | 16 ++++++++++------
 arch/powerpc/include/asm/nohash/32/pgalloc.h |  3 ++-
 arch/powerpc/include/asm/nohash/64/pgalloc.h | 12 ++++++++----
 arch/powerpc/include/asm/pgalloc.h           | 14 ++++++++++++++
 arch/powerpc/mm/hugetlbpage.c                |  2 +-
 arch/powerpc/mm/pgtable_32.c                 |  2 +-
 arch/powerpc/mm/pgtable_64.c                 | 20 ++++++++++++++------
 8 files changed, 52 insertions(+), 20 deletions(-)

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Balbir Singh

author

2017-05-02 05:17 UTC #1 lore raw

PATCH v3 1/3 powerpc/mm/book(e)(3s)/64: Add page table accounting

Introduce a helper pgtable_gfp_flags() which just returns the current gfp flags and adds __GFP_ACCOUNT to account for page table allocation. The generic helper is added to include/asm/pgalloc.h and has two variants - WARNING ugly bits ahead

1. If the header is included from a module, no check for mm == &init_mm is done, since init_mm is not exported
2. For kernel includes, the check is done and required see (3e79ec7 arch: x86: charge page tables to kmemcg)

The fundamental assumption is that no module should be doing pgd/pud/pmd and pte alloc's on behalf of init_mm directly.

NOTE: This adds an overhead to pmd/pud/pgd allocations similar to x86. The other alternative was to implement pmd_alloc_kernel/pud_alloc_kernel and pgd_alloc_kernel with their offset variants.

For 4k page size, pte_alloc_one no longer calls
pte_alloc_one_kernel.

Signed-off-by Balbir Singh <[email protected]>

---
 arch/powerpc/include/asm/book3s/32/pgalloc.h |  3 ++-
 arch/powerpc/include/asm/book3s/64/pgalloc.h | 16 ++++++++++------
 arch/powerpc/include/asm/nohash/64/pgalloc.h | 11 +++++++----
 arch/powerpc/include/asm/pgalloc.h           | 14 ++++++++++++++
 arch/powerpc/mm/pgtable_64.c                 | 20 ++++++++++++++------
 5 files changed, 47 insertions(+), 17 deletions(-)

arch/powerpc/include/asm/book3s/32/pgalloc.h +2 -1

index d310546..a120e7f 100644
@@ -31,7 +31,8 @@ extern struct kmem_cache *pgtable_cache[];
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
+			pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)

arch/powerpc/include/asm/book3s/64/pgalloc.h +10 -6

index cd5e7aa..20b1485 100644
@@ -53,10 +53,11 @@ extern void __tlb_remove_table(void *_table);
 static inline pgd_t *radix__pgd_alloc(struct mm_struct *mm)
 {
 #ifdef CONFIG_PPC_64K_PAGES
-	return (pgd_t *)__get_free_page(PGALLOC_GFP);
+	return (pgd_t *)__get_free_page(pgtable_gfp_flags(mm, PGALLOC_GFP));
 #else
 	struct page *page;
-	page = alloc_pages(PGALLOC_GFP | __GFP_REPEAT, 4);
+	page = alloc_pages(pgtable_gfp_flags(mm, PGALLOC_GFP | __GFP_REPEAT),
+				4);
 	if (!page)
 		return NULL;
 	return (pgd_t *) page_address(page);

@@ -76,7 +77,8 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
 	if (radix_enabled())
 		return radix__pgd_alloc(mm);
-	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
+		pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)

@@ -93,7 +95,8 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
 
 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
+		pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pud_free(struct mm_struct *mm, pud_t *pud)

@@ -119,7 +122,8 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
+		pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)

@@ -168,7 +172,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
 	struct page *page;
 	pte_t *pte;
 
-	pte = pte_alloc_one_kernel(mm, address);
+	pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT);
 	if (!pte)
 		return NULL;
 	page = virt_to_page(pte);

arch/powerpc/include/asm/nohash/64/pgalloc.h +7 -4

index 897d2e1..9721c78 100644
@@ -43,7 +43,8 @@ extern struct kmem_cache *pgtable_cache[];
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
+			pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)

@@ -57,7 +58,8 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 
 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
+			pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pud_free(struct mm_struct *mm, pud_t *pud)

@@ -96,7 +98,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
 	struct page *page;
 	pte_t *pte;
 
-	pte = pte_alloc_one_kernel(mm, address);
+	pte = (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT);
 	if (!pte)
 		return NULL;
 	page = virt_to_page(pte);

@@ -189,7 +191,8 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
 
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
+			pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)

arch/powerpc/include/asm/pgalloc.h +14 -0

index 0413457..d795c5d 100644
@@ -3,6 +3,20 @@
 
 #include <linux/mm.h>
 
+#ifndef MODULE
+static inline gfp_t pgtable_gfp_flags(struct mm_struct *mm, gfp_t gfp)
+{
+	if (unlikely(mm == &init_mm))
+		return gfp;
+	return gfp | __GFP_ACCOUNT;
+}
+#else /* !MODULE */
+static inline gfp_t pgtable_gfp_flags(struct mm_struct *mm, gfp_t gfp)
+{
+	return gfp | __GFP_ACCOUNT;
+}
+#endif /* MODULE */
+
 #ifdef CONFIG_PPC_BOOK3S
 #include <asm/book3s/pgalloc.h>
 #else

arch/powerpc/mm/pgtable_64.c +14 -6

index db93cf7..8d2d674 100644
@@ -351,12 +351,20 @@ static pte_t *get_from_cache(struct mm_struct *mm)
 static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
 {
 	void *ret = NULL;
-	struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO);
-	if (!page)
-		return NULL;
-	if (!kernel && !pgtable_page_ctor(page)) {
-		__free_page(page);
-		return NULL;
+	struct page *page;
+
+	if (!kernel) {
+		page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
+		if (!page)
+			return NULL;
+		if (!pgtable_page_ctor(page)) {
+			__free_page(page);
+			return NULL;
+		}
+	} else {
+		page = alloc_page(PGALLOC_GFP);
+		if (!page)
+			return NULL;
 	}
 
 	ret = page_address(page);

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Michael Ellerman

2017-06-08 04:07 UTC #2 lore raw

Re: [v3,1/3] powerpc/mm/book(e)(3s)/64: Add page table accounting

Quoted 21 lines · Balbir Singh

Introduce a helper pgtable_gfp_flags() which just returns the current gfp flags and adds __GFP_ACCOUNT to account for page table allocation. The generic helper is added to include/asm/pgalloc.h and has two variants - WARNING ugly bits ahead
1. If the header is included from a module, no check for mm == &init_mm is done, since init_mm is not exported
2. For kernel includes, the check is done and required see (3e79ec7 arch: x86: charge page tables to kmemcg)
The fundamental assumption is that no module should be doing pgd/pud/pmd and pte alloc's on behalf of init_mm directly.
NOTE: This adds an overhead to pmd/pud/pgd allocations similar to x86. The other alternative was to implement pmd_alloc_kernel/pud_alloc_kernel and pgd_alloc_kernel with their offset variants.

Quoted 4 lines

For 4k page size, pte_alloc_one no longer calls pte_alloc_one_kernel.
Signed-off-by Balbir Singh <[email protected]>

Series applied to powerpc next, thanks.

https://git.kernel.org/powerpc/c/de3b87611dd1f3c00f4e42fe298457

cheers

Balbir Singh

author

2017-05-02 05:17 UTC #3 lore raw

PATCH v3 2/3 powerpc/mm/book(e)(3s)/32: Add page table accounting

In reply to Balbir Singh

Add support in pte_alloc_one() and pgd_alloc() by
passing __GFP_ACCOUNT in the flags

Signed-off-by Balbir Singh <[email protected]>

---
 arch/powerpc/include/asm/nohash/32/pgalloc.h | 3 ++-
 arch/powerpc/mm/pgtable_32.c                 | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

arch/powerpc/include/asm/nohash/32/pgalloc.h +2 -1

index 6331392..cc369a7 100644
@@ -31,7 +31,8 @@ extern struct kmem_cache *pgtable_cache[];
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
+	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
+			pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)

arch/powerpc/mm/pgtable_32.c +1 -1

index a65c0b4..dc1e0c2 100644
@@ -60,7 +60,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
 {
 	struct page *ptepage;
 
-	gfp_t flags = GFP_KERNEL | __GFP_ZERO;
+	gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_ACCOUNT;
 
 	ptepage = alloc_pages(flags, 0);
 	if (!ptepage)

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Balbir Singh

author

2017-05-02 05:17 UTC #4 lore raw

PATCH v3 3/3 powerpc/mm/hugetlb: Add support for page accounting

In reply to Balbir Singh

Add __GFP_ACCOUNT to __hugepte_alloc()

Signed-off-by Balbir Singh <[email protected]>

---
 arch/powerpc/mm/hugetlbpage.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

arch/powerpc/mm/hugetlbpage.c +1 -1

index a4f33de..94e56b1 100644
@@ -77,7 +77,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 		num_hugepd = 1;
 	}
 
-	new = kmem_cache_zalloc(cachep, GFP_KERNEL);
+	new = kmem_cache_zalloc(cachep, pgtable_gfp_flags(mm, GFP_KERNEL));
 
 	BUG_ON(pshift > HUGEPD_SHIFT_MASK);
 	BUG_ON((unsigned long)new & HUGEPD_SHIFT_MASK);

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>