Skip to contentSkip to conversation

mm/khugepaged: Introduce release_pte_folio() to replace release_pte_page()

PATCH Applied Vishal Moola (Oracle) 4 participants 7 messages Jan 14, 2023 – Feb 13, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/2 mm/khugepaged: Introduce release_pte_folio() to replace release_pte_page()

release_pte_page() is converted to be a wrapper for release_pte_folio() to help facilitate the khugepaged conversion to folios.

This replaces 3 calls to compound_head() with 1, and saves 85 bytes of kernel text.

---
 mm/khugepaged.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
mm/khugepaged.c +10 -5
index e1c7c1f357ef..4888e8688401 100644
@@ -492,13 +492,18 @@ void __khugepaged_exit(struct mm_struct *mm)
 	}
 }
 
+static void release_pte_folio(struct folio *folio)
+{
+	node_stat_mod_folio(folio,
+			NR_ISOLATED_ANON + folio_is_file_lru(folio),
+			-folio_nr_pages(folio));
+	folio_unlock(folio);
+	folio_putback_lru(folio);
+}
+
 static void release_pte_page(struct page *page)
 {
-	mod_node_page_state(page_pgdat(page),
-			NR_ISOLATED_ANON + page_is_file_lru(page),
-			-compound_nr(page));
-	unlock_page(page);
-	putback_lru_page(page);
+	release_pte_folio(page_folio(page));
 }
 
 static void release_pte_pages(pte_t *pte, pte_t *_pte,
Signature · 1 line
2.38.1
#1 lore raw

PATCH 2/2 mm/khugepaged: Convert release_pte_pages() to use folios

Converts release_pte_pages() to use folios instead of pages.

---
 mm/khugepaged.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
mm/khugepaged.c +7 -7
index 4888e8688401..27d010431ece 100644
@@ -509,20 +509,20 @@ static void release_pte_page(struct page *page)
 static void release_pte_pages(pte_t *pte, pte_t *_pte,
 		struct list_head *compound_pagelist)
 {
-	struct page *page, *tmp;
+	struct folio *folio, *tmp;
 
 	while (--_pte >= pte) {
 		pte_t pteval = *_pte;
 
-		page = pte_page(pteval);
+		folio = pfn_folio(pte_pfn(pteval));
 		if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
-				!PageCompound(page))
-			release_pte_page(page);
+				!folio_test_large(folio))
+			release_pte_folio(folio);
 	}
 
-	list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
-		list_del(&page->lru);
-		release_pte_page(page);
+	list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
+		list_del(&folio->lru);
+		release_pte_folio(folio);
 	}
 }
 
Signature · 1 line
2.38.1
#2 lore raw

Re: [PATCH 2/2] mm/khugepaged: Convert release_pte_pages() to use folios

Hi,

Quoted 3 lines · Vishal Moola (Oracle)

Converts release_pte_pages() to use folios instead of pages.

This patch has been merged some time ago to linux-next as commit 9bdfeea46f49 ("mm/khugepaged: convert release_pte_pages() to use folios"). It took me a while to bisect this (mainly because I was busy with other things), but I finally found that this change is responsible for the following kernel panic:

Unable to handle kernel paging request at virtual address fffffc0000000008

Mem abort info:
  ESR = 0x0000000096000006
  EC = 0x25: DABT (current EL), IL = 32 bits
  SET = 0, FnV = 0
  EA = 0, S1PTW = 0
  FSC = 0x06: level 2 translation fault
Data abort info:
  ISV = 0, ISS = 0x00000006
  CM = 0, WnR = 0
swapper pgtable: 4k pages, 48-bit VAs, pgdp=0000000021efa000 [fffffc0000000008] pgd=10000000df05a003, p4d=10000000df05a003, pud=10000000df059003, pmd=0000000000000000

Internal error: Oops: 0000000096000006 [#1] PREEMPT SMP
Modules linked in: ip_tables x_tables ipv6
CPU: 7 PID: 61 Comm: khugepaged Not tainted 6.2.0-rc4+ #13307
Hardware name: Samsung TM2E board (DT)
pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
pc : hpage_collapse_scan_pmd+0x12ec/0x1a20
lr : hpage_collapse_scan_pmd+0x14b0/0x1a20

sp : ffff80000be13c20
x29: ffff80000be13c20 x28: 0000000000000001 x27: fffffc0000d3f5c0
x26: fffffc0000d3f600 x25: 00000000000001f9 x24: 0000000000000007
x23: ffff0000296f9dd8 x22: ffff800009e5b490 x21: 0000000000000000
x20: 000000000000000f x19: ffff80000a9d0000 x18: ffff80000af52e58
x17: 0000000000000028 x16: 0000000000009249 x15: ffff80000af971f8
x14: 0000000000000000 x13: 00000000000443a0 x12: 0000000000040000
x11: 000000000fffffff x10: ffff000024928880 x9 : ffff80000b5c6e98 x8 : ffff000024928000 x7 : 00000000b35d04b9 x6 : 0000000000000000 x5 : fffffc0000000000 x4 : ffff8000cbf2e000 x3 : 0000000000000000 x2 : 0000000000000000 x1 : 0000000000000000 x0 : fffffc0000000000

Call trace:
  hpage_collapse_scan_pmd+0x12ec/0x1a20
  khugepaged+0x7e0/0x8dc
  kthread+0x118/0x11c
  ret_from_fork+0x10/0x20
Code: d34cbc43 cb813061 d37ae421 8b050020 (f9400404)
---[ end trace 0000000000000000 ]---
Kernel panic - not syncing: Oops: Fatal exception

SMP: stopping secondary CPUs
Kernel Offset: disabled
CPU features: 0x8c000,41c78100,0000421b
Memory Limit: none

---[ end Kernel panic - not syncing: Oops: Fatal exception ]---

Reverting it on top of recent linux-next fixes the issue, so it looks that some kind of a corner case is missing in this patch. I can reproduce it usually during the system shutdown, 1 of 20 times on the average.

Quoted 26 lines
---
  mm/khugepaged.c | 14 +++++++-------
  1 file changed, 7 insertions(+), 7 deletions(-)
mm/khugepaged.c +4 -4
index 4888e8688401..27d010431ece 100644
@@ -509,20 +509,20 @@ static void release_pte_page(struct page *page)
  static void release_pte_pages(pte_t *pte, pte_t *_pte,
  		struct list_head *compound_pagelist)
  {
-	struct page *page, *tmp;
+	struct folio *folio, *tmp;
 
  	while (--_pte >= pte) {
  		pte_t pteval = *_pte;
 
-		page = pte_page(pteval);
+		folio = pfn_folio(pte_pfn(pteval));
  		if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
-				!PageCompound(page))
-			release_pte_page(page);
+				!folio_test_large(folio))
+			release_pte_folio(folio);
  	}

mm/khugepaged.c static void release_pte_page(struct page *page) · line 528

Quoted 10 lines
mm/khugepaged.c +3 -3
 
-	list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
-		list_del(&page->lru);
-		release_pte_page(page);
+	list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
+		list_del(&folio->lru);
+		release_pte_folio(folio);
  	}
  }
 

Best regards

Signature · 2 lines
Marek Szyprowski, PhD
Samsung R&D Institute Poland
#3 lore raw

Re: [PATCH 2/2] mm/khugepaged: Convert release_pte_pages() to use folios

Hi,

Quoted 55 lines · Marek Szyprowski

Hi,

Quoted 3 lines · Vishal Moola (Oracle)

Converts release_pte_pages() to use folios instead of pages.

This patch has been merged some time ago to linux-next as commit 9bdfeea46f49 ("mm/khugepaged: convert release_pte_pages() to use folios"). It took me a while to bisect this (mainly because I was busy with other things), but I finally found that this change is responsible for the following kernel panic:

Unable to handle kernel paging request at virtual address fffffc0000000008

Mem abort info:
  ESR = 0x0000000096000006
  EC = 0x25: DABT (current EL), IL = 32 bits
  SET = 0, FnV = 0
  EA = 0, S1PTW = 0
  FSC = 0x06: level 2 translation fault
Data abort info:
  ISV = 0, ISS = 0x00000006
  CM = 0, WnR = 0
swapper pgtable: 4k pages, 48-bit VAs, pgdp=0000000021efa000 [fffffc0000000008] pgd=10000000df05a003, p4d=10000000df05a003, pud=10000000df059003, pmd=0000000000000000

Internal error: Oops: 0000000096000006 [#1] PREEMPT SMP
Modules linked in: ip_tables x_tables ipv6
CPU: 7 PID: 61 Comm: khugepaged Not tainted 6.2.0-rc4+ #13307
Hardware name: Samsung TM2E board (DT)
pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
pc : hpage_collapse_scan_pmd+0x12ec/0x1a20
lr : hpage_collapse_scan_pmd+0x14b0/0x1a20

sp : ffff80000be13c20
x29: ffff80000be13c20 x28: 0000000000000001 x27: fffffc0000d3f5c0
x26: fffffc0000d3f600 x25: 00000000000001f9 x24: 0000000000000007
x23: ffff0000296f9dd8 x22: ffff800009e5b490 x21: 0000000000000000
x20: 000000000000000f x19: ffff80000a9d0000 x18: ffff80000af52e58
x17: 0000000000000028 x16: 0000000000009249 x15: ffff80000af971f8
x14: 0000000000000000 x13: 00000000000443a0 x12: 0000000000040000
x11: 000000000fffffff x10: ffff000024928880 x9 : ffff80000b5c6e98 x8 : ffff000024928000 x7 : 00000000b35d04b9 x6 : 0000000000000000 x5 : fffffc0000000000 x4 : ffff8000cbf2e000 x3 : 0000000000000000 x2 : 0000000000000000 x1 : 0000000000000000 x0 : fffffc0000000000

Call trace:
   hpage_collapse_scan_pmd+0x12ec/0x1a20
   khugepaged+0x7e0/0x8dc
   kthread+0x118/0x11c
   ret_from_fork+0x10/0x20
Code: d34cbc43 cb813061 d37ae421 8b050020 (f9400404)
---[ end trace 0000000000000000 ]---
Kernel panic - not syncing: Oops: Fatal exception

SMP: stopping secondary CPUs
Kernel Offset: disabled
CPU features: 0x8c000,41c78100,0000421b
Memory Limit: none

Quoted 7 lines
---[ end Kernel panic - not syncing: Oops: Fatal exception ]---

Reverting it on top of recent linux-next fixes the issue, so it looks that some kind of a corner case is missing in this patch. I can reproduce it usually during the system shutdown, 1 of 20 times on the average.

I have debugging this issue since this morning too!

Quoted 16 lines
Quoted 16 lines
---
   mm/khugepaged.c | 14 +++++++-------
   1 file changed, 7 insertions(+), 7 deletions(-)
mm/khugepaged.c +1 -1
index 4888e8688401..27d010431ece 100644
@@ -509,20 +509,20 @@ static void release_pte_page(struct page *page)
   static void release_pte_pages(pte_t *pte, pte_t *_pte,
   		struct list_head *compound_pagelist)
   {
-	struct page *page, *tmp;
+	struct folio *folio, *tmp;
 
Quoted 5 lines

mm/khugepaged.c static void release_pte_page(struct page *page) · line 517

Quoted 5 lines
mm/khugepaged.c +1 -1
   	while (--_pte >= pte) {
   		pte_t pteval = *_pte;
 
-		page = pte_page(pteval);
+		folio = pfn_folio(pte_pfn(pteval));

The issue lies here: before using pteval in pfn_folio(), we should test it. The following patch fixes the issue for me:

mm/khugepaged.c +0 -0
index eb38bd1b1b2f..fef3414b481b 100644
@@ -514,10 +514,12 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte,

        while (--_pte >= pte) {
                pte_t pteval = *_pte;

-               folio = pfn_folio(pte_pfn(pteval));
-               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) && -                               !folio_test_large(folio))
-                       release_pte_folio(folio);
+               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) { +                       folio = pfn_folio(pte_pfn(pteval));
+
+                       if (!folio_test_large(folio))
+                               release_pte_folio(folio);
+               }
        }

        list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {

@Marek: could you give it a try?

I can send a separate patch if needed, let me know.

Thanks,

Alex

Quoted 17 lines
Quoted 6 lines
mm/khugepaged.c +2 -2
   		if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
-				!PageCompound(page))
-			release_pte_page(page);
+				!folio_test_large(folio))
+			release_pte_folio(folio);
   	}

mm/khugepaged.c static void release_pte_page(struct page *page) · line 528

Quoted 10 lines
mm/khugepaged.c +3 -3
 
-	list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
-		list_del(&page->lru);
-		release_pte_page(page);
+	list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
+		list_del(&folio->lru);
+		release_pte_folio(folio);
   	}
   }
 

Best regards

#4 lore raw

Re: [PATCH 2/2] mm/khugepaged: Convert release_pte_pages() to use folios

Quoted 2 lines · Alexandre Ghiti

The issue lies here: before using pteval in pfn_folio(), we should test it. The following patch fixes the issue for me:

Thanks for debugging it. I'd rather see this written as ...

		pte_t pteval = *_pte;

+ unsigned long pfn;

+ if (pte_none(pteval))
+ continue;
+ pfn = pte_pfn(pteval);
+ if (is_zero_pfn(pfn))
+ continue;
+ folio = pfn_folio(pfn);
+ if (folio_test_large(folio))

+			continue;
		release_pte_folio(folio);

makes sense?

Quoted 48 lines
mm/khugepaged.c +6 -4
index eb38bd1b1b2f..fef3414b481b 100644
@@ -514,10 +514,12 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte,
         while (--_pte >= pte) {
                 pte_t pteval = *_pte;
 
-               folio = pfn_folio(pte_pfn(pteval));
-               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
-                               !folio_test_large(folio))
-                       release_pte_folio(folio);
+               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) {
+                       folio = pfn_folio(pte_pfn(pteval));
+
+                       if (!folio_test_large(folio))
+                               release_pte_folio(folio);
+               }
         }
 
         list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {

@Marek: could you give it a try?

I can send a separate patch if needed, let me know.

Thanks,

Alex

Quoted 15 lines
Quoted 4 lines
   		if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
(quoted diff) +1 -2
-				!PageCompound(page))
-			release_pte_page(page);
+				!folio_test_large(folio))

Review

Quoted 10 lines
(quoted diff) +4 -3
+			release_pte_folio(folio);
   	}
-	list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
-		list_del(&page->lru);
-		release_pte_page(page);
+	list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
+		list_del(&folio->lru);
+		release_pte_folio(folio);
   	}
   }

Best regards

#5 lore raw

Re: [PATCH 2/2] mm/khugepaged: Convert release_pte_pages() to use folios

Hi Matthew,

Quoted 18 lines · Matthew Wilcox
Quoted 2 lines · Alexandre Ghiti

The issue lies here: before using pteval in pfn_folio(), we should test it. The following patch fixes the issue for me:

Thanks for debugging it. I'd rather see this written as ...

		pte_t pteval = *_pte;
(quoted diff) +9 -0
+		unsigned long pfn;
 
+		if (pte_none(pteval))
+			continue;
+		pfn = pte_pfn(pteval);
+		if (is_zero_pfn(pfn))
+			continue;
+		folio = pfn_folio(pfn);
+		if (folio_test_large(folio))
+			continue;
		release_pte_folio(folio);
Quoted 1 line

makes sense?

Sure, that's fine by me, I can send that or I'll add my tested-by on what you send, whatever suits you.

Alex

Quoted 48 lines
Quoted 48 lines
mm/khugepaged.c +6 -4
index eb38bd1b1b2f..fef3414b481b 100644
@@ -514,10 +514,12 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte,
         while (--_pte >= pte) {
                 pte_t pteval = *_pte;
 
-               folio = pfn_folio(pte_pfn(pteval));
-               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
-                               !folio_test_large(folio))
-                       release_pte_folio(folio);
+               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) {
+                       folio = pfn_folio(pte_pfn(pteval));
+
+                       if (!folio_test_large(folio))
+                               release_pte_folio(folio);
+               }
         }
 
         list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {

@Marek: could you give it a try?

I can send a separate patch if needed, let me know.

Thanks,

Alex

Quoted 15 lines
Quoted 4 lines
    		if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
(quoted diff) +1 -2
-				!PageCompound(page))
-			release_pte_page(page);
+				!folio_test_large(folio))

Review

Quoted 10 lines
(quoted diff) +4 -3
+			release_pte_folio(folio);
    	}
-	list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
-		list_del(&page->lru);
-		release_pte_page(page);
+	list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
+		list_del(&folio->lru);
+		release_pte_folio(folio);
    	}
    }

Best regards

#6 lore raw

Re: [PATCH 2/2] mm/khugepaged: Convert release_pte_pages() to use folios

Quoted 22 lines · Alexandre Ghiti

Hi Matthew,

Quoted 18 lines · Matthew Wilcox
Quoted 2 lines · Alexandre Ghiti

The issue lies here: before using pteval in pfn_folio(), we should test it. The following patch fixes the issue for me:

Thanks for debugging it. I'd rather see this written as ...

              pte_t pteval = *_pte;
(quoted diff) +9 -0
+             unsigned long pfn;
 
+             if (pte_none(pteval))
+                     continue;
+             pfn = pte_pfn(pteval);
+             if (is_zero_pfn(pfn))
+                     continue;
+             folio = pfn_folio(pfn);
+             if (folio_test_large(folio))
+                     continue;
              release_pte_folio(folio);
Quoted 5 lines
Quoted 1 line

makes sense?

Sure, that's fine by me, I can send that or I'll add my tested-by on what you send, whatever suits you.

Thanks for debugging this! I'll send a fix patch using Matthew's approach later today.

Quoted 51 lines

Alex

Quoted 48 lines
Quoted 48 lines
mm/khugepaged.c +6 -4
index eb38bd1b1b2f..fef3414b481b 100644
@@ -514,10 +514,12 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte,
         while (--_pte >= pte) {
                 pte_t pteval = *_pte;
 
-               folio = pfn_folio(pte_pfn(pteval));
-               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
-                               !folio_test_large(folio))
-                       release_pte_folio(folio);
+               if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) {
+                       folio = pfn_folio(pte_pfn(pteval));
+
+                       if (!folio_test_large(folio))
+                               release_pte_folio(folio);
+               }
         }
 
         list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {

@Marek: could you give it a try?

I can send a separate patch if needed, let me know.

Thanks,

Alex

Quoted 15 lines
Quoted 4 lines
                   if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)) &&
(quoted diff) +1 -2
-                          !PageCompound(page))
-                  release_pte_page(page);
+                          !folio_test_large(folio))

Review

Quoted 10 lines
(quoted diff) +4 -3
+                  release_pte_folio(folio);
           }
-  list_for_each_entry_safe(page, tmp, compound_pagelist, lru) {
-          list_del(&page->lru);
-          release_pte_page(page);
+  list_for_each_entry_safe(folio, tmp, compound_pagelist, lru) {
+          list_del(&folio->lru);
+          release_pte_folio(folio);
           }
    }

Best regards