mm, hwpoison: improve handling workload related to hugetlb and memory_hotplug
Subsystems (8)
v7 cover mm, hwpoison: improve handling workload related to hugetlb and memory_hotplug
Hi,
This patchset tries to solve the issue among memory_hotplug, hugetlb and hwpoison. In this patchset, memory hotplug handles hwpoison pages like below:
- hwpoison pages should not prevent memory hotremove,
- memory block with hwpoison pages should not be onlined.
Changes in this version:
- (patch 1/4) determine migratable_cleared atomically by introducing TESTCLEARHPAGEFLAG (suggested by Miaohe),
- (patch 4/4) cleaned up declarations of counter API,
- rebased onto akpm/mm-unstable on Oct 23.
Thanks,
Naoya Horiguchi
v1: https://lore.kernel.org/linux-mm/[email protected]/T
v2: https://lore.kernel.org/linux-mm/[email protected]/T
v3-5: https://lore.kernel.org/linux-mm/[email protected]/T
v6: https://lore.kernel.org/linux-mm/[email protected]/T
---
Summary:
Naoya Horiguchi (4):
mm,hwpoison,hugetlb,memory_hotplug: hotremove memory section with hwpoisoned hugepage mm/hwpoison: move definitions of num_poisoned_pages_* to memory-failure.c mm/hwpoison: pass pfn to num_poisoned_pages_*()
mm/hwpoison: introduce per-memory_block hwpoison counter
arch/parisc/kernel/pdt.c | 5 ++-- drivers/base/memory.c | 38 ++++++++++++++++++++++++++ include/linux/hugetlb.h | 19 ++++++++++--- include/linux/memory.h | 3 +++ include/linux/mm.h | 29 ++++++++++++++++++-- include/linux/swapops.h | 24 ++--------------- mm/hugetlb.c | 9 ++++--- mm/internal.h | 8 ------ mm/memory-failure.c | 69 ++++++++++++++++++++++++++---------------------- mm/sparse.c | 2 -- 10 files changed, 130 insertions(+), 76 deletions(-)
PATCH v7 1/4 mm,hwpoison,hugetlb,memory_hotplug: hotremove memory section with hwpoisoned hugepage
From: Naoya Horiguchi <[email protected]>
HWPoisoned page is not supposed to be accessed once marked, but currently such accesses can happen during memory hotremove because do_migrate_range() can be called before dissolve_free_huge_pages() is called.
Clear HPageMigratable for hwpoisoned hugepages to prevent them from being migrated. This should be done in hugetlb_lock to avoid race against isolate_hugetlb().
get_hwpoison_huge_page() needs to have a flag to show it's called from unpoison to take refcount of hwpoisoned hugepages, so add it.
---
ChangeLog v3 -> v7:
- introduce TESTCLEARHPAGEFLAG() to determine the value of migratable_cleared
ChangeLog v3 -> v6:
- introduce migratable_cleared to remember that HPageMigratable is cleared in error handling. It's needed to cancel when an error event is filtered by hwpoison_filter(). (Thanks to Miaohe)
ChangeLog v2 -> v3
- move to the approach of clearing HPageMigratable instead of shifting dissolve_free_huge_pages.
--- include/linux/hugetlb.h | 19 +++++++++++++++---- include/linux/mm.h | 6 ++++-- mm/hugetlb.c | 9 +++++---- mm/memory-failure.c | 19 +++++++++++++++---- 4 files changed, 39 insertions(+), 14 deletions(-)
include/linux/hugetlb.h +15 -4
@@ -183,8 +183,9 @@ bool hugetlb_reserve_pages(struct inode *inode, long from, long to,long hugetlb_unreserve_pages (struct inode *inode,long start,long end,long freed);int isolate_hugetlb (struct page *page,struct list_head *list); -int get_hwpoison_huge_page (struct page *page,bool *hugetlb); -int get_huge_page_for_hwpoison (unsigned long pfn,int flags); +int get_hwpoison_huge_page (struct page *page,bool *hugetlb,bool unpoison); +int get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared);void putback_active_hugepage (struct page *page);void move_hugetlb_state (struct folio *old_folio,struct folio *new_folio,int reason);void free_huge_page (struct page *page);
@@ -391,12 +392,13 @@ static inline int isolate_hugetlb(struct page *page, struct list_head *list)return -EBUSY ; } -static inline int get_hwpoison_huge_page (struct page *page,bool *hugetlb) +static inline int get_hwpoison_huge_page (struct page *page,bool *hugetlb,bool unpoison) {return 0 ; } -static inline int get_huge_page_for_hwpoison (unsigned long pfn,int flags) +static inline int get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {return 0 ; }
@@ -614,6 +616,10 @@ void folio_clear_hugetlb_##flname(struct folio *folio) \ } \ static inline void ClearHPage##uname(struct page *page) \ { clear_bit(HPG_##flname, &(page->private)); } + +#define TESTCLEARHPAGEFLAG (uname, flname) \ +static inline int TestClearHPage##uname(struct page *page) \ + { return test_and_clear_bit(HPG_##flname, &(page->private)); }#else #define TESTHPAGEFLAG (uname, flname) \ static inline bool \
@@ -635,6 +641,10 @@ folio_clear_hugetlb_##flname(struct folio *folio) \ { } \ static inline void ClearHPage##uname(struct page *page) \ { } + +#define TESTCLEARHPAGEFLAG (uname, flname) \ +static inline int TestClearHPage##uname(struct page *page) \ + { return 0; }#endif #define HPAGEFLAG (uname, flname) \
@@ -647,6 +657,7 @@ static inline void ClearHPage##uname(struct page *page) \*/ HPAGEFLAG(RestoreReserve, restore_reserve) HPAGEFLAG(Migratable, migratable) + TESTCLEARHPAGEFLAG(Migratable, migratable) HPAGEFLAG(Temporary, temporary) HPAGEFLAG(Freed, freed)HPAGEFLAG (VmemmapOptimized, vmemmap_optimized)
include/linux/mm.h +4 -2
@@ -3246,9 +3246,11 @@ extern void shake_page(struct page *p);extern atomic_long_t num_poisoned_pages__read_mostly ;extern int soft_offline_page (unsigned long pfn,int flags);#ifdef CONFIG_MEMORY_FAILURE -extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags); +extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared);#else -static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags) +static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {return 0 ; }
mm/hugetlb.c +5 -4
@@ -7292,7 +7292,7 @@ int isolate_hugetlb(struct page *page, struct list_head *list)return ret; } -int get_hwpoison_huge_page (struct page *page,bool *hugetlb) +int get_hwpoison_huge_page (struct page *page,bool *hugetlb,bool unpoison) {int ret =0 ;
@@ -7302,7 +7302,7 @@ int get_hwpoison_huge_page(struct page *page, bool *hugetlb) *hugetlb = true;if (HPageFreed (page)) ret =0 ; -else if (HPageMigratable (page)) +else if (HPageMigratable (page) || unpoison) ret =get_page_unless_zero (page);else ret = -EBUSY ;
@@ -7311,12 +7311,13 @@ int get_hwpoison_huge_page(struct page *page, bool *hugetlb)return ret; } -int get_huge_page_for_hwpoison (unsigned long pfn,int flags) +int get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {int ret;spin_lock_irq (&hugetlb_lock); - ret =__get_huge_page_for_hwpoison (pfn, flags); + ret =__get_huge_page_for_hwpoison (pfn, flags, migratable_cleared);spin_unlock_irq (&hugetlb_lock);return ret; }
mm/memory-failure.c +15 -4
@@ -1250,7 +1250,7 @@ static int __get_hwpoison_page(struct page *page, unsigned long flags)int ret =0 ;bool hugetlb = false; - ret =get_hwpoison_huge_page (head, &hugetlb); + ret =get_hwpoison_huge_page (head, &hugetlb, false);if (hugetlb)return ret;
@@ -1340,7 +1340,7 @@ static int __get_unpoison_page(struct page *page)int ret =0 ;bool hugetlb = false; - ret =get_hwpoison_huge_page (head, &hugetlb); + ret =get_hwpoison_huge_page (head, &hugetlb, true);if (hugetlb)return ret;
@@ -1791,7 +1791,8 @@ void hugetlb_clear_page_hwpoison(struct page *hpage)* -EBUSY - the hugepage is busy (try to retry) * -EHWPOISON - the hugepage is already hwpoisoned */ -int __get_huge_page_for_hwpoison (unsigned long pfn,int flags) +int __get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {struct page *page =pfn_to_page (pfn);struct page *head =compound_head (page);
@@ -1821,6 +1822,13 @@ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags) goto out; } +/* +* Clearing HPageMigratable for hwpoisoned hugepages to prevent them +* from being migrated by memory hotremove. +*/ +if (count_increased) + *migratable_cleared =TestClearHPageMigratable (head); +return ret; out:if (count_increased)
@@ -1840,10 +1848,11 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlbstruct page *p =pfn_to_page (pfn);struct page *head;unsigned long page_flags; +bool migratable_cleared = false; *hugetlb =1 ; retry: - res =get_huge_page_for_hwpoison (pfn, flags); + res =get_huge_page_for_hwpoison (pfn, flags, &migratable_cleared);if (res ==2 ) {/* fallback to normal page handling */ *hugetlb =0 ;return 0 ;
@@ -1867,6 +1876,8 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlbif (hwpoison_filter (p)) {hugetlb_clear_page_hwpoison (head); +if (migratable_cleared) +SetHPageMigratable (head);unlock_page (head);if (res ==1 )put_page (head);
Signature · 1 line
2.25.1
Miaohe Lin
maintainerRe: [PATCH v7 1/4] mm,hwpoison,hugetlb,memory_hotplug: hotremove memory section with hwpoisoned hugepage
Quoted 13 lines · Naoya Horiguchi
From: Naoya Horiguchi <[email protected]>
HWPoisoned page is not supposed to be accessed once marked, but currently such accesses can happen during memory hotremove because
do_migrate_range()can be called beforedissolve_free_huge_pages()is called.Clear HPageMigratable for hwpoisoned hugepages to prevent them from being migrated. This should be done in
hugetlb_lockto avoid race againstisolate_hugetlb().
get_hwpoison_huge_page()needs to have a flag to show it's called from unpoison to take refcount of hwpoisoned hugepages, so add it.
Quoted 7 lines
Reported-by Miaohe Lin <[email protected]>Signed-off-by Naoya Horiguchi <[email protected]>Reviewed-by Oscar Salvador <[email protected]>Reviewed-by Miaohe Lin <[email protected]>---
ChangeLog v3 -> v7:
- introduceTESTCLEARHPAGEFLAG()to determine the value ofmigratable_cleared
Many thanks for update, Naoya. I'm sorry but TestClearHPageMigratable() might be somewhat overkill. As we discussed in previous thread:
"""
I think I might be nitpicking... But it seems ClearHPageMigratable is not enough here.
1. In MF_COUNT_INCREASED case, we don't know whether HPageMigratable is set.
2. Even if HPageMigratable is set, there might be a race window before we clear HPageMigratable?
So "*migratable_cleared = TestClearHPageMigratable" might be better? But I might be wrong. """
The case 2 should be a dumb problem(sorry about it). HPageMigratable() is always cleared while holding the hugetlb_lock which is already held by get_huge_page_for_hwpoison(). So the only case we should care about is case 1 and that can be handled by below more efficient pattern:
if (HPageMigratable)
ClearHPageMigratable()
So the overhead of test and clear atomic ops can be avoided. But this is trival.
Anyway, this patch still looks good to me. And my Reviewed-by tag still applies. Many thanks.
Thanks,
Miaohe Lin
PATCH v8 1/4 mm,hwpoison,hugetlb,memory_hotplug: hotremove memory section with hwpoisoned hugepage
Quoted 38 lines · Miaohe Lin
Quoted 13 lines · Naoya Horiguchi
From: Naoya Horiguchi <[email protected]>
HWPoisoned page is not supposed to be accessed once marked, but currently such accesses can happen during memory hotremove because
do_migrate_range()can be called beforedissolve_free_huge_pages()is called.Clear HPageMigratable for hwpoisoned hugepages to prevent them from being migrated. This should be done in
hugetlb_lockto avoid race againstisolate_hugetlb().
get_hwpoison_huge_page()needs to have a flag to show it's called from unpoison to take refcount of hwpoisoned hugepages, so add it.Quoted 7 lines
Reported-by Miaohe Lin <[email protected]>Signed-off-by Naoya Horiguchi <[email protected]>Reviewed-by Oscar Salvador <[email protected]>Reviewed-by Miaohe Lin <[email protected]>---
ChangeLog v3 -> v7:
- introduceTESTCLEARHPAGEFLAG()to determine the value ofmigratable_clearedMany thanks for update, Naoya. I'm sorry but
TestClearHPageMigratable()might be somewhat overkill. As we discussed in previous thread:"""
I think I might be nitpicking... But it seems ClearHPageMigratable is not enough here.
1. InMF_COUNT_INCREASEDcase, we don't know whether HPageMigratable is set.
2. Even if HPageMigratable is set, there might be a race window before we clear HPageMigratable?
So "*migratable_cleared= TestClearHPageMigratable" might be better? But I might be wrong. """The case 2 should be a dumb problem(sorry about it).
HPageMigratable()is always cleared while holding thehugetlb_lockwhich is already held byget_huge_page_for_hwpoison(). So the only case we should care about is case 1 and that can be handled by below more efficient pattern:
if (HPageMigratable)
ClearHPageMigratable()
Quoted 3 lines
So the overhead of test and clear atomic ops can be avoided. But this is trival.
Anyway, this patch still looks good to me. And my Reviewed-by tag still applies. Many thanks.
OK, so I replace this 1/4 with the following one, thank you.
- Naoya Horiguchi
---
From 22cbd7649d1e23db272306f8d066edb15d4e322c Mon Sep 17 00:00:00 2001
From: Naoya Horiguchi <[email protected]>
Date: Tue, 25 Oct 2022 14:18:10 +0900
Subject: [PATCH v8 1/4] mm,hwpoison,hugetlb,memory_hotplug: hotremove memory section
with hwpoisoned hugepage
HWPoisoned page is not supposed to be accessed once marked, but currently such accesses can happen during memory hotremove because do_migrate_range() can be called before dissolve_free_huge_pages() is called.
Clear HPageMigratable for hwpoisoned hugepages to prevent them from being migrated. This should be done in hugetlb_lock to avoid race against isolate_hugetlb().
get_hwpoison_huge_page() needs to have a flag to show it's called from unpoison to take refcount of hwpoisoned hugepages, so add it.
---
ChangeLog v7 -> v8:
- remove TestClearHPageMigratable and reduce to test and clear separately.
ChangeLog v3 -> v7:
- introduce TESTCLEARHPAGEFLAG() to determine the value of migratable_cleared
ChangeLog v3 -> v6:
- introduce migratable_cleared to remember that HPageMigratable is cleared in error handling. It's needed to cancel when an error event is filtered by hwpoison_filter(). (Thanks to Miaohe)
ChangeLog v2 -> v3
- move to the approach of clearing HPageMigratable instead of shifting dissolve_free_huge_pages.
--- include/linux/hugetlb.h | 10 ++++++---- include/linux/mm.h | 6 ++++-- mm/hugetlb.c | 9 +++++---- mm/memory-failure.c | 21 +++++++++++++++++---- 4 files changed, 32 insertions(+), 14 deletions(-)
include/linux/hugetlb.h +6 -4
@@ -183,8 +183,9 @@ bool hugetlb_reserve_pages(struct inode *inode, long from, long to,long hugetlb_unreserve_pages (struct inode *inode,long start,long end,long freed);int isolate_hugetlb (struct page *page,struct list_head *list); -int get_hwpoison_huge_page (struct page *page,bool *hugetlb); -int get_huge_page_for_hwpoison (unsigned long pfn,int flags); +int get_hwpoison_huge_page (struct page *page,bool *hugetlb,bool unpoison); +int get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared);void putback_active_hugepage (struct page *page);void move_hugetlb_state (struct folio *old_folio,struct folio *new_folio,int reason);void free_huge_page (struct page *page);
@@ -391,12 +392,13 @@ static inline int isolate_hugetlb(struct page *page, struct list_head *list)return -EBUSY ; } -static inline int get_hwpoison_huge_page (struct page *page,bool *hugetlb) +static inline int get_hwpoison_huge_page (struct page *page,bool *hugetlb,bool unpoison) {return 0 ; } -static inline int get_huge_page_for_hwpoison (unsigned long pfn,int flags) +static inline int get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {return 0 ; }
include/linux/mm.h +4 -2
@@ -3246,9 +3246,11 @@ extern void shake_page(struct page *p);extern atomic_long_t num_poisoned_pages__read_mostly ;extern int soft_offline_page (unsigned long pfn,int flags);#ifdef CONFIG_MEMORY_FAILURE -extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags); +extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared);#else -static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags) +static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {return 0 ; }
mm/hugetlb.c +5 -4
@@ -7292,7 +7292,7 @@ int isolate_hugetlb(struct page *page, struct list_head *list)return ret; } -int get_hwpoison_huge_page (struct page *page,bool *hugetlb) +int get_hwpoison_huge_page (struct page *page,bool *hugetlb,bool unpoison) {int ret =0 ;
@@ -7302,7 +7302,7 @@ int get_hwpoison_huge_page(struct page *page, bool *hugetlb) *hugetlb = true;if (HPageFreed (page)) ret =0 ; -else if (HPageMigratable (page)) +else if (HPageMigratable (page) || unpoison) ret =get_page_unless_zero (page);else ret = -EBUSY ;
@@ -7311,12 +7311,13 @@ int get_hwpoison_huge_page(struct page *page, bool *hugetlb)return ret; } -int get_huge_page_for_hwpoison (unsigned long pfn,int flags) +int get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {int ret;spin_lock_irq (&hugetlb_lock); - ret =__get_huge_page_for_hwpoison (pfn, flags); + ret =__get_huge_page_for_hwpoison (pfn, flags, migratable_cleared);spin_unlock_irq (&hugetlb_lock);return ret; }
mm/memory-failure.c +17 -4
@@ -1250,7 +1250,7 @@ static int __get_hwpoison_page(struct page *page, unsigned long flags)int ret =0 ;bool hugetlb = false; - ret =get_hwpoison_huge_page (head, &hugetlb); + ret =get_hwpoison_huge_page (head, &hugetlb, false);if (hugetlb)return ret;
@@ -1340,7 +1340,7 @@ static int __get_unpoison_page(struct page *page)int ret =0 ;bool hugetlb = false; - ret =get_hwpoison_huge_page (head, &hugetlb); + ret =get_hwpoison_huge_page (head, &hugetlb, true);if (hugetlb)return ret;
@@ -1791,7 +1791,8 @@ void hugetlb_clear_page_hwpoison(struct page *hpage)* -EBUSY - the hugepage is busy (try to retry) * -EHWPOISON - the hugepage is already hwpoisoned */ -int __get_huge_page_for_hwpoison (unsigned long pfn,int flags) +int __get_huge_page_for_hwpoison (unsigned long pfn,int flags, +bool *migratable_cleared) {struct page *page =pfn_to_page (pfn);struct page *head =compound_head (page);
@@ -1821,6 +1822,15 @@ int __get_huge_page_for_hwpoison(unsigned long pfn, int flags) goto out; } +/* +* Clearing HPageMigratable for hwpoisoned hugepages to prevent them +* from being migrated by memory hotremove. +*/ +if (count_increased &&HPageMigratable (head)) { +ClearHPageMigratable (head); + *migratable_cleared = true; + } +return ret; out:if (count_increased)
@@ -1840,10 +1850,11 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlbstruct page *p =pfn_to_page (pfn);struct page *head;unsigned long page_flags; +bool migratable_cleared = false; *hugetlb =1 ; retry: - res =get_huge_page_for_hwpoison (pfn, flags); + res =get_huge_page_for_hwpoison (pfn, flags, &migratable_cleared);if (res ==2 ) {/* fallback to normal page handling */ *hugetlb =0 ;return 0 ;
@@ -1867,6 +1878,8 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlbif (hwpoison_filter (p)) {hugetlb_clear_page_hwpoison (head); +if (migratable_cleared) +SetHPageMigratable (head);unlock_page (head);if (res ==1 )put_page (head);
Signature · 1 line
2.37.3.518.g79f2338b37
Miaohe Lin
maintainerRe: [PATCH v8 1/4] mm,hwpoison,hugetlb,memory_hotplug: hotremove memory section with hwpoisoned hugepage
Quoted 39 lines · Naoya Horiguchi
Quoted 38 lines · Miaohe Lin
Quoted 13 lines · Naoya Horiguchi
From: Naoya Horiguchi <[email protected]>
HWPoisoned page is not supposed to be accessed once marked, but currently such accesses can happen during memory hotremove because
do_migrate_range()can be called beforedissolve_free_huge_pages()is called.Clear HPageMigratable for hwpoisoned hugepages to prevent them from being migrated. This should be done in
hugetlb_lockto avoid race againstisolate_hugetlb().
get_hwpoison_huge_page()needs to have a flag to show it's called from unpoison to take refcount of hwpoisoned hugepages, so add it.Quoted 7 lines
Reported-by Miaohe Lin <[email protected]>Signed-off-by Naoya Horiguchi <[email protected]>Reviewed-by Oscar Salvador <[email protected]>Reviewed-by Miaohe Lin <[email protected]>---
ChangeLog v3 -> v7:
- introduceTESTCLEARHPAGEFLAG()to determine the value ofmigratable_clearedMany thanks for update, Naoya. I'm sorry but
TestClearHPageMigratable()might be somewhat overkill. As we discussed in previous thread:"""
I think I might be nitpicking... But it seems ClearHPageMigratable is not enough here.
1. InMF_COUNT_INCREASEDcase, we don't know whether HPageMigratable is set.
2. Even if HPageMigratable is set, there might be a race window before we clear HPageMigratable?
So "*migratable_cleared= TestClearHPageMigratable" might be better? But I might be wrong. """The case 2 should be a dumb problem(sorry about it).
HPageMigratable()is always cleared while holding thehugetlb_lockwhich is already held byget_huge_page_for_hwpoison(). So the only case we should care about is case 1 and that can be handled by below more efficient pattern:
if (HPageMigratable)
ClearHPageMigratable()
Quoted 5 lines
Quoted 3 lines
So the overhead of test and clear atomic ops can be avoided. But this is trival.
Anyway, this patch still looks good to me. And my Reviewed-by tag still applies. Many thanks.
OK, so I replace this 1/4 with the following one, thank you.
Many thanks for your update, Naoya. ;) This version looks good to me too.
Thanks,
Miaohe Lin
PATCH v7 2/4 mm/hwpoison: move definitions of num_poisoned_pages_* to memory-failure.c
In reply to Naoya HoriguchiFrom: Naoya Horiguchi <[email protected]>
These interfaces will be used by drivers/base/memory.c by later patch, so as a preparatory work move them to more common header file visible to the file.
---
ChangeLog v3 -> v6:
- remove static in definition of num_poisoned_pages_inc() to fix build error.
ChangeLog v2 -> v3:
- added declaration of num_poisoned_pages_inc() in #ifdef CONFIG_MEMORY_FAILURE
--- arch/parisc/kernel/pdt.c | 3 +-- include/linux/mm.h | 5 +++++ include/linux/swapops.h | 24 ++---------------------- mm/memory-failure.c | 10 ++++++++++ 4 files changed, 18 insertions(+), 24 deletions(-)
arch/parisc/kernel/pdt.c +1 -2
@@ -18,8 +18,7 @@#include <linux/kthread.h> #include <linux/initrd.h> #include <linux/pgtable.h> -#include <linux/swap.h> -#include <linux/swapops.h> +#include <linux/mm.h> #include <asm/pdc.h> #include <asm/pdcpat.h>
include/linux/mm.h +5 -0
@@ -3248,12 +3248,17 @@ extern int soft_offline_page(unsigned long pfn, int flags);#ifdef CONFIG_MEMORY_FAILURE extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags,bool *migratable_cleared); +extern void num_poisoned_pages_inc (void );#else static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags,bool *migratable_cleared) {return 0 ; } + +static inline void num_poisoned_pages_inc (void ) +{ +}#endif #ifndef arch_memory_failure
include/linux/swapops.h +2 -22
@@ -581,8 +581,6 @@ static inline int is_pmd_migration_entry(pmd_t pmd)#ifdef CONFIG_MEMORY_FAILURE -extern atomic_long_t num_poisoned_pages__read_mostly ; -/* * Support for hardware poisoned pages */
@@ -610,17 +608,7 @@ static inline struct page *hwpoison_entry_to_page(swp_entry_t entry) return p; } -static inline void num_poisoned_pages_inc (void ) -{ -atomic_long_inc (&num_poisoned_pages); -} - -static inline void num_poisoned_pages_sub (long i) -{ -atomic_long_sub (i, &num_poisoned_pages); -} - -#else /* CONFIG_MEMORY_FAILURE */ +#else static inline swp_entry_t make_hwpoison_entry (struct page *page) {
@@ -636,15 +624,7 @@ static inline struct page *hwpoison_entry_to_page(swp_entry_t entry) {return NULL ; } - -static inline void num_poisoned_pages_inc (void ) -{ -} - -static inline void num_poisoned_pages_sub (long i) -{ -} -#endif /* CONFIG_MEMORY_FAILURE */ +#endif static inline int non_swap_entry (swp_entry_t entry) {
mm/memory-failure.c +10 -0
@@ -74,6 +74,16 @@ atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);static bool hw_memory_failure__read_mostly = false; +inline void num_poisoned_pages_inc (void ) +{ +atomic_long_inc (&num_poisoned_pages); +} + +static inline void num_poisoned_pages_sub (long i) +{ +atomic_long_sub (i, &num_poisoned_pages); +} +/* * Return values: * 1: the page is dissolved (if needed) and taken off from buddy,
Signature · 1 line
2.25.1
PATCH v7 3/4 mm/hwpoison: pass pfn to num_poisoned_pages_*()
In reply to Naoya HoriguchiFrom: Naoya Horiguchi <[email protected]>
No functional change.
---
ChangeLog v2 -> v3:
- added declaration of num_poisoned_pages_inc() in #ifdef CONFIG_MEMORY_FAILURE
--- arch/parisc/kernel/pdt.c | 2 +- include/linux/mm.h | 4 ++-- mm/memory-failure.c | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-)
arch/parisc/kernel/pdt.c +1 -1
@@ -231,7 +231,7 @@ void __init pdc_pdt_init(void)/* mark memory page bad */ memblock_reserve (pdt_entry[i] &PAGE_MASK ,PAGE_SIZE ); -num_poisoned_pages_inc (); +num_poisoned_pages_inc (addr >>PAGE_SHIFT ); } }
include/linux/mm.h +2 -2
@@ -3248,7 +3248,7 @@ extern int soft_offline_page(unsigned long pfn, int flags);#ifdef CONFIG_MEMORY_FAILURE extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags,bool *migratable_cleared); -extern void num_poisoned_pages_inc (void ); +extern void num_poisoned_pages_inc (unsigned long pfn);#else static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags,bool *migratable_cleared)
@@ -3256,7 +3256,7 @@ static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, return 0; } -static inline void num_poisoned_pages_inc (void ) +static inline void num_poisoned_pages_inc (unsigned long pfn) { }#endif
mm/memory-failure.c +7 -7
@@ -74,12 +74,12 @@ atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);static bool hw_memory_failure__read_mostly = false; -inline void num_poisoned_pages_inc (void ) +inline void num_poisoned_pages_inc (unsigned long pfn) {atomic_long_inc (&num_poisoned_pages); } -static inline void num_poisoned_pages_sub (long i) +static inline void num_poisoned_pages_sub (unsigned long pfn,long i) {atomic_long_sub (i, &num_poisoned_pages); }
@@ -125,7 +125,7 @@ static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, booif (release)put_page (page);page_ref_inc (page); -num_poisoned_pages_inc (); +num_poisoned_pages_inc (page_to_pfn (page));return true; }
@@ -1198,7 +1198,7 @@ static int action_result(unsigned long pfn, enum mf_action_page_type type, {trace_memory_failure_event (pfn, type, result); -num_poisoned_pages_inc (); +num_poisoned_pages_inc (pfn);pr_err ("%#lx: recovery action for %s: %s\n" , pfn, action_page_types[type], action_name[result]);
@@ -1747,7 +1747,7 @@ static int hugetlb_set_page_hwpoison(struct page *hpage, struct page *page)llist_add (&raw_hwp->node, head);/* the first error event will be counted in action_result(). */ if (ret) -num_poisoned_pages_inc (); +num_poisoned_pages_inc (page_to_pfn (page)); }else {/* * Failed to save raw error info. We no longer trace all
@@ -2421,7 +2421,7 @@ int unpoison_memory(unsigned long pfn) unlock_mutex:mutex_unlock (&mf_mutex);if (!ret || freeit) { -num_poisoned_pages_sub (count); +num_poisoned_pages_sub (pfn, count);unpoison_pr_info ("Unpoison: Software-unpoisoned page %#lx\n" ,page_to_pfn (p), &unpoison_rs); }
@@ -2637,5 +2637,5 @@ void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) } }if (total) -num_poisoned_pages_sub (total); +num_poisoned_pages_sub (0 , total); }
Signature · 1 line
2.25.1
PATCH v7 4/4 mm/hwpoison: introduce per-memory_block hwpoison counter
In reply to Naoya HoriguchiFrom: Naoya Horiguchi <[email protected]>
Currently PageHWPoison flag does not behave well when experiencing memory hotremove/hotplug. Any data field in struct page is unreliable when the associated memory is offlined, and the current mechanism can't tell whether a memory block is onlined because a new memory devices is installed or because previous failed offline operations are undone. Especially if there's a hwpoisoned memory, it's unclear what the best option is.
So introduce a new mechanism to make struct memory_block remember that a memory block has hwpoisoned memory inside it. And make any online event fail if the onlining memory block contains hwpoison. struct memory_block is freed and reallocated over ACPI-based hotremove/hotplug, but not over sysfs-based hotremove/hotplug. So the new counter can distinguish these cases.
---
ChangeLog v6 -> v7:
- remove 'extern' in declarations of num_poisoned_pages_{inc,sub} because they are not necessary and definition have 'inline'.
- remove unneeded declaration of memblk_nr_poison_{inc,sub}.
ChangeLog v5 -> v6:
- fix build errors over memblk_nr_poison_inc() and memblk_nr_poison_sub(),
- pass "struct memory_block *" to memblk_nr_poison() instead of pfn,
- removed clear_hwpoisoned_pages() and call num_poisoned_pages_sub() directly.
- add static keyword to the definition of memblk_nr_poison().
- Mioahe added Reviewed-by for v5, but I have some non trivial changes in v6, so let me hold to add it.
- unpoison_memory() properly cancels per-memblk hwpoison counter.
ChangeLog v4 -> v5:
- add Reported-by of lkp bot,
- check both CONFIG_MEMORY_FAILURE and CONFIG_MEMORY_HOTPLUG in introduced #ifdefs, intending to fix "undefined reference" errors in aarch64.
ChangeLog v3 -> v4:
- fix build error (https://lore.kernel.org/linux-mm/[email protected]/)
by using memblk_nr_poison() to access to the member ->nr_hwpoison
--- drivers/base/memory.c | 38 ++++++++++++++++++++++++++++++++++++++ include/linux/memory.h | 3 +++ include/linux/mm.h | 20 +++++++++++++++++++- mm/internal.h | 8 -------- mm/memory-failure.c | 36 +++++++++++------------------------- mm/sparse.c | 2 -- 6 files changed, 71 insertions(+), 36 deletions(-)
drivers/base/memory.c +38 -0
@@ -175,6 +175,15 @@ int memory_notify(unsigned long val, void *v)return blocking_notifier_call_chain (&memory_chain, val, v); } +#if defined(CONFIG_MEMORY_FAILURE ) && defined(CONFIG_MEMORY_HOTPLUG ) +static unsigned long memblk_nr_poison (struct memory_block *mem); +#else +static inline unsigned long memblk_nr_poison (struct memory_block *mem) +{ +return 0 ; +} +#endif +static int memory_block_online (struct memory_block *mem) {unsigned long start_pfn =section_nr_to_pfn (mem->start_section_nr);
@@ -183,6 +192,9 @@ static int memory_block_online(struct memory_block *mem)struct zone *zone;int ret; +if (memblk_nr_poison (mem)) +return -EHWPOISON ; + zone =zone_for_pfn_range (mem->online_type, mem->nid, mem->group, start_pfn, nr_pages);
@@ -864,6 +876,7 @@ void remove_memory_block_devices(unsigned long start, unsigned long size) mem =find_memory_block_by_id (block_id);if (WARN_ON_ONCE (!mem))continue ; +num_poisoned_pages_sub (-1UL ,memblk_nr_poison (mem));unregister_memory_block_under_nodes (mem);remove_memory_block (mem); }
@@ -1164,3 +1177,28 @@ int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func, }return ret; } + +#if defined(CONFIG_MEMORY_FAILURE ) && defined(CONFIG_MEMORY_HOTPLUG ) +void memblk_nr_poison_inc (unsigned long pfn) +{ +const unsigned long block_id =pfn_to_block_id (pfn); +struct memory_block *mem =find_memory_block_by_id (block_id); + +if (mem) +atomic_long_inc (&mem->nr_hwpoison); +} + +void memblk_nr_poison_sub (unsigned long pfn,long i) +{ +const unsigned long block_id =pfn_to_block_id (pfn); +struct memory_block *mem =find_memory_block_by_id (block_id); + +if (mem) +atomic_long_sub (i, &mem->nr_hwpoison); +} + +static unsigned long memblk_nr_poison (struct memory_block *mem) +{ +return atomic_long_read (&mem->nr_hwpoison); +} +#endif
include/linux/memory.h +3 -0
@@ -84,6 +84,9 @@ struct memory_block {unsigned long nr_vmemmap_pages;struct memory_group *group;/* group (if any) for this block */ struct list_head group_next;/* next block inside memory group */ +#if defined(CONFIG_MEMORY_FAILURE ) && defined(CONFIG_MEMORY_HOTPLUG ) +atomic_long_t nr_hwpoison; +#endif };int arch_get_memory_phys_device (unsigned long start_pfn);
include/linux/mm.h +19 -1
@@ -3248,7 +3248,8 @@ extern int soft_offline_page(unsigned long pfn, int flags);#ifdef CONFIG_MEMORY_FAILURE extern int __get_huge_page_for_hwpoison (unsigned long pfn,int flags,bool *migratable_cleared); -extern void num_poisoned_pages_inc (unsigned long pfn); +void num_poisoned_pages_inc (unsigned long pfn); +void num_poisoned_pages_sub (unsigned long pfn,long i);#else static inline int __get_huge_page_for_hwpoison (unsigned long pfn,int flags,bool *migratable_cleared)
@@ -3259,6 +3260,23 @@ static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,static inline void num_poisoned_pages_inc (unsigned long pfn) { } + +static inline void num_poisoned_pages_sub (unsigned long pfn,long i) +{ +} +#endif + +#if defined(CONFIG_MEMORY_FAILURE ) && defined(CONFIG_MEMORY_HOTPLUG ) +extern void memblk_nr_poison_inc (unsigned long pfn); +extern void memblk_nr_poison_sub (unsigned long pfn,long i); +#else +static inline void memblk_nr_poison_inc (unsigned long pfn) +{ +} + +static inline void memblk_nr_poison_sub (unsigned long pfn,long i) +{ +}#endif #ifndef arch_memory_failure
mm/internal.h +0 -8
@@ -708,14 +708,6 @@ extern u64 hwpoison_filter_flags_value;extern u64 hwpoison_filter_memcg;extern u32 hwpoison_filter_enable; -#ifdef CONFIG_MEMORY_FAILURE -void clear_hwpoisoned_pages (struct page *memmap,int nr_pages); -#else -static inline void clear_hwpoisoned_pages (struct page *memmap,int nr_pages) -{ -} -#endif -extern unsigned long __must_check vm_mmap_pgoff (struct file *,unsigned long ,unsigned long ,unsigned long ,unsigned long ,unsigned long );
mm/memory-failure.c +11 -25
@@ -77,11 +77,14 @@ static bool hw_memory_failure __read_mostly = false;inline void num_poisoned_pages_inc (unsigned long pfn) {atomic_long_inc (&num_poisoned_pages); +memblk_nr_poison_inc (pfn); } -static inline void num_poisoned_pages_sub (unsigned long pfn,long i) +inline void num_poisoned_pages_sub (unsigned long pfn,long i) {atomic_long_sub (i, &num_poisoned_pages); +if (pfn !=-1UL ) +memblk_nr_poison_sub (pfn, i); }/*
@@ -1712,6 +1715,8 @@ static unsigned long __free_raw_hwp_pages(struct page *hpage, bool move_flag)if (move_flag)SetPageHWPoison (p->page); +else +num_poisoned_pages_sub (page_to_pfn (p->page),1 );kfree (p); count++; }
@@ -2339,6 +2344,7 @@ int unpoison_memory(unsigned long pfn)int ret = -EBUSY ;int freeit =0 ;unsigned long count =1 ; +bool huge = false; static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST);
@@ -2387,6 +2393,7 @@ int unpoison_memory(unsigned long pfn) ret =get_hwpoison_page (p,MF_UNPOISON );if (!ret) {if (PageHuge (p)) { + huge = true; count =free_raw_hwp_pages (page, false);if (count ==0 ) { ret = -EBUSY ;
@@ -2402,6 +2409,7 @@ int unpoison_memory(unsigned long pfn) pfn, &unpoison_rs); }else {if (PageHuge (p)) { + huge = true; count =free_raw_hwp_pages (page, false);if (count ==0 ) { ret = -EBUSY ;
@@ -2421,7 +2429,8 @@ int unpoison_memory(unsigned long pfn) unlock_mutex:mutex_unlock (&mf_mutex);if (!ret || freeit) { -num_poisoned_pages_sub (pfn, count); +if (!huge) +num_poisoned_pages_sub (pfn,1 );unpoison_pr_info ("Unpoison: Software-unpoisoned page %#lx\n" ,page_to_pfn (p), &unpoison_rs); }
@@ -2616,26 +2625,3 @@ int soft_offline_page(unsigned long pfn, int flags)return ret; } - -void clear_hwpoisoned_pages (struct page *memmap,int nr_pages) -{ -int i, total =0 ; - -/* -* A further optimization is to have per section refcounted -* num_poisoned_pages. But that would need more space per memmap, so -* for now just do a quick global check to speed up this routine in the -* absence of bad pages. -*/ -if (atomic_long_read (&num_poisoned_pages) ==0 ) -return ; - -for (i =0 ; i < nr_pages; i++) { -if (PageHWPoison (&memmap[i])) { - total++; -ClearPageHWPoison (&memmap[i]); - } - } -if (total) -num_poisoned_pages_sub (0 , total); -}
mm/sparse.c +0 -2
@@ -926,8 +926,6 @@ void sparse_remove_section(struct mem_section *ms, unsigned long pfn, unsigned long nr_pages, unsigned long map_offset, struct vmem_altmap *altmap) { - clear_hwpoisoned_pages(pfn_to_page(pfn) + map_offset, - nr_pages - map_offset);section_deactivate (pfn ,nr_pages ,altmap ); }#endif /* CONFIG_MEMORY_HOTPLUG */
Signature · 1 line
2.25.1