mm/migrate: put dest folio on deferred split list if source was there.
Subsystems (4)
PATCH v2 1/1 mm/migrate: put dest folio on deferred split list if source was there.
From: Zi Yan <[email protected]>
Commit 616b8371539a6 ("mm: thp: enable thp migration in generic path") did not check if a THP is on deferred split list before migration, thus, the destination THP is never put on deferred split list even if the source THP might be. The opportunity of reclaiming free pages in a partially mapped THP during deferred list scanning is lost, but no other harmful consequence is present[1]. Checking source folio deferred split list status before page unmapped and add destination folio to the list if source was after migration.
[1]: https://lore.kernel.org/linux-mm/[email protected]/
From v1:
1. Used dst to get correct deferred split list after migration (per Ryan Roberts).
--- mm/huge_memory.c | 22 ---------------------- mm/internal.h | 23 +++++++++++++++++++++++ mm/migrate.c | 26 +++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 23 deletions(-)
mm/huge_memory.c +0 -22
@@ -766,28 +766,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)return pmd; } -#ifdef CONFIG_MEMCG -static inline -struct deferred_split *get_deferred_split_queue (struct folio *folio) -{ -struct mem_cgroup *memcg =folio_memcg (folio); -struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); - -if (memcg) -return &memcg->deferred_split_queue; -else -return &pgdat->deferred_split_queue; -} -#else -static inline -struct deferred_split *get_deferred_split_queue (struct folio *folio) -{ -struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); - -return &pgdat->deferred_split_queue; -} -#endif -void folio_prep_large_rmappable (struct folio *folio) {if (!folio || !folio_test_large (folio))
mm/internal.h +23 -0
@@ -1107,6 +1107,29 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, unsigned int flags); +#ifdef CONFIG_MEMCG +static inline +struct deferred_split *get_deferred_split_queue (struct folio *folio) +{ +struct mem_cgroup *memcg =folio_memcg (folio); +struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); + +if (memcg) +return &memcg->deferred_split_queue; +else +return &pgdat->deferred_split_queue; +} +#else +static inline +struct deferred_split *get_deferred_split_queue (struct folio *folio) +{ +struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); + +return &pgdat->deferred_split_queue; +} +#endif + +/* * mm/mmap.c */
mm/migrate.c +25 -1
@@ -20,6 +20,7 @@#include <linux/pagemap.h> #include <linux/buffer_head.h> #include <linux/mm_inline.h> +#include <linux/mmzone.h> #include <linux/nsproxy.h> #include <linux/ksm.h> #include <linux/rmap.h>
@@ -1037,7 +1038,10 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,enum {PAGE_WAS_MAPPED =BIT (0 ),PAGE_WAS_MLOCKED =BIT (1 ), -PAGE_OLD_STATES =PAGE_WAS_MAPPED |PAGE_WAS_MLOCKED , +PAGE_WAS_ON_DEFERRED_LIST =BIT (2 ), +PAGE_OLD_STATES =PAGE_WAS_MAPPED | +PAGE_WAS_MLOCKED | +PAGE_WAS_ON_DEFERRED_LIST , }; static void __migrate_folio_record(struct folio *dst,
@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true; +if (folio_test_large_rmappable (src) && + !list_empty (&src->_deferred_list)) { +struct deferred_split *ds_queue =get_deferred_split_queue (src); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len--; +list_del_init (&src->_deferred_list); +spin_unlock (&ds_queue->split_queue_lock); + old_page_state |=PAGE_WAS_ON_DEFERRED_LIST ; + } +if (folio_test_mlocked (src)) old_page_state |=PAGE_WAS_MLOCKED ;
@@ -1307,6 +1322,15 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,if (old_page_state &PAGE_WAS_MAPPED )remove_migration_ptes (src, dst, false); +if (old_page_state &PAGE_WAS_ON_DEFERRED_LIST ) { +struct deferred_split *ds_queue =get_deferred_split_queue (dst); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len++; +list_add (&dst->_deferred_list, &ds_queue->split_queue); +spin_unlock (&ds_queue->split_queue_lock); + } + out_unlock_both:folio_unlock (dst);set_page_owner_migrate_reason (&dst->page, reason);
Signature · 1 line
2.43.0
Quoted 4 lines · Zi Yan
mm/migrate.c +0
-0@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true;
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1184
Quoted 10 lines
mm/migrate.c +10
-0+if (folio_test_large_rmappable (src) && + !list_empty (&src->_deferred_list)) { +struct deferred_split *ds_queue =get_deferred_split_queue (src); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len--; +list_del_init (&src->_deferred_list); +spin_unlock (&ds_queue->split_queue_lock); + old_page_state |=PAGE_WAS_ON_DEFERRED_LIST ; + }
I have a few problems with this ...
Trivial: your whitespace is utterly broken. You can't use a single tab for both indicating control flow change and for line-too-long.
Slightly more important: You're checking list_empty outside the lock (which is fine in order to avoid unnecessarily acquiring the lock), but you need to re-check it inside the lock in case of a race. And you didn't mark it as data_race(), so KMSAN will whinge.
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in deferred_split_scan() that a folio with a positive refcount will not be removed from the list.
Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it. Documentation/mm/transhuge.rst is clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.
If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
Ryan Roberts
reviewerQuoted 6 lines · Matthew Wilcox
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1175
Quoted 5 lines · Zi Yan
mm/migrate.c +1
-0@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true; + if (folio_test_large_rmappable(src) &&
I think you also need to check that the order > 1, now that we support order-1 pagecache folios? _deferred_list only exists if order > 1.
Quoted 12 lines
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1184
Quoted 9 lines
mm/migrate.c +9
-0+ !list_empty(&src->_deferred_list)) { +struct deferred_split *ds_queue =get_deferred_split_queue (src); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len--; +list_del_init (&src->_deferred_list); +spin_unlock (&ds_queue->split_queue_lock); + old_page_state |=PAGE_WAS_ON_DEFERRED_LIST ; + }I have a few problems with this ...
Quoted 7 lines
Trivial: your whitespace is utterly broken. You can't use a single tab for both indicating control flow change and for line-too-long.
Slightly more important: You're checking
list_emptyoutside the lock (which is fine in order to avoid unnecessarily acquiring the lock), but you need to re-check it inside the lock in case of a race. And you didn't mark it asdata_race(), so KMSAN will whinge.
I've seen data_race() used around list_empty() without the lock held inconsistently (see deferred_split_folio()). What are the rules? Given that we are not doing a memory access here, I don't really understand why it is needed? list_empty() uses READ_ONCE() which I thought would be sufficient? (I've just added a similar lockless check in my swap-out series - will add data_race() if needed, but previously concluded it's not).
Quoted 12 lines
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.
Quoted 3 lines
If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
This comment makes me wonder what we do in split_huge_page_to_list_to_order() if the target order is greater than 1 and the input folio is on the deferred split list. Looks like we currently just remove it from the deferred list. Is there a case for putting any output folios that are still partially mapped back on the deferred list, or splitting them to a lower order such that all output folios are fully mapped, and all unmapped portions are freed?
Zi Yan
reviewerQuoted 54 lines · Ryan Roberts
Quoted 6 lines · Matthew Wilcox
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1175
Quoted 5 lines · Zi Yan
mm/migrate.c +1
-0@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true; + if (folio_test_large_rmappable(src) &&I think you also need to check that the order > 1, now that we support order-1 pagecache folios?
_deferred_listonly exists if order > 1.Quoted 12 lines
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1184
Quoted 9 lines
mm/migrate.c +9
-0+ !list_empty(&src->_deferred_list)) { +struct deferred_split *ds_queue =get_deferred_split_queue (src); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len--; +list_del_init (&src->_deferred_list); +spin_unlock (&ds_queue->split_queue_lock); + old_page_state |=PAGE_WAS_ON_DEFERRED_LIST ; + }I have a few problems with this ...
Quoted 7 lines
Trivial: your whitespace is utterly broken. You can't use a single tab for both indicating control flow change and for line-too-long.
Slightly more important: You're checking
list_emptyoutside the lock (which is fine in order to avoid unnecessarily acquiring the lock), but you need to re-check it inside the lock in case of a race. And you didn't mark it asdata_race(), so KMSAN will whinge.I've seen
data_race()used aroundlist_empty()without the lock held inconsistently (seedeferred_split_folio()). What are the rules? Given that we are not doing a memory access here, I don't really understand why it is needed?list_empty()usesREAD_ONCE()which I thought would be sufficient? (I've just added a similar lockless check in my swap-out series - will adddata_race()if needed, but previously concluded it's not).Quoted 12 lines
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.Quoted 3 lines
If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
Quoted 6 lines
This comment makes me wonder what we do in
split_huge_page_to_list_to_order()if the target order is greater than 1 and the input folio is on the deferred split list. Looks like we currently just remove it from the deferred list. Is there a case for putting any output folios that are still partially mapped back on the deferred list, or splitting them to a lower order such that all output folios are fully mapped, and all unmapped portions are freed?
I probably would let the caller of split_huge_page_to_list_to_order() to decide whether output folios should be put back in deferred list. The caller should determine the right order to split. Letting split_huge_page_to_list_to_order() change new_order will confuse the caller and complicate the handling of output folios.
--
Best Regards,
Yan, Zi
Zi Yan
reviewerQuoted 18 lines · Matthew Wilcox
Quoted 4 lines · Zi Yan
mm/migrate.c +0
-0@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true;mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1184
Quoted 10 lines
mm/migrate.c +10
-0+if (folio_test_large_rmappable (src) && + !list_empty (&src->_deferred_list)) { +struct deferred_split *ds_queue =get_deferred_split_queue (src); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len--; +list_del_init (&src->_deferred_list); +spin_unlock (&ds_queue->split_queue_lock); + old_page_state |=PAGE_WAS_ON_DEFERRED_LIST ; + }I have a few problems with this ...
Quoted 2 lines
Trivial: your whitespace is utterly broken. You can't use a single tab for both indicating control flow change and for line-too-long.
Got it. Will not do this any more.
Quoted 5 lines
Slightly more important: You're checking
list_emptyoutside the lock (which is fine in order to avoid unnecessarily acquiring the lock), but you need to re-check it inside the lock in case of a race. And you didn't mark it asdata_race(), so KMSAN will whinge.
Got it and will check data_race() related changes. Will fix.
Quoted 4 lines
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.
What is the issue here? I thought as long as the split_queue_lock is held, it should be OK to manipulate the list.
Quoted 11 lines
Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
You are suggesting:
1. checking if the folio is on deferred split list or not
2. if yes, split the folio
3. if split fails, fail the migration as well.
It sounds reasonable to me. The split folios should be migrated since the before-split folio wants to be migrated. This split is not because no new page cannot be allocated, thus the split folios should go into ret_folios list instead of split_folios list.
Thank you for the comments.
--
Best Regards,
Yan, Zi
Quoted 7 lines · Zi Yan
Quoted 3 lines · Matthew Wilcox
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.What is the issue here? I thought as long as the
split_queue_lockis held, it should be OK to manipulate the list.
I just worked this out yesterday:
https://lore.kernel.org/linux-mm/[email protected]/
(the last chunk, starting with Ryan asking me "what about the first bug you found")
Quoted 16 lines
Quoted 10 lines
Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
You are suggesting:
1. checking if the folio is on deferred split list or not
2. if yes, split the folio
3. if split fails, fail the migration as well.
Quoted 4 lines
It sounds reasonable to me. The split folios should be migrated since the before-split folio wants to be migrated. This split is not because no new page cannot be allocated, thus the split folios should go into
ret_folioslist instead ofsplit_folioslist.
Yes, I'm happy for the split folios to be migrated. Bonus points if you want to figure out what order to split the folio to ;-) I don't think it's critical.
Zi Yan
reviewerQuoted 6 lines · Matthew Wilcox
Quoted 5 lines · Zi Yan
Quoted 3 lines · Matthew Wilcox
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.
Quoted 7 lines
Quoted 2 lines
What is the issue here? I thought as long as the
split_queue_lockis held, it should be OK to manipulate the list.I just worked this out yesterday:
https://lore.kernel.org/linux-mm/[email protected]/
(the last chunk, starting with Ryan asking me "what about the first bug you found")
Hmm, like you said a folio with a positive refcount will not be removed from ds_queue->split_queue, it will have no chance going to the separate list in deferred_list_scan() and list_del_init() will not corrupt that list. So it should be safe. Or the issue is that before migration adding a refcount, the folio is removed from ds_queue->split_queue and put on the list in deferred_list_scan(), as a result, any manipulation of folio->_deferred_list could corrupt the list. Basically, !list_empty(&folio->_deferred_list) cannot tell if the folio is on ds_queue->split_queue or another list. I am not sure about why "a positive refcount" is related here.
That makes me wonder whether ds_queue->split_queue_lock is also needed for list_for_each_entry_safe() in deferred_split_scan(). Basically, ds_queue->split_queue_lock protects folio->_deferred_list in addition to ds_queue->split_queue.
Quoted 16 lines
Quoted 16 lines
Quoted 10 lines
Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
You are suggesting:
1. checking if the folio is on deferred split list or not
2. if yes, split the folio
3. if split fails, fail the migration as well.
Quoted 8 lines
Quoted 4 lines
It sounds reasonable to me. The split folios should be migrated since the before-split folio wants to be migrated. This split is not because no new page cannot be allocated, thus the split folios should go into
ret_folioslist instead ofsplit_folioslist.Yes, I'm happy for the split folios to be migrated. Bonus points if you want to figure out what order to split the folio to ;-) I don't think it's critical.
--
Best Regards,
Yan, Zi
Quoted 16 lines · Zi Yan
Quoted 6 lines · Matthew Wilcox
Quoted 5 lines · Zi Yan
Quoted 3 lines · Matthew Wilcox
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.Quoted 7 lines
Quoted 2 lines
What is the issue here? I thought as long as the
split_queue_lockis held, it should be OK to manipulate the list.I just worked this out yesterday:
https://lore.kernel.org/linux-mm/[email protected]/
(the last chunk, starting with Ryan asking me "what about the first bug you found")
Quoted 4 lines
Hmm, like you said a folio with a positive refcount will not be removed from
ds_queue->split_queue, it will have no chance going to the separate list indeferred_list_scan()andlist_del_init()will not corrupt that list.
You've misread it. Folios with a _zero_ refcount are not removed from the list in deferred_split_scan. Folios with a positive refcount are removed from the per-node or per-cgroup list _at which point there is an undocumented assumption_ that they will not be removed from the local list because they have a positive refcount.
Quoted 44 lines
So it should be safe. Or the issue is that before migration adding a refcount, the folio is removed from
ds_queue->split_queueand put on the list indeferred_list_scan(), as a result, any manipulation offolio->_deferred_listcould corrupt the list. Basically, !list_empty(&folio->_deferred_list) cannot tell if the folio is onds_queue->split_queueor another list. I am not sure about why "a positive refcount" is related here.That makes me wonder whether
ds_queue->split_queue_lockis also needed forlist_for_each_entry_safe()indeferred_split_scan(). Basically,ds_queue->split_queue_lockprotectsfolio->_deferred_listin addition tods_queue->split_queue.Quoted 16 lines
Quoted 16 lines
Quoted 10 lines
Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
You are suggesting:
1. checking if the folio is on deferred split list or not
2. if yes, split the folio
3. if split fails, fail the migration as well.Quoted 8 lines
Quoted 4 lines
It sounds reasonable to me. The split folios should be migrated since the before-split folio wants to be migrated. This split is not because no new page cannot be allocated, thus the split folios should go into
ret_folioslist instead ofsplit_folioslist.Yes, I'm happy for the split folios to be migrated. Bonus points if you want to figure out what order to split the folio to ;-) I don't think it's critical.
--
Best Regards,
Yan, Zi
Zi Yan
reviewerQuoted 22 lines · Matthew Wilcox
Quoted 16 lines · Zi Yan
Quoted 6 lines · Matthew Wilcox
Quoted 5 lines · Zi Yan
Quoted 3 lines · Matthew Wilcox
Much more important: You're doing this with a positive refcount, which breaks the (undocumented) logic in
deferred_split_scan()that a folio with a positive refcount will not be removed from the list.Quoted 7 lines
Quoted 2 lines
What is the issue here? I thought as long as the
split_queue_lockis held, it should be OK to manipulate the list.I just worked this out yesterday:
https://lore.kernel.org/linux-mm/[email protected]/
(the last chunk, starting with Ryan asking me "what about the first bug you found")Quoted 4 lines
Hmm, like you said a folio with a positive refcount will not be removed from
ds_queue->split_queue, it will have no chance going to the separate list indeferred_list_scan()andlist_del_init()will not corrupt that list.
Quoted 5 lines
You've misread it. Folios with a _zero_ refcount are not removed from the list in
deferred_split_scan. Folios with a positive refcount are removed from the per-node or per-cgroup list_atwhich point there is an undocumentedassumption_that they will not be removed from the local list because they have a positive refcount.
But that sounds very subtle if not broken. As an outsider of deferred_split_scan(), only !list_empty(folio->_deferred_list) is checked. The condition can be true if the folio is on split_queue or local list of deferred_split_scan() with elevated refcount. In that case, the folio cannot be removed from the list (either split_queue or local list) even if split_queue_lock is held, since local list manipulation is not under split_queue_lock. This makes _deferred_list a one-way train to anyone except deferred_split_scan(), namely folios can only be added into _deferred_list until they are freed or split by deferred_split_scan().
Is that intended? If yes, maybe we should document it. If not, using split_queue_lock to protect local list, or more explicitly folio->_deferred_list might be better?
Quoted 36 lines
Quoted 36 lines
So it should be safe. Or the issue is that before migration adding a refcount, the folio is removed from
ds_queue->split_queueand put on the list indeferred_list_scan(), as a result, any manipulation offolio->_deferred_listcould corrupt the list. Basically, !list_empty(&folio->_deferred_list) cannot tell if the folio is onds_queue->split_queueor another list. I am not sure about why "a positive refcount" is related here.That makes me wonder whether
ds_queue->split_queue_lockis also needed forlist_for_each_entry_safe()indeferred_split_scan(). Basically,ds_queue->split_queue_lockprotectsfolio->_deferred_listin addition tods_queue->split_queue.Quoted 21 lines
Quoted 20 lines
Quoted 10 lines
Maximally important: Wer shouldn't be doing any of this! This folio is on the deferred split list. We shouldn't be migrating it as a single entity; we should be splitting it now that we're in a context where we can do the right thing and split it.
Documentation/mm/transhuge.rstis clear that we don't split it straight away due to locking context. Splitting it on migration is clearly the right thing to do.If splitting fails, we should just fail the migration; splitting fails due to excess references, and if the source folio has excess references, then migration would fail too.
You are suggesting:
1. checking if the folio is on deferred split list or not
2. if yes, split the folio
3. if split fails, fail the migration as well.It sounds reasonable to me. The split folios should be migrated since the before-split folio wants to be migrated. This split is not because no new page cannot be allocated, thus the split folios should go into
ret_folioslist instead ofsplit_folioslist.
Quoted 8 lines
Quoted 8 lines
Quoted 3 lines
Yes, I'm happy for the split folios to be migrated. Bonus points if you want to figure out what order to split the folio to ;-) I don't think it's critical.
--
Best Regards,
Yan, Zi
--
Best Regards,
Yan, Zi
Quoted 7 lines · Zi Yan
Quoted 4 lines · Matthew Wilcox
Folios with a positive refcount are
removed from the per-node or per-cgroup list_atwhich point there is an undocumentedassumption_that they will not be removed from the local list because they have a positive refcount.But that sounds very subtle if not broken. As an outsider of
I merely deduced this requirement; I didn't come up with it ...
Quoted 9 lines
deferred_split_scan(), only !list_empty(folio->_deferred_list) is checked. The condition can be true if the folio is onsplit_queueor local list ofdeferred_split_scan()with elevated refcount. In that case, the folio cannot be removed from the list (eithersplit_queueor local list) even ifsplit_queue_lockis held, since local list manipulation is not undersplit_queue_lock. This makes_deferred_lista one-way train to anyone exceptdeferred_split_scan(), namely folios can only be added into_deferred_listuntil they are freed or split bydeferred_split_scan().
Quoted 3 lines
Is that intended? If yes, maybe we should document it. If not, using
split_queue_lockto protect local list, or more explicitlyfolio->_deferred_listmight be better?
To be fair, the folio can be split by anybody as split_huge_page_to_list_to_order() is careful to only manipulate the deferred list while the refcount is frozen at 0. I'm still trying to figure out where to document this behaviour of the deferred list that someone (for example, your good self) would actually see it.
Zi Yan
reviewerQuoted 24 lines · Matthew Wilcox
Quoted 7 lines · Zi Yan
Quoted 4 lines · Matthew Wilcox
Folios with a positive refcount are
removed from the per-node or per-cgroup list_atwhich point there is an undocumentedassumption_that they will not be removed from the local list because they have a positive refcount.But that sounds very subtle if not broken. As an outsider of
I merely deduced this requirement; I didn't come up with it ...
Quoted 9 lines
deferred_split_scan(), only !list_empty(folio->_deferred_list) is checked. The condition can be true if the folio is onsplit_queueor local list ofdeferred_split_scan()with elevated refcount. In that case, the folio cannot be removed from the list (eithersplit_queueor local list) even ifsplit_queue_lockis held, since local list manipulation is not undersplit_queue_lock. This makes_deferred_lista one-way train to anyone exceptdeferred_split_scan(), namely folios can only be added into_deferred_listuntil they are freed or split bydeferred_split_scan().Quoted 3 lines
Is that intended? If yes, maybe we should document it. If not, using
split_queue_lockto protect local list, or more explicitlyfolio->_deferred_listmight be better?
Quoted 5 lines
To be fair, the folio can be split by anybody as
split_huge_page_to_list_to_order()is careful to only manipulate the deferred list while the refcount is frozen at 0. I'm still trying to figure out where to document this behaviour of the deferred list that someone (for example, your good self) would actually see it.
Yes, split is not affected. For migration, hypothetically if a folio on deferred_list wants to be migrated without split, it needs to removed and the dst folio needs to be added during frozen phase. That seems to be very heavy operation to just get a folio out of deferred_list. But we can change it when we want to do that in other scenarios.
BTW, s/_deferred_list/_deferred_list_only_removable_during_scan/ works best for me in terms of documentation. ;)
--
Best Regards,
Yan, Zi
Quoted 10 lines · Matthew Wilcox
Quoted 7 lines · Zi Yan
Quoted 4 lines · Matthew Wilcox
Folios with a positive refcount are
removed from the per-node or per-cgroup list_atwhich point there is an undocumentedassumption_that they will not be removed from the local list because they have a positive refcount.But that sounds very subtle if not broken. As an outsider of
I merely deduced this requirement; I didn't come up with it ...
My understanding is that this requirement is because of just local list in deferred_split_scan().
Using fbatch instead of local list here as your created for that issue debugging can eliminate this subtlety?
Regards
Yin, Fengwei
Quoted 22 lines
Quoted 10 lines
deferred_split_scan(), only !list_empty(folio->_deferred_list) is checked. The condition can be true if the folio is onsplit_queueor local list ofdeferred_split_scan()with elevated refcount. In that case, the folio cannot be removed from the list (eithersplit_queueor local list) even ifsplit_queue_lockis held, since local list manipulation is not undersplit_queue_lock. This makes_deferred_lista one-way train to anyone exceptdeferred_split_scan(), namely folios can only be added into_deferred_listuntil they are freed or split bydeferred_split_scan().Quoted 3 lines
Is that intended? If yes, maybe we should document it. If not, using
split_queue_lockto protect local list, or more explicitlyfolio->_deferred_listmight be better?To be fair, the folio can be split by anybody as
split_huge_page_to_list_to_order()is careful to only manipulate the deferred list while the refcount is frozen at 0. I'm still trying to figure out where to document this behaviour of the deferred list that someone (for example, your good self) would actually see it.
Quoted 10 lines · Fengwei
Quoted 7 lines · Matthew Wilcox
Quoted 6 lines · Zi Yan
Quoted 4 lines · Matthew Wilcox
Folios with a positive refcount are
removed from the per-node or per-cgroup list_atwhich point there is an undocumentedassumption_that they will not be removed from the local list because they have a positive refcount.
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
But that sounds very subtle if not broken. As an outsider of
I merely deduced this requirement; I didn't come up with it ...
My understanding is that this requirement is because of just local list in
deferred_split_scan().Using fbatch instead of local list here as your created for that issue debugging can eliminate this subtlety?
May not good idea because it's possible the folios in fbatch can be removed from deferred_list by migration.
Quoted 28 lines
Regards
Yin, FengweiQuoted 22 lines
Quoted 10 lines
deferred_split_scan(), only !list_empty(folio->_deferred_list) is checked. The condition can be true if the folio is onsplit_queueor local list ofdeferred_split_scan()with elevated refcount. In that case, the folio cannot be removed from the list (eithersplit_queueor local list) even ifsplit_queue_lockis held, since local list manipulation is not undersplit_queue_lock. This makes_deferred_lista one-way train to anyone exceptdeferred_split_scan(), namely folios can only be added into_deferred_listuntil they are freed or split bydeferred_split_scan().Quoted 3 lines
Is that intended? If yes, maybe we should document it. If not, using
split_queue_lockto protect local list, or more explicitlyfolio->_deferred_listmight be better?To be fair, the folio can be split by anybody as
split_huge_page_to_list_to_order()is careful to only manipulate the deferred list while the refcount is frozen at 0. I'm still trying to figure out where to document this behaviour of the deferred list that someone (for example, your good self) would actually see it.
Baolin Wang
reviewerQuoted 117 lines · Zi Yan
From: Zi Yan <[email protected]>
Commit
616b8371539a6("mm: thp: enable thp migration in generic path") did not check if a THP is on deferred split list before migration, thus, the destination THP is never put on deferred split list even if the source THP might be. The opportunity of reclaiming free pages in a partially mapped THP during deferred list scanning is lost, but no other harmful consequence is present[1]. Checking source folio deferred split list status before page unmapped and add destination folio to the list if source was after migration.[1]: https://lore.kernel.org/linux-mm/[email protected]/
From v1:
1. Used dst to get correct deferred split list after migration (per Ryan Roberts).Fixes 616b8371539a ("mm: thp: enable thp migration in generic path")Signed-off-by Zi Yan <[email protected]>--- mm/huge_memory.c | 22 ---------------------- mm/internal.h | 23 +++++++++++++++++++++++ mm/migrate.c | 26 +++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 23 deletions(-)mm/huge_memory.c +0
-22@@ -766,28 +766,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)return pmd; } -#ifdef CONFIG_MEMCG -static inline -struct deferred_split *get_deferred_split_queue (struct folio *folio) -{ -struct mem_cgroup *memcg =folio_memcg (folio); -struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); - -if (memcg) -return &memcg->deferred_split_queue; -else -return &pgdat->deferred_split_queue; -} -#else -static inline -struct deferred_split *get_deferred_split_queue (struct folio *folio) -{ -struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); - -return &pgdat->deferred_split_queue; -} -#endif -void folio_prep_large_rmappable (struct folio *folio) {if (!folio || !folio_test_large (folio))mm/internal.h +23
-0@@ -1107,6 +1107,29 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, unsigned int flags); +#ifdef CONFIG_MEMCG +static inline +struct deferred_split *get_deferred_split_queue (struct folio *folio) +{ +struct mem_cgroup *memcg =folio_memcg (folio); +struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); + +if (memcg) +return &memcg->deferred_split_queue; +else +return &pgdat->deferred_split_queue; +} +#else +static inline +struct deferred_split *get_deferred_split_queue (struct folio *folio) +{ +struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); + +return &pgdat->deferred_split_queue; +} +#endif + +/* * mm/mmap.c */ mm/migrate.c +5
-1@@ -20,6 +20,7 @@#include <linux/pagemap.h> #include <linux/buffer_head.h> #include <linux/mm_inline.h> +#include <linux/mmzone.h> #include <linux/nsproxy.h> #include <linux/ksm.h> #include <linux/rmap.h> @@ -1037,7 +1038,10 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,enum {PAGE_WAS_MAPPED =BIT (0 ),PAGE_WAS_MLOCKED =BIT (1 ), -PAGE_OLD_STATES =PAGE_WAS_MAPPED |PAGE_WAS_MLOCKED , +PAGE_WAS_ON_DEFERRED_LIST =BIT (2 ), +PAGE_OLD_STATES =PAGE_WAS_MAPPED | +PAGE_WAS_MLOCKED | +PAGE_WAS_ON_DEFERRED_LIST , }; static void __migrate_folio_record(struct folio *dst,
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1175
Quoted 4 lines
mm/migrate.c +1
-0@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true; + if (folio_test_large_rmappable(src) &&
IMO, you should check folio_test_large() before calling folio_test_large_rmappable(), since the PG_large_rmappable flag is stored in the first tail page.
Quoted 29 lines
mm/migrate.c +19
-0+ !list_empty(&src->_deferred_list)) { +struct deferred_split *ds_queue =get_deferred_split_queue (src); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len--; +list_del_init (&src->_deferred_list); +spin_unlock (&ds_queue->split_queue_lock); + old_page_state |=PAGE_WAS_ON_DEFERRED_LIST ; + } +if (folio_test_mlocked (src)) old_page_state |=PAGE_WAS_MLOCKED ;@@ -1307,6 +1322,15 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,if (old_page_state &PAGE_WAS_MAPPED )remove_migration_ptes (src, dst, false); +if (old_page_state &PAGE_WAS_ON_DEFERRED_LIST ) { +struct deferred_split *ds_queue =get_deferred_split_queue (dst); + +spin_lock (&ds_queue->split_queue_lock); + ds_queue->split_queue_len++; +list_add (&dst->_deferred_list, &ds_queue->split_queue); +spin_unlock (&ds_queue->split_queue_lock); + } + out_unlock_both:folio_unlock (dst);set_page_owner_migrate_reason (&dst->page, reason);
Zi Yan
reviewerQuoted 113 lines · Baolin Wang
Quoted 112 lines · Zi Yan
From: Zi Yan <[email protected]>
Commit
616b8371539a6("mm: thp: enable thp migration in generic path") did not check if a THP is on deferred split list before migration, thus, the destination THP is never put on deferred split list even if the source THP might be. The opportunity of reclaiming free pages in a partially mapped THP during deferred list scanning is lost, but no other harmful consequence is present[1]. Checking source folio deferred split list status before page unmapped and add destination folio to the list if source was after migration.[1]: https://lore.kernel.org/linux-mm/[email protected]/
From v1:
1. Used dst to get correct deferred split list after migration (per Ryan Roberts).Fixes 616b8371539a ("mm: thp: enable thp migration in generic path")Signed-off-by Zi Yan <[email protected]>--- mm/huge_memory.c | 22 ---------------------- mm/internal.h | 23 +++++++++++++++++++++++ mm/migrate.c | 26 +++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 23 deletions(-)mm/huge_memory.c +0
-21@@ -766,28 +766,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)return pmd; } -#ifdef CONFIG_MEMCG -static inline -struct deferred_split *get_deferred_split_queue (struct folio *folio) -{ -struct mem_cgroup *memcg =folio_memcg (folio); -struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); - -if (memcg) -return &memcg->deferred_split_queue; -else -return &pgdat->deferred_split_queue; -} -#else -static inline -struct deferred_split *get_deferred_split_queue (struct folio *folio) -{ -struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); - -return &pgdat->deferred_split_queue; -} -#endif -void folio_prep_large_rmappable (struct folio *folio) {if (!folio || !folio_test_large (folio))mm/internal.h +22
-0@@ -1107,6 +1107,29 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, unsigned int flags); +#ifdef CONFIG_MEMCG +static inline +struct deferred_split *get_deferred_split_queue (struct folio *folio) +{ +struct mem_cgroup *memcg =folio_memcg (folio); +struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); + +if (memcg) +return &memcg->deferred_split_queue; +else +return &pgdat->deferred_split_queue; +} +#else +static inline +struct deferred_split *get_deferred_split_queue (struct folio *folio) +{ +struct pglist_data *pgdat =NODE_DATA (folio_nid (folio)); + +return &pgdat->deferred_split_queue; +} +#endif + +/* * mm/mmap.c */ mm/migrate.c +5
-1@@ -20,6 +20,7 @@#include <linux/pagemap.h> #include <linux/buffer_head.h> #include <linux/mm_inline.h> +#include <linux/mmzone.h> #include <linux/nsproxy.h> #include <linux/ksm.h> #include <linux/rmap.h> @@ -1037,7 +1038,10 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,enum {PAGE_WAS_MAPPED =BIT (0 ),PAGE_WAS_MLOCKED =BIT (1 ), -PAGE_OLD_STATES =PAGE_WAS_MAPPED |PAGE_WAS_MLOCKED , +PAGE_WAS_ON_DEFERRED_LIST =BIT (2 ), +PAGE_OLD_STATES =PAGE_WAS_MAPPED | +PAGE_WAS_MLOCKED | +PAGE_WAS_ON_DEFERRED_LIST , };
Quoted 8 lines
mm/migrate.c static int migrate_folio_unmap(new_folio_t get_new_folio, · line 1175
Quoted 6 lines
mm/migrate.c +1
-0static void __migrate_folio_record(struct folio *dst,@@ -1168,6 +1172,17 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,folio_lock (src); } locked = true; + if (folio_test_large_rmappable(src) &&IMO, you should check
folio_test_large()before callingfolio_test_large_rmappable(), since thePG_large_rmappableflag is stored in the first tail page.
You are right. Ryan also pointed this out in another email. Will fix. Thanks.
--
Best Regards,
Yan, Zi