Invalidate BH LRU during page migration
Subsystems (4)
A page containing buffer_heads can be pinned if any of its constituent buffer_heads belongs to the BH LRU cache [1]. After going through several iterations of a patch that attempts to solve this by removing BH entries inside of the drop_buffers() function, which in the worst-case could be called for each migrated page, Minchan Kim suggested that we invalidate the entire BH LRU once, just before we start migrating pages. Additionally, Matthew Wilcox suggested that we invalidate the BH LRU inside of lru_add_drain_all(), so as to benefit functions like other functions that would be impacted by pinned pages [2].
TODO:
- It should be possible to remove the initial setting of bh_migration_done = false; in migrate_prep by passing this in as a parameter to invalidate_bh_lru(), but we'd still need a matching bh_migration_done = true; call.
- To really benefit other callers of lru_add_drain_all() other than __alloc_contig_migrate_range() in the CMA allocaiton path, we'd need to add additional calls of bh_migration_done = false;
[1] https://elixir.bootlin.com/linux/latest/source/fs/buffer.c#L1238
[2] https://lore.kernel.org/linux-fsdevel/[email protected]/
Chris Goldsworthy (1):
[RFC] mm: fs: Invalidate BH LRU during page migration
fs/buffer.c | 6 ++++++ include/linux/buffer_head.h | 3 +++ include/linux/migrate.h | 2 ++ mm/migrate.c | 18 ++++++++++++++++++ mm/page_alloc.c | 3 +++ mm/swap.c | 3 +++ 6 files changed, 35 insertions(+)
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project
RFC PATCH 1/1 mm: fs: Invalidate BH LRU during page migration
Pages containing buffer_heads that are in the buffer_head LRU cache will be pinned and thus cannot be migrated. Correspondingly, invalidate the BH LRU before a migration starts and stop any buffer_head from being cached in the LRU, until migration has finished.
--- fs/buffer.c | 6 ++++++ include/linux/buffer_head.h | 3 +++ include/linux/migrate.h | 2 ++ mm/migrate.c | 18 ++++++++++++++++++ mm/page_alloc.c | 3 +++ mm/swap.c | 3 +++ 6 files changed, 35 insertions(+)
fs/buffer.c +6 -0
@@ -1289,6 +1289,8 @@ static inline void check_irqs_on(void)#endif } +bool bh_migration_done = true; +/* * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is * inserted at the front, and the buffer_head at the back if any is evicted.
@@ -1303,6 +1305,9 @@ static void bh_lru_install(struct buffer_head *bh)check_irqs_on ();bh_lru_lock (); +if (!bh_migration_done) + goto out; + b =this_cpu_ptr (&bh_lrus);for (i =0 ; i <BH_LRU_SIZE ; i++) {swap (evictee, b->bhs[i]);
@@ -1313,6 +1318,7 @@ static void bh_lru_install(struct buffer_head *bh) }get_bh (bh); +out:bh_lru_unlock ();brelse (evictee); }
include/linux/buffer_head.h +3 -0
@@ -193,6 +193,9 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size, gfp_t gfp);struct buffer_head *__bread_gfp (struct block_device *,sector_t block,unsigned size,gfp_t gfp); + +extern bool bh_migration_done; +void invalidate_bh_lrus (void );struct buffer_head *alloc_buffer_head (gfp_t gfp_flags);void free_buffer_head (struct buffer_head * bh);
include/linux/migrate.h +2 -0
@@ -46,6 +46,7 @@ extern int isolate_movable_page(struct page *page, isolate_mode_t mode);extern void putback_movable_page (struct page *page);extern void migrate_prep (void ); +extern void migrate_finish (void );extern void migrate_prep_local (void );extern void migrate_page_states (struct page *newpage,struct page *page);extern void migrate_page_copy (struct page *newpage,struct page *page);
@@ -67,6 +68,7 @@ static inline int isolate_movable_page(struct page *page, isolate_mode_t mode) {return -EBUSY ; }static inline int migrate_prep (void ) {return -ENOSYS ; } +static inline int migrate_finish (void ) {return -ENOSYS ; }static inline int migrate_prep_local (void ) {return -ENOSYS ; }static inline void migrate_page_states (struct page *newpage,struct page *page)
mm/migrate.c +18 -0
@@ -64,6 +64,19 @@*/ void migrate_prep (void ) { + bh_migration_done = false; + +/* +* This barrier ensures that callers of bh_lru_install() between +* the barrier and the call to invalidate_bh_lrus() read +* bh_migration_done() as false. +*/ +/* +* TODO: Remove me? lru_add_drain_all() already has an smp_mb(), +* but it would be good to ensure that the barrier isn't forgotten. +*/ +smp_mb (); +/* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have
@@ -73,6 +86,11 @@ void migrate_prep(void)lru_add_drain_all (); } +void migrate_finish (void ) +{ + bh_migration_done = true; +} +/* Do the necessary work of migrate_prep but not if it involves other CPUs */ void migrate_prep_local (void ) {
mm/page_alloc.c +3 -0
@@ -8493,6 +8493,9 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, ret =migrate_pages (&cc->migratepages, alloc_migration_target,NULL , (unsigned long )&mtc, cc->mode,MR_CONTIG_RANGE ); } + +migrate_finish (); +if (ret <0 ) {putback_movable_pages (&cc->migratepages);return ret;
mm/swap.c +3 -0
@@ -36,6 +36,7 @@#include <linux/hugetlb.h> #include <linux/page_idle.h> #include <linux/local_lock.h> +#include <linux/buffer_head.h> #include "internal.h"
@@ -759,6 +760,8 @@ void lru_add_drain_all(void)if (WARN_ON (!mm_percpu_wq))return ; +invalidate_bh_lrus (); +/* * Guarantee pagevec counter stores visible by this CPU are visible to * other CPUs before loading the current drain generation.
Signature · 2 lines
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project
Andrew Morton
maintainerRe: [PATCH] [RFC] mm: fs: Invalidate BH LRU during page migration
Quoted 5 lines · On Mon, 1 Feb 2021 22:55:47 -0800 Chris Goldsworthy <[email protected]>…
Pages containing
buffer_headsthat are in thebuffer_headLRU cache will be pinned and thus cannot be migrated. Correspondingly, invalidate the BH LRU before a migration starts and stop anybuffer_headfrom being cached in the LRU, until migration has finished.
It's 16 pages max, system-wide. That isn't much.
Please include here a full description of why this is a problem and how serious it is and of the user-visible impact.
Quoted 12 lines
fs/buffer.c +2
-0@@ -1289,6 +1289,8 @@ static inline void check_irqs_on(void)#endif } +bool bh_migration_done = true; +/* * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is * inserted at the front, and the buffer_head at the back if any is evicted.
fs/buffer.c static void bh_lru_install(struct buffer_head *bh) · line 1313
Quoted 9 lines
fs/buffer.c +3
-0@@ -1303,6 +1305,9 @@ static void bh_lru_install(struct buffer_head *bh)check_irqs_on ();bh_lru_lock (); +if (!bh_migration_done) + goto out; + b =this_cpu_ptr (&bh_lrus);for (i =0 ; i <BH_LRU_SIZE ; i++) {swap (evictee, b->bhs[i]);
Seems a bit hacky, but I guess it'll work.
I expect the code won't compile with CONFIG_BLOCK=n && CONFIG_MIGRATION=y. Due to bh_migration_done being unimplemented.
I suggest you add an interface function (buffer_block_lrus()?) and arrange for an empty inlined stub version when CONFIG_BLOCK=n.
Quoted 11 lines
..
mm/migrate.c +2
-0@@ -64,6 +64,19 @@*/ void migrate_prep (void ) { + bh_migration_done = false; +
mm/migrate.c line 78
Quoted 10 lines
mm/migrate.c +10
-0+/* +* This barrier ensures that callers of bh_lru_install() between +* the barrier and the call to invalidate_bh_lrus() read +* bh_migration_done() as false. +*/ +/* +* TODO: Remove me? lru_add_drain_all() already has an smp_mb(), +* but it would be good to ensure that the barrier isn't forgotten. +*/ +smp_mb ();
This stuff can be taken care of over in buffer_block_lrus() in fs/buffer.c.
Quoted 11 lines
mm/swap.c +1
-0@@ -36,6 +36,7 @@#include <linux/hugetlb.h> #include <linux/page_idle.h> #include <linux/local_lock.h> +#include <linux/buffer_head.h> #include "internal.h"
mm/swap.c void lru_add_drain_all(void) · line 764
Quoted 5 lines
mm/swap.c +2
-0@@ -759,6 +760,8 @@ void lru_add_drain_all(void)if (WARN_ON (!mm_percpu_wq))return ; +invalidate_bh_lrus (); +
Add a comment explaining why we're doing this? Mention that bn_lru buffers can pin pages, preventing migration.
Re: [PATCH] [RFC] mm: fs: Invalidate BH LRU during page migration
In reply to Chris GoldsworthyQuoted 5 lines · Chris Goldsworthy
Pages containing
buffer_headsthat are in thebuffer_headLRU cache will be pinned and thus cannot be migrated. Correspondingly, invalidate the BH LRU before a migration starts and stop anybuffer_headfrom being cached in the LRU, until migration has finished.
Thanks for the work, Chris. I have a few of comments below.
Quoted 18 lines
Signed-off-by Chris Goldsworthy <[email protected]>Cc Minchan Kim <[email protected]>Cc Matthew Wilcox <[email protected]>--- fs/buffer.c | 6 ++++++ include/linux/buffer_head.h | 3 +++ include/linux/migrate.h | 2 ++ mm/migrate.c | 18 ++++++++++++++++++ mm/page_alloc.c | 3 +++ mm/swap.c | 3 +++ 6 files changed, 35 insertions(+)fs/buffer.c +0
-0
fs/buffer.c static inline void check_irqs_on(void) · line 1292
Quoted 4 lines
fs/buffer.c +1
-0@@ -1289,6 +1289,8 @@ static inline void check_irqs_on(void)#endif } +bool bh_migration_done = true;
How about "bh_lru_disable"?
fs/buffer.c static void bh_lru_install(struct buffer_head *bh) · line 1310
Quoted 11 lines
fs/buffer.c +4
-0+/* * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is * inserted at the front, and the buffer_head at the back if any is evicted. @@ -1303,6 +1305,9 @@ static void bh_lru_install(struct buffer_head *bh)check_irqs_on ();bh_lru_lock (); +if (!bh_migration_done) + goto out; +
Let's add why we want it in the description in bh_lru_install's description.
Quoted 108 lines
fs/buffer.c +1
-0b =this_cpu_ptr (&bh_lrus);for (i =0 ; i <BH_LRU_SIZE ; i++) {swap (evictee, b->bhs[i]);@@ -1313,6 +1318,7 @@ static void bh_lru_install(struct buffer_head *bh) }get_bh (bh); +out:bh_lru_unlock ();brelse (evictee); }include/linux/buffer_head.h +3
-0@@ -193,6 +193,9 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size, gfp_t gfp);struct buffer_head *__bread_gfp (struct block_device *,sector_t block,unsigned size,gfp_t gfp); + +extern bool bh_migration_done; +void invalidate_bh_lrus (void );struct buffer_head *alloc_buffer_head (gfp_t gfp_flags);void free_buffer_head (struct buffer_head * bh);include/linux/migrate.h +2
-0@@ -46,6 +46,7 @@ extern int isolate_movable_page(struct page *page, isolate_mode_t mode);extern void putback_movable_page (struct page *page);extern void migrate_prep (void ); +extern void migrate_finish (void );extern void migrate_prep_local (void );extern void migrate_page_states (struct page *newpage,struct page *page);extern void migrate_page_copy (struct page *newpage,struct page *page);@@ -67,6 +68,7 @@ static inline int isolate_movable_page(struct page *page, isolate_mode_t mode) {return -EBUSY ; }static inline int migrate_prep (void ) {return -ENOSYS ; } +static inline int migrate_finish (void ) {return -ENOSYS ; }static inline int migrate_prep_local (void ) {return -ENOSYS ; }static inline void migrate_page_states (struct page *newpage,struct page *page)mm/migrate.c +18
-0@@ -64,6 +64,19 @@*/ void migrate_prep (void ) { + bh_migration_done = false; + +/* +* This barrier ensures that callers of bh_lru_install() between +* the barrier and the call to invalidate_bh_lrus() read +* bh_migration_done() as false. +*/ +/* +* TODO: Remove me? lru_add_drain_all() already has an smp_mb(), +* but it would be good to ensure that the barrier isn't forgotten. +*/ +smp_mb (); +/* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have @@ -73,6 +86,11 @@ void migrate_prep(void)lru_add_drain_all (); } +void migrate_finish (void ) +{ + bh_migration_done = true; +} +/* Do the necessary work of migrate_prep but not if it involves other CPUs */ void migrate_prep_local (void ) {mm/page_alloc.c +3
-0@@ -8493,6 +8493,9 @@ static int __alloc_contig_migrate_range(struct compact_control *cc, ret =migrate_pages (&cc->migratepages, alloc_migration_target,NULL , (unsigned long )&mtc, cc->mode,MR_CONTIG_RANGE ); } + +migrate_finish (); +if (ret <0 ) {putback_movable_pages (&cc->migratepages);return ret;mm/swap.c +1
-0@@ -36,6 +36,7 @@#include <linux/hugetlb.h> #include <linux/page_idle.h> #include <linux/local_lock.h> +#include <linux/buffer_head.h> #include "internal.h"
mm/swap.c void lru_add_drain_all(void) · line 763
Quoted 4 lines
mm/swap.c +1
-0@@ -759,6 +760,8 @@ void lru_add_drain_all(void)if (WARN_ON (!mm_percpu_wq))return ; +invalidate_bh_lrus ();
Instead of adding a new IPI there, how about adding need_bh_lru_drain(cpu) in lru_add_drain_all and then calls invalidate_bh_lru in lru_add_drain_cpu? Not a strong but looks like more harmonized with existing LRU draining code.
Thanks for the work.
Re: [PATCH] [RFC] mm: fs: Invalidate BH LRU during page migration
In reply to Chris Goldsworthyfs/buffer.c static inline void check_irqs_on(void) · line 1292
Quoted 5 lines · Chris Goldsworthy
fs/buffer.c +1
-0@@ -1289,6 +1289,8 @@ static inline void check_irqs_on(void)#endif } +bool bh_migration_done = true;
What protects this global variable? Or is there some subtle reason it doesn't need protection, in which case, please put that in a comment.