mm: fs: invalidate bh_lrus for only cold path
Subsystems (2)
PATCH v3 1/1 mm: fs: invalidate bh_lrus for only cold path
kernel test robot reported the regression of fio.write_iops[1] with [2].
Since lru_add_drain is called frequently, invalidate bh_lrus there could increase bh_lrus cache miss ratio, which needs more IO in the end.
This patch moves the bh_lrus invalidation from the hot path( e.g., zap_page_range, pagevec_release) to cold path(i.e., lru_add_drain_all, lru_cache_disable).
[1] https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/
[2] 8cc621d2f45d, mm: fs: invalidate BH LRU during page migration
---
* v2: https://lore.kernel.org/lkml/20210601145425.1396981-1-minchan(a)kernel.org/
* v1: https://lore.kernel.org/lkml/YK0oQ76zX0uVZCwQ(a)google.com/
fs/buffer.c | 8 ++++++-- include/linux/buffer_head.h | 4 ++-- mm/swap.c | 19 ++++++++++++++++--- 3 files changed, 24 insertions(+), 7 deletions(-)
fs/buffer.c +6 -2
@@ -1425,12 +1425,16 @@ void invalidate_bh_lrus(void) }EXPORT_SYMBOL_GPL (invalidate_bh_lrus); -void invalidate_bh_lrus_cpu (int cpu) +/* +* It's called from workqueue context so we need a bh_lru_lock to close +* the race with preemption/irq. +*/ +void invalidate_bh_lrus_cpu (void ) {struct bh_lru *b;bh_lru_lock (); - b =per_cpu_ptr (&bh_lrus, cpu); + b =this_cpu_ptr (&bh_lrus);__invalidate_bh_lrus (b);bh_lru_unlock (); }
include/linux/buffer_head.h +2 -2
@@ -194,7 +194,7 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,struct buffer_head *__bread_gfp (struct block_device *,sector_t block,unsigned size,gfp_t gfp);void invalidate_bh_lrus (void ); -void invalidate_bh_lrus_cpu (int cpu); +void invalidate_bh_lrus_cpu (void );bool has_bh_in_lru (int cpu,void *dummy);struct buffer_head *alloc_buffer_head (gfp_t gfp_flags);void free_buffer_head (struct buffer_head * bh);
@@ -408,7 +408,7 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }static inline void invalidate_inode_buffers (struct inode *inode) {}static inline int remove_inode_buffers (struct inode *inode) {return 1 ; }static inline int sync_mapping_buffers (struct address_space *mapping) {return 0 ; } -static inline void invalidate_bh_lrus_cpu (int cpu) {} +static inline void invalidate_bh_lrus_cpu (void ) {}static inline bool has_bh_in_lru (int cpu,void *dummy) {return false; }#define buffer_heads_over_limit 0
mm/swap.c +16 -3
@@ -620,7 +620,6 @@ void lru_add_drain_cpu(int cpu)pagevec_lru_move_fn (pvec, lru_lazyfree_fn);activate_page_drain (cpu); -invalidate_bh_lrus_cpu (cpu); }/**
@@ -703,6 +702,20 @@ void lru_add_drain(void)local_unlock (&lru_pvecs.lock); } +/* +* It's called from per-cpu workqueue context in SMP case so +* lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on +* the same cpu. It shouldn't be a problem in !SMP case since +* the core is only one and the locks will disable preemption. +*/ +static void lru_add_and_bh_lrus_drain (void ) +{ +local_lock (&lru_pvecs.lock); +lru_add_drain_cpu (smp_processor_id ()); +local_unlock (&lru_pvecs.lock); +invalidate_bh_lrus_cpu (); +} +void lru_add_drain_cpu_zone (struct zone *zone) {local_lock (&lru_pvecs.lock);
@@ -717,7 +730,7 @@ static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);static void lru_add_drain_per_cpu (struct work_struct *dummy) { -lru_add_drain (); +lru_add_and_bh_lrus_drain (); }/*
@@ -858,7 +871,7 @@ void lru_cache_disable(void)*/ __lru_add_drain_all (true);#else -lru_add_drain (); +lru_add_and_bh_lrus_drain ();#endif }
Signature · 1 line
2.33.0.309.g3052b89438-goog
Andrew, Could you take a look?
Quoted 123 lines · Minchan Kim
kernel test robot reported the regression of fio.write_iops[1] with [2].
Since
lru_add_drainis called frequently, invalidatebh_lrusthere could increasebh_lruscache miss ratio, which needs more IO in the end.This patch moves the
bh_lrusinvalidation from the hotpath( e.g., zap_page_range, pagevec_release)to coldpath(i.e., lru_add_drain_all, lru_cache_disable).[1] https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/
[2]8cc621d2f45d, mm: fs: invalidate BH LRU during page migrationReviewed-by Chris Goldsworthy <[email protected]>Reported-by kernel test robot <[email protected]>Signed-off-by Minchan Kim <[email protected]>---
* v2: https://lore.kernel.org/lkml/20210601145425.1396981-1-minchan(a)kernel.org/
* v1: https://lore.kernel.org/lkml/YK0oQ76zX0uVZCwQ(a)google.com/fs/buffer.c | 8 ++++++-- include/linux/buffer_head.h | 4 ++-- mm/swap.c | 19 ++++++++++++++++--- 3 files changed, 24 insertions(+), 7 deletions(-)fs/buffer.c +6
-2@@ -1425,12 +1425,16 @@ void invalidate_bh_lrus(void) }EXPORT_SYMBOL_GPL (invalidate_bh_lrus); -void invalidate_bh_lrus_cpu (int cpu) +/* +* It's called from workqueue context so we need a bh_lru_lock to close +* the race with preemption/irq. +*/ +void invalidate_bh_lrus_cpu (void ) {struct bh_lru *b;bh_lru_lock (); - b =per_cpu_ptr (&bh_lrus, cpu); + b =this_cpu_ptr (&bh_lrus);__invalidate_bh_lrus (b);bh_lru_unlock (); }include/linux/buffer_head.h +2
-2@@ -194,7 +194,7 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,struct buffer_head *__bread_gfp (struct block_device *,sector_t block,unsigned size,gfp_t gfp);void invalidate_bh_lrus (void ); -void invalidate_bh_lrus_cpu (int cpu); +void invalidate_bh_lrus_cpu (void );bool has_bh_in_lru (int cpu,void *dummy);struct buffer_head *alloc_buffer_head (gfp_t gfp_flags);void free_buffer_head (struct buffer_head * bh);@@ -408,7 +408,7 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }static inline void invalidate_inode_buffers (struct inode *inode) {}static inline int remove_inode_buffers (struct inode *inode) {return 1 ; }static inline int sync_mapping_buffers (struct address_space *mapping) {return 0 ; } -static inline void invalidate_bh_lrus_cpu (int cpu) {} +static inline void invalidate_bh_lrus_cpu (void ) {}static inline bool has_bh_in_lru (int cpu,void *dummy) {return false; }#define buffer_heads_over_limit 0mm/swap.c +16
-3@@ -620,7 +620,6 @@ void lru_add_drain_cpu(int cpu)pagevec_lru_move_fn (pvec, lru_lazyfree_fn);activate_page_drain (cpu); -invalidate_bh_lrus_cpu (cpu); }/** @@ -703,6 +702,20 @@ void lru_add_drain(void)local_unlock (&lru_pvecs.lock); } +/* +* It's called from per-cpu workqueue context in SMP case so +* lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on +* the same cpu. It shouldn't be a problem in !SMP case since +* the core is only one and the locks will disable preemption. +*/ +static void lru_add_and_bh_lrus_drain (void ) +{ +local_lock (&lru_pvecs.lock); +lru_add_drain_cpu (smp_processor_id ()); +local_unlock (&lru_pvecs.lock); +invalidate_bh_lrus_cpu (); +} +void lru_add_drain_cpu_zone (struct zone *zone) {local_lock (&lru_pvecs.lock);@@ -717,7 +730,7 @@ static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);static void lru_add_drain_per_cpu (struct work_struct *dummy) { -lru_add_drain (); +lru_add_and_bh_lrus_drain (); }/* @@ -858,7 +871,7 @@ void lru_cache_disable(void)*/ __lru_add_drain_all (true);#else -lru_add_drain (); +lru_add_and_bh_lrus_drain ();#endif }Signature · 1 line
2.33.0.309.g3052b89438-goog
Quoted 11 lines · Minchan Kim
kernel test robot reported the regression of fio.write_iops[1] with [2].
Since
lru_add_drainis called frequently, invalidatebh_lrusthere could increasebh_lruscache miss ratio, which needs more IO in the end.This patch moves the
bh_lrusinvalidation from the hotpath( e.g., zap_page_range, pagevec_release)to coldpath(i.e., lru_add_drain_all, lru_cache_disable).
Was this confirmed to fix the regression?
I only see the "tested with 5.14" that the regression was still there
https://lore.kernel.org/lkml/034fc860-d0d0-0c61-09d2-3c41c4f020c6(a)intel.com/
I don't see a confirmation that this patch fixed it.
It looks likely, but if you have the confirmation somewhere, it would help to link that too.
Or did I miss it?
Linus
Quoted 17 lines · Linus Torvalds
Quoted 11 lines · Minchan Kim
kernel test robot reported the regression of fio.write_iops[1] with [2].
Since
lru_add_drainis called frequently, invalidatebh_lrusthere could increasebh_lruscache miss ratio, which needs more IO in the end.This patch moves the
bh_lrusinvalidation from the hotpath( e.g., zap_page_range, pagevec_release)to coldpath(i.e., lru_add_drain_all, lru_cache_disable).Was this confirmed to fix the regression?
I only see the "tested with 5.14" that the regression was still there
Quoted 8 lines
https://lore.kernel.org/lkml/034fc860-d0d0-0c61-09d2-3c41c4f020c6(a)intel.com/I don't see a confirmation that this patch fixed it.
It looks likely, but if you have the confirmation somewhere, it would help to link that too.
Or did I miss it?
I have no idea why I couldn't find the reply from lore.kernel.org/lkml/
The message id was 89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com
in the thread(https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/)
Xing, Zhengjun zhengjun.xing(a)intel.com confirmed
Quote -
``` Hi Minchan, ... I test the patch, the regression reduced to -2.9%. ```
Since then, I modified the patch a little more and couldn't find the message from the public URL. Thus, do we need to confirm it again against latest kernel?
Xing, could you confirm whether this patch fixes the regression?
Quoted 4 lines · Minchan Kim
I have no idea why I couldn't find the reply from lore.kernel.org/lkml/
The message id was 89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com
in the thread(https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/)
Hmm. That message-id isn't found anywhere.
Maybe it was sent just to you personally?
[ Goes off and looks ]
Nope, I can see it in my mails too, and yeah, I see
To: Minchan Kim <[email protected]>, kernel test robot
<[email protected]>
Cc: Linus Torvalds <[email protected]>, Chris
Goldsworthy <[email protected]>, Laura Abbott <[email protected]>, David Hildenbrand <[email protected]>, John Dias <[email protected]>, Matthew Wilcox <[email protected]>, Michal Hocko <[email protected]>, Suren Baghdasaryan <[email protected]>, Vlastimil Babka <[email protected]>, Andrew Morton <[email protected]>, LKML <[email protected]>, lkp(a)lists.01.org, lkp(a)intel.com
References: <20210520083144.GD14190@xsang-OptiPlex-9020>
<[email protected]>
From: "Xing, Zhengjun" <[email protected]>
Message-ID: <[email protected]>
but lore has no idea about it:
https://lore.kernel.org/all/89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com/
just says "Message-ID <[email protected]> not found".
Strange.
Something presumably caused the list to drop that email.
Linus
Quoted 25 lines · Linus Torvalds
Hmm. That message-id isn't found anywhere.
Maybe it was sent just to you personally?
[ Goes off and looks ]
Nope, I can see it in my mails too, and yeah, I see
To: Minchan Kim <[email protected]>, kernel test robot
<[email protected]>
Cc: Linus Torvalds <[email protected]>, Chris
Goldsworthy <[email protected]>, Laura Abbott <[email protected]>, David Hildenbrand <[email protected]>, John Dias <[email protected]>, Matthew Wilcox <[email protected]>, Michal Hocko <[email protected]>, Suren Baghdasaryan <[email protected]>, Vlastimil Babka <[email protected]>, Andrew Morton <[email protected]>, LKML <[email protected]>, lkp(a)lists.01.org, lkp(a)intel.com
References: <20210520083144.GD14190@xsang-OptiPlex-9020>
<[email protected]>
From: "Xing, Zhengjun" <[email protected]>
Message-ID: <[email protected]>but lore has no idea about it:
Quoted 8 lines
https://lore.kernel.org/all/89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com/
just says "Message-ID <[email protected]> not found".
Strange.
Something presumably caused the list to drop that email.
I think vger is backed up. I've flagged this with John, hopefully he'll be able to figure out what's keeping things from hitting the archives.
-K