Skip to contentSkip to conversation

mm: fs: invalidate bh_lrus for only cold path

PATCH In review Minchan Kim 3 participants 6 messages Sep 7, 2021 – Sep 21, 2021

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 mm: fs: invalidate bh_lrus for only cold path

kernel test robot reported the regression of fio.write_iops[1] with [2].

Since lru_add_drain is called frequently, invalidate bh_lrus there could increase bh_lrus cache miss ratio, which needs more IO in the end.

This patch moves the bh_lrus invalidation from the hot path( e.g., zap_page_range, pagevec_release) to cold path(i.e., lru_add_drain_all, lru_cache_disable).

[1] https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/
[2] 8cc621d2f45d, mm: fs: invalidate BH LRU during page migration

---
* v2: https://lore.kernel.org/lkml/20210601145425.1396981-1-minchan(a)kernel.org/
* v1: https://lore.kernel.org/lkml/YK0oQ76zX0uVZCwQ(a)google.com/

 fs/buffer.c                 |  8 ++++++--
 include/linux/buffer_head.h |  4 ++--
 mm/swap.c                   | 19 ++++++++++++++++---
 3 files changed, 24 insertions(+), 7 deletions(-)
fs/buffer.c +6 -2
index ab7573d72dd7..c615387aedca 100644
@@ -1425,12 +1425,16 @@ void invalidate_bh_lrus(void)
 }
 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
 
-void invalidate_bh_lrus_cpu(int cpu)
+/*
+ * It's called from workqueue context so we need a bh_lru_lock to close
+ * the race with preemption/irq.
+ */
+void invalidate_bh_lrus_cpu(void)
 {
 	struct bh_lru *b;
 
 	bh_lru_lock();
-	b = per_cpu_ptr(&bh_lrus, cpu);
+	b = this_cpu_ptr(&bh_lrus);
 	__invalidate_bh_lrus(b);
 	bh_lru_unlock();
 }
include/linux/buffer_head.h +2 -2
index 6486d3c19463..36f33685c8c0 100644
@@ -194,7 +194,7 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
 struct buffer_head *__bread_gfp(struct block_device *,
 				sector_t block, unsigned size, gfp_t gfp);
 void invalidate_bh_lrus(void);
-void invalidate_bh_lrus_cpu(int cpu);
+void invalidate_bh_lrus_cpu(void);
 bool has_bh_in_lru(int cpu, void *dummy);
 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
 void free_buffer_head(struct buffer_head * bh);
@@ -408,7 +408,7 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }
 static inline void invalidate_inode_buffers(struct inode *inode) {}
 static inline int remove_inode_buffers(struct inode *inode) { return 1; }
 static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
-static inline void invalidate_bh_lrus_cpu(int cpu) {}
+static inline void invalidate_bh_lrus_cpu(void) {}
 static inline bool has_bh_in_lru(int cpu, void *dummy) { return false; }
 #define buffer_heads_over_limit 0
 
mm/swap.c +16 -3
index 897200d27dd0..af3cad4e5378 100644
@@ -620,7 +620,6 @@ void lru_add_drain_cpu(int cpu)
 		pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
 
 	activate_page_drain(cpu);
-	invalidate_bh_lrus_cpu(cpu);
 }
 
 /**
@@ -703,6 +702,20 @@ void lru_add_drain(void)
 	local_unlock(&lru_pvecs.lock);
 }
 
+/*
+ * It's called from per-cpu workqueue context in SMP case so
+ * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
+ * the same cpu. It shouldn't be a problem in !SMP case since
+ * the core is only one and the locks will disable preemption.
+ */
+static void lru_add_and_bh_lrus_drain(void)
+{
+	local_lock(&lru_pvecs.lock);
+	lru_add_drain_cpu(smp_processor_id());
+	local_unlock(&lru_pvecs.lock);
+	invalidate_bh_lrus_cpu();
+}
+
 void lru_add_drain_cpu_zone(struct zone *zone)
 {
 	local_lock(&lru_pvecs.lock);
@@ -717,7 +730,7 @@ static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
 
 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 {
-	lru_add_drain();
+	lru_add_and_bh_lrus_drain();
 }
 
 /*
@@ -858,7 +871,7 @@ void lru_cache_disable(void)
 	 */
 	__lru_add_drain_all(true);
 #else
-	lru_add_drain();
+	lru_add_and_bh_lrus_drain();
 #endif
 }
 
Signature · 1 line
2.33.0.309.g3052b89438-goog
#1 lore raw

Andrew, Could you take a look?

Quoted 123 lines · Minchan Kim

kernel test robot reported the regression of fio.write_iops[1] with [2].

Since lru_add_drain is called frequently, invalidate bh_lrus there could increase bh_lrus cache miss ratio, which needs more IO in the end.

This patch moves the bh_lrus invalidation from the hot path( e.g., zap_page_range, pagevec_release) to cold path(i.e., lru_add_drain_all, lru_cache_disable).

[1] https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/
[2] 8cc621d2f45d, mm: fs: invalidate BH LRU during page migration

---
* v2: https://lore.kernel.org/lkml/20210601145425.1396981-1-minchan(a)kernel.org/
* v1: https://lore.kernel.org/lkml/YK0oQ76zX0uVZCwQ(a)google.com/

 fs/buffer.c                 |  8 ++++++--
 include/linux/buffer_head.h |  4 ++--
 mm/swap.c                   | 19 ++++++++++++++++---
 3 files changed, 24 insertions(+), 7 deletions(-)
fs/buffer.c +6 -2
index ab7573d72dd7..c615387aedca 100644
@@ -1425,12 +1425,16 @@ void invalidate_bh_lrus(void)
 }
 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
 
-void invalidate_bh_lrus_cpu(int cpu)
+/*
+ * It's called from workqueue context so we need a bh_lru_lock to close
+ * the race with preemption/irq.
+ */
+void invalidate_bh_lrus_cpu(void)
 {
 	struct bh_lru *b;
 
 	bh_lru_lock();
-	b = per_cpu_ptr(&bh_lrus, cpu);
+	b = this_cpu_ptr(&bh_lrus);
 	__invalidate_bh_lrus(b);
 	bh_lru_unlock();
 }
include/linux/buffer_head.h +2 -2
index 6486d3c19463..36f33685c8c0 100644
@@ -194,7 +194,7 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
 struct buffer_head *__bread_gfp(struct block_device *,
 				sector_t block, unsigned size, gfp_t gfp);
 void invalidate_bh_lrus(void);
-void invalidate_bh_lrus_cpu(int cpu);
+void invalidate_bh_lrus_cpu(void);
 bool has_bh_in_lru(int cpu, void *dummy);
 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
 void free_buffer_head(struct buffer_head * bh);
@@ -408,7 +408,7 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }
 static inline void invalidate_inode_buffers(struct inode *inode) {}
 static inline int remove_inode_buffers(struct inode *inode) { return 1; }
 static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
-static inline void invalidate_bh_lrus_cpu(int cpu) {}
+static inline void invalidate_bh_lrus_cpu(void) {}
 static inline bool has_bh_in_lru(int cpu, void *dummy) { return false; }
 #define buffer_heads_over_limit 0
 
mm/swap.c +16 -3
index 897200d27dd0..af3cad4e5378 100644
@@ -620,7 +620,6 @@ void lru_add_drain_cpu(int cpu)
 		pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
 
 	activate_page_drain(cpu);
-	invalidate_bh_lrus_cpu(cpu);
 }
 
 /**
@@ -703,6 +702,20 @@ void lru_add_drain(void)
 	local_unlock(&lru_pvecs.lock);
 }
 
+/*
+ * It's called from per-cpu workqueue context in SMP case so
+ * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
+ * the same cpu. It shouldn't be a problem in !SMP case since
+ * the core is only one and the locks will disable preemption.
+ */
+static void lru_add_and_bh_lrus_drain(void)
+{
+	local_lock(&lru_pvecs.lock);
+	lru_add_drain_cpu(smp_processor_id());
+	local_unlock(&lru_pvecs.lock);
+	invalidate_bh_lrus_cpu();
+}
+
 void lru_add_drain_cpu_zone(struct zone *zone)
 {
 	local_lock(&lru_pvecs.lock);
@@ -717,7 +730,7 @@ static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
 
 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 {
-	lru_add_drain();
+	lru_add_and_bh_lrus_drain();
 }
 
 /*
@@ -858,7 +871,7 @@ void lru_cache_disable(void)
 	 */
 	__lru_add_drain_all(true);
 #else
-	lru_add_drain();
+	lru_add_and_bh_lrus_drain();
 #endif
 }
 
Signature · 1 line
2.33.0.309.g3052b89438-goog
#2 lore raw
In reply to Minchan Kim
Quoted 11 lines · Minchan Kim

kernel test robot reported the regression of fio.write_iops[1] with [2].

Since lru_add_drain is called frequently, invalidate bh_lrus there could increase bh_lrus cache miss ratio, which needs more IO in the end.

This patch moves the bh_lrus invalidation from the hot path( e.g., zap_page_range, pagevec_release) to cold path(i.e., lru_add_drain_all, lru_cache_disable).

Was this confirmed to fix the regression?

I only see the "tested with 5.14" that the regression was still there

https://lore.kernel.org/lkml/034fc860-d0d0-0c61-09d2-3c41c4f020c6(a)intel.com/

I don't see a confirmation that this patch fixed it.

It looks likely, but if you have the confirmation somewhere, it would help to link that too.

Or did I miss it?

Linus

#3 lore raw
Quoted 17 lines · Linus Torvalds
Quoted 11 lines · Minchan Kim

kernel test robot reported the regression of fio.write_iops[1] with [2].

Since lru_add_drain is called frequently, invalidate bh_lrus there could increase bh_lrus cache miss ratio, which needs more IO in the end.

This patch moves the bh_lrus invalidation from the hot path( e.g., zap_page_range, pagevec_release) to cold path(i.e., lru_add_drain_all, lru_cache_disable).

Was this confirmed to fix the regression?

I only see the "tested with 5.14" that the regression was still there

Quoted 8 lines
   https://lore.kernel.org/lkml/034fc860-d0d0-0c61-09d2-3c41c4f020c6(a)intel.com/

I don't see a confirmation that this patch fixed it.

It looks likely, but if you have the confirmation somewhere, it would help to link that too.

Or did I miss it?

I have no idea why I couldn't find the reply from lore.kernel.org/lkml/
The message id was 89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com
in the thread(https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/)

Xing, Zhengjun zhengjun.xing(a)intel.com confirmed

Quote -

```
Hi Minchan,

...

I test the patch, the regression reduced to -2.9%.
```

Since then, I modified the patch a little more and couldn't find the message from the public URL. Thus, do we need to confirm it again against latest kernel?

Xing, could you confirm whether this patch fixes the regression?

#4 lore raw
Quoted 4 lines · Minchan Kim

I have no idea why I couldn't find the reply from lore.kernel.org/lkml/
The message id was 89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com
in the thread(https://lore.kernel.org/lkml/20210520083144.GD14190(a)xsang-OptiPlex-9020/)

Hmm. That message-id isn't found anywhere.

Maybe it was sent just to you personally?

[ Goes off and looks ]

Nope, I can see it in my mails too, and yeah, I see

To: Minchan Kim <[email protected]>, kernel test robot
<[email protected]>
Cc: Linus Torvalds <[email protected]>, Chris
Goldsworthy <[email protected]>, Laura Abbott <[email protected]>, David Hildenbrand <[email protected]>, John Dias <[email protected]>, Matthew Wilcox <[email protected]>, Michal Hocko <[email protected]>, Suren Baghdasaryan <[email protected]>, Vlastimil Babka <[email protected]>, Andrew Morton <[email protected]>, LKML <[email protected]>, lkp(a)lists.01.org, lkp(a)intel.com
References: <20210520083144.GD14190@xsang-OptiPlex-9020>
<[email protected]>
From: "Xing, Zhengjun" <[email protected]>
Message-ID: <[email protected]>

but lore has no idea about it:

https://lore.kernel.org/all/89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com/

just says "Message-ID <[email protected]> not found".

Strange.

Something presumably caused the list to drop that email.

Linus

#5 lore raw
Quoted 25 lines · Linus Torvalds

Hmm. That message-id isn't found anywhere.

Maybe it was sent just to you personally?

[ Goes off and looks ]

Nope, I can see it in my mails too, and yeah, I see

To: Minchan Kim <[email protected]>, kernel test robot
<[email protected]>
Cc: Linus Torvalds <[email protected]>, Chris
Goldsworthy <[email protected]>, Laura Abbott <[email protected]>, David Hildenbrand <[email protected]>, John Dias <[email protected]>, Matthew Wilcox <[email protected]>, Michal Hocko <[email protected]>, Suren Baghdasaryan <[email protected]>, Vlastimil Babka <[email protected]>, Andrew Morton <[email protected]>, LKML <[email protected]>, lkp(a)lists.01.org, lkp(a)intel.com
References: <20210520083144.GD14190@xsang-OptiPlex-9020>
<[email protected]>
From: "Xing, Zhengjun" <[email protected]>
Message-ID: <[email protected]>

but lore has no idea about it:

Quoted 8 lines

https://lore.kernel.org/all/89e2b66b-c706-c020-bff5-b815dcd5c461(a)intel.com/

just says "Message-ID <[email protected]> not found".

Strange.

Something presumably caused the list to drop that email.

I think vger is backed up. I've flagged this with John, hopefully he'll be able to figure out what's keeping things from hitting the archives.

-K