fs/buffer.c: remove per-CPU buffer_head lookup cache
Subsystems (6)
PATCH 1/1 fs/buffer.c: remove per-CPU buffer_head lookup cache
For certain types of applications (for example PLC software or RAN processing), upon occurrence of an event, it is necessary to complete a certain task in a maximum amount of time (deadline).
One way to express this requirement is with a pair of numbers, deadline time and execution time, where:
* deadline time: length of time between event and deadline.
* execution time: length of time it takes for processing of event to occur on a particular hardware platform (uninterrupted).
The particular values depend on use-case. For the case where the realtime application executes in a virtualized guest, an IPI which must be serviced in the host will cause the following sequence of events:
1) VM-exit
2) execution of IPI (and function call)
3) VM-entry
Which causes an excess of 50us latency as observed by cyclictest (this violates the latency requirement of vRAN application with 1ms TTI, for example).
invalidate_bh_lrus calls an IPI on each CPU that has non empty per-CPU cache:
on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:
114 ns per __find_get_block
68 ns per __find_get_block_slow
So remove the per-CPU buffer_head caching.
Test program:
#define NRLOOPS 200000
static int __init example_init(void)
{
ktime_t s, e;
s64 delta;
int i, suc;
bdev = blkdev_get_by_path("/dev/loop0", FMODE_READ, NULL);
if (IS_ERR(bdev)) {
printk(KERN_ERR "failed to load /dev/loop0\n");
return -ENODEV;
}
suc = 0;
delta = 0;
for (i=0; i < NRLOOPS; i++) {
struct buffer_head *bh;
s = ktime_get();
bh = __find_get_block(bdev, 1, 512);
e = ktime_get();
if (bh) {
suc++;
__brelse(bh);
}
delta = delta + ktime_to_ns(ktime_sub(e, s));
}
printk(KERN_ERR "%lld ns per __find_get_block (suc=%d)\n", delta/NRLOOPS, suc);
suc = 0;
delta = 0;
for (i=0; i < NRLOOPS; i++) {
struct buffer_head *bh;
s = ktime_get();
bh = __find_get_block_slow(bdev, 1);
e = ktime_get();
if (bh) {
suc++;
__brelse(bh);
}
delta = delta + ktime_to_ns(ktime_sub(e, s));
}
printk(KERN_ERR "%lld ns per __find_get_block_slow (suc=%d)\n", delta/NRLOOPS, suc);
return 0;
}---
block/bdev.c | 2 - fs/buffer.c | 209 +++++------------------------------------------------------------------------------------------------------------------------------------------------------- fs/jbd2/revoke.c | 9 ++---- fs/mpage.c | 3 -- fs/ocfs2/journal.c | 3 -- fs/reiserfs/reiserfs.h | 2 - include/linux/buffer_head.h | 11 ++------ mm/migrate.c | 12 +------- mm/swap.c | 4 -- 9 files changed, 21 insertions(+), 234 deletions(-)
block/bdev.c +0 -2
@@ -72,7 +72,6 @@ static void kill_bdev(struct block_device *bdev)if (mapping_empty (mapping))return ; -invalidate_bh_lrus ();truncate_inode_pages (mapping,0 ); }
@@ -82,7 +81,6 @@ void invalidate_bdev(struct block_device *bdev)struct address_space *mapping = bdev->bd_inode->i_mapping;if (mapping->nrpages) { -invalidate_bh_lrus ();lru_add_drain_all ();/* make sure all lru add caches are flushed */ invalidate_mapping_pages (mapping,0 ,-1 ); }
fs/buffer.c +8 -201
264 lines not on this page · raw message
fs/jbd2/revoke.c +4 -5
@@ -345,7 +345,7 @@ int jbd2_journal_revoke(handle_t *handle, unsigned long long blocknr, bh = bh_in;if (!bh) { - bh =__find_get_block (bdev, blocknr, journal->j_blocksize); + bh =__find_get_block (bdev, blocknr);if (bh)BUFFER_TRACE (bh,"found on hash" ); }
@@ -355,7 +355,7 @@ int jbd2_journal_revoke(handle_t *handle, unsigned long long blocknr,/* If there is a different buffer_head lying around in * memory anywhere... */ - bh2 =__find_get_block (bdev, blocknr, journal->j_blocksize); + bh2 =__find_get_block (bdev, blocknr);if (bh2) {/* ... and it has RevokeValid status... */ if (bh2 != bh &&buffer_revokevalid (bh2))
@@ -466,7 +466,7 @@ int jbd2_journal_cancel_revoke(handle_t *handle, struct journal_head *jh)* state machine will get very upset later on. */ if (need_cancel) {struct buffer_head *bh2; - bh2 =__find_get_block (bh->b_bdev, bh->b_blocknr, bh->b_size); + bh2 =__find_get_block (bh->b_bdev, bh->b_blocknr);if (bh2) {if (bh2 != bh)clear_buffer_revoked (bh2);
@@ -496,8 +496,7 @@ void jbd2_clear_buffer_revoked_flags(journal_t *journal)struct buffer_head *bh; record = (struct jbd2_revoke_record_s *)list_entry; bh =__find_get_block (journal->j_fs_dev, - record->blocknr, - journal->j_blocksize); + record->blocknr);if (bh) {clear_buffer_revoked (bh);__brelse (bh);
fs/mpage.c +1 -2
@@ -634,8 +634,7 @@ static int __mpage_writepage(struct folio *folio, struct writeback_control *wbc,if (boundary || (first_unmapped != blocks_per_page)) { bio =mpage_bio_submit_write (bio);if (boundary_block) { -write_boundary_block (boundary_bdev, - boundary_block,1 << blkbits); +write_boundary_block (boundary_bdev, boundary_block); } }else { mpd->last_block_in_bio = blocks[blocks_per_page -1 ];
fs/ocfs2/journal.c +1 -2
@@ -1212,8 +1212,7 @@ static int ocfs2_force_read_journal(struct inode *inode) }for (i =0 ; i < p_blocks; i++, p_blkno++) { - bh =__find_get_block (osb->sb->s_bdev, p_blkno, - osb->sb->s_blocksize); + bh =__find_get_block (osb->sb->s_bdev, p_blkno);/* block not cached. */ if (!bh)continue ;
fs/reiserfs/reiserfs.h +1 -1
@@ -2810,7 +2810,7 @@ struct reiserfs_journal_header {#define journal_hash (t,sb,block) ((t)[_jhashfn((sb),(block)) & JBH_HASH_MASK])/* We need these to make journal.c code more readable */ -#define journal_find_get_block (s, block) __find_get_block(SB_JOURNAL(s)->j_dev_bd, block, s->s_blocksize) +#define journal_find_get_block (s, block) __find_get_block(SB_JOURNAL(s)->j_dev_bd, block)#define journal_getblk (s, block) __getblk(SB_JOURNAL(s)->j_dev_bd, block, s->s_blocksize)#define journal_bread (s, block) __bread(SB_JOURNAL(s)->j_dev_bd, block, s->s_blocksize)
include/linux/buffer_head.h +3 -8
@@ -227,8 +227,7 @@ static inline void clean_bdev_bh_alias(struct buffer_head *bh)void mark_buffer_async_write (struct buffer_head *bh);void __wait_on_buffer (struct buffer_head *);wait_queue_head_t *bh_waitq_head (struct buffer_head *bh); -struct buffer_head *__find_get_block (struct block_device *bdev,sector_t block, -unsigned size); +struct buffer_head *__find_get_block (struct block_device *bdev,sector_t block);struct buffer_head *__getblk_gfp (struct block_device *bdev,sector_t block,unsigned size,gfp_t gfp);void __brelse (struct buffer_head *);
@@ -236,9 +235,6 @@ void __bforget(struct buffer_head *);void __breadahead (struct block_device *,sector_t block,unsigned int size);struct buffer_head *__bread_gfp (struct block_device *,sector_t block,unsigned size,gfp_t gfp); -void invalidate_bh_lrus (void ); -void invalidate_bh_lrus_cpu (void ); -bool has_bh_in_lru (int cpu,void *dummy);struct buffer_head *alloc_buffer_head (gfp_t gfp_flags);void free_buffer_head (struct buffer_head * bh);void unlock_buffer (struct buffer_head *bh);
@@ -247,8 +243,7 @@ int sync_dirty_buffer(struct buffer_head *bh);int __sync_dirty_buffer (struct buffer_head *bh,blk_opf_t op_flags);void write_dirty_buffer (struct buffer_head *bh,blk_opf_t op_flags);void submit_bh (blk_opf_t ,struct buffer_head *); -void write_boundary_block (struct block_device *bdev, -sector_t bblock,unsigned blocksize); +void write_boundary_block (struct block_device *bdev,sector_t bblock);int bh_uptodate_or_lock (struct buffer_head *bh);int __bh_read (struct buffer_head *bh,blk_opf_t op_flags,bool wait); void __bh_read_batch(int nr, struct buffer_head *bhs[],
@@ -375,7 +370,7 @@ sb_getblk_gfp(struct super_block *sb, sector_t block, gfp_t gfp)static inline struct buffer_head *sb_find_get_block (struct super_block *sb,sector_t block) { -return __find_get_block (sb->s_bdev, block, sb->s_blocksize); +return __find_get_block (sb->s_bdev, block); } static inline void
mm/migrate.c +2 -10
@@ -747,9 +747,7 @@ static int __buffer_migrate_folio(struct address_space *mapping,if (check_refs) {bool busy; -bool invalidated = false; -recheck_buffers: busy = false;spin_lock (&mapping->private_lock); bh = head;
@@ -761,14 +759,8 @@ static int __buffer_migrate_folio(struct address_space *mapping, bh = bh->b_this_page; }while (bh != head);if (busy) { -if (invalidated) { - rc = -EAGAIN ; - goto unlock_buffers; - } -spin_unlock (&mapping->private_lock); -invalidate_bh_lrus (); - invalidated = true; - goto recheck_buffers; + rc = -EAGAIN ; + goto unlock_buffers; } }
mm/swap.c +1 -3
@@ -765,7 +765,6 @@ static void lru_add_and_bh_lrus_drain(void)local_lock (&cpu_fbatches.lock);lru_add_drain_cpu (smp_processor_id ());local_unlock (&cpu_fbatches.lock); -invalidate_bh_lrus_cpu ();mlock_drain_local (); }
@@ -798,8 +797,7 @@ static bool cpu_needs_drain(unsigned int cpu)folio_batch_count (&fbatches->lru_deactivate) ||folio_batch_count (&fbatches->lru_lazyfree) ||folio_batch_count (&fbatches->activate) || -need_mlock_drain (cpu) || -has_bh_in_lru (cpu,NULL ); +need_mlock_drain (cpu); }/*
Quoted 7 lines · Marcelo Tosatti
Upon closer investigation, it was found that in current codebase,
lookup_bh_lruis slower than__find_get_block_slow:114 ns per
__find_get_block
68 ns per__find_get_block_slowSo remove the per-CPU
buffer_headcaching.
LOL. That's amazing. I can't even see why it's so expensive. The local_irq_disable(), perhaps? Your test case is the best possible one for lookup_bh_lru() where you're not even doing the copy.
Quoted 47 lines · Marcelo Tosatti
Upon closer investigation, it was found that in current codebase,
lookup_bh_lruis slower than__find_get_block_slow:114 ns per
__find_get_block
68 ns per__find_get_block_slowSo remove the per-CPU
buffer_headcaching.Test program:
#define NRLOOPS 200000
static int__initexample_init(void){ ktime_t s, e; s64 delta; int i, suc; bdev = blkdev_get_by_path("/dev/loop0", FMODE_READ, NULL); if (IS_ERR(bdev)) { printk(KERN_ERR "failed to load /dev/loop0\n"); return -ENODEV; } suc = 0; delta = 0; for (i=0; i < NRLOOPS; i++) { struct buffer_head *bh; s = ktime_get(); bh = __find_get_block(bdev, 1, 512); e = ktime_get(); if (bh) { suc++; __brelse(bh); } delta = delta + ktime_to_ns(ktime_sub(e, s)); } printk(KERN_ERR "%lld ns per __find_get_block (suc=%d)\n", delta/NRLOOPS, suc); suc = 0; delta = 0; for (i=0; i < NRLOOPS; i++) { struct buffer_head *bh; s = ktime_get(); bh = __find_get_block_slow(bdev, 1);
Quoted 8 lines
e = ktime_get();
if (bh) {
suc++;
__brelse(bh);
}
delta = delta + ktime_to_ns(ktime_sub(e, s));
}
printk(KERN_ERR "%lld ns per __find_get_block_slow (suc=%d)\n", delta/NRLOOPS, suc);It occurs to me that this is close to being the best-case scenario for page-cache lookup as well as for lru lookup. Can you re-run it with block 4UL * 1024 * 1024 * 1024 instead of block 1?
Quoted 4 lines · Matthew Wilcox
Quoted 3 lines · Marcelo Tosatti
Upon closer investigation, it was found that in current codebase,
lookup_bh_lruis slower than__find_get_block_slow:
Quoted 8 lines
Quoted 4 lines
114 ns per
__find_get_block
68 ns per__find_get_block_slowSo remove the per-CPU
buffer_headcaching.LOL. That's amazing. I can't even see why it's so expensive. The
local_irq_disable(), perhaps? Your test case is the best possible one forlookup_bh_lru()where you're not even doing the copy.
I think it's even simpler than that.
i.e. the lookaside cache is being missed, so it's a pure cost and the code is always having to call __find_get_block_slow() anyway. Peeking at 16 buffers to not find a match is just as expensive as walking 3-4 tree levels in an Xarray to find the buffer in the first place....
IMO, this is an example of how lookaside caches are only a benefit if the working set of items largely fits in the lookaside cache and the cache lookup itself is much, much slower than a lookaside cache miss.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Quoted 10 lines · Dave Chinner
Quoted 9 lines · Matthew Wilcox
Quoted 7 lines · Marcelo Tosatti
Upon closer investigation, it was found that in current codebase,
lookup_bh_lruis slower than__find_get_block_slow:114 ns per
__find_get_block
68 ns per__find_get_block_slowSo remove the per-CPU
buffer_headcaching.
Quoted 8 lines
Quoted 3 lines
LOL. That's amazing. I can't even see why it's so expensive. The
local_irq_disable(), perhaps? Your test case is the best possible one forlookup_bh_lru()where you're not even doing the copy.I think it's even simpler than that.
i.e. the lookaside cache is being missed, so it's a pure cost and the code is always having to call
__find_get_block_slow()anyway.
How does that happen?
__find_get_block(struct block_device *bdev, sector_t block, unsigned size)
{
struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
if (bh == NULL) {
/* __find_get_block_slow will mark the page accessed */
bh = __find_get_block_slow(bdev, block);
if (bh)
bh_lru_install(bh);The second (and all subsequent) calls to __find_get_block() should find the BH in the LRU.
Quoted 4 lines
IMO, this is an example of how lookaside caches are only a benefit if the working set of items largely fits in the lookaside cache and the cache lookup itself is much, much slower than a lookaside cache miss.
But the test code he posted always asks for the same buffer each time. So it should find it in the lookaside cache?
Quoted 35 lines · Matthew Wilcox
Quoted 10 lines · Dave Chinner
Quoted 9 lines · Matthew Wilcox
Quoted 7 lines · Marcelo Tosatti
Upon closer investigation, it was found that in current codebase,
lookup_bh_lruis slower than__find_get_block_slow:114 ns per
__find_get_block
68 ns per__find_get_block_slowSo remove the per-CPU
buffer_headcaching.Quoted 8 lines
Quoted 3 lines
LOL. That's amazing. I can't even see why it's so expensive. The
local_irq_disable(), perhaps? Your test case is the best possible one forlookup_bh_lru()where you're not even doing the copy.I think it's even simpler than that.
i.e. the lookaside cache is being missed, so it's a pure cost and the code is always having to call
__find_get_block_slow()anyway.How does that happen?
__find_get_block(struct block_device *bdev, sector_t block, unsigned size){ struct buffer_head *bh = lookup_bh_lru(bdev, block, size); if (bh == NULL) { /* __find_get_block_slow will mark the page accessed */ bh = __find_get_block_slow(bdev, block); if (bh) bh_lru_install(bh);The second (and all subsequent) calls to
__find_get_block()should find the BH in the LRU.
Quoted 7 lines
Quoted 4 lines
IMO, this is an example of how lookaside caches are only a benefit if the working set of items largely fits in the lookaside cache and the cache lookup itself is much, much slower than a lookaside cache miss.
But the test code he posted always asks for the same buffer each time. So it should find it in the lookaside cache?
Oh.
for (i = 0; ....) {
bh = __find_get_block(bdev, 1, 512);That's a '1' being passed to __find_get_block, not 'i'.
/me goes and gets more coffee.
Maybe it's CONFIG_PREEMPT_RT=y doing something to the locks that isn't obvious here...
-Dave.
Signature · 2 lines
Dave Chinner [email protected]
Nice one!
Quoted 4 lines · Matthew Wilcox
Quoted 3 lines · Marcelo Tosatti
Upon closer investigation, it was found that in current codebase,
lookup_bh_lruis slower than__find_get_block_slow:
Quoted 8 lines
Quoted 4 lines
114 ns per
__find_get_block
68 ns per__find_get_block_slowSo remove the per-CPU
buffer_headcaching.LOL. That's amazing. I can't even see why it's so expensive. The
local_irq_disable(), perhaps? Your test case is the best possible one forlookup_bh_lru()where you're not even doing the copy.
Oops, that was due to incorrect buffer size being looked up versus installed size.
About 15ns is due to irq disablement.
6ns due to checking preempt is disabled (from __this_cpu_read).
So the actual numbers for the single block lookup are (searching for the same block number repeatedly):
42 ns per __find_get_block
68 ns per __find_get_block_slow
And increases linearly as the test increases the number of blocks which are searched for:
say mod 3 is __find_get_block(blocknr=1) __find_get_block(blocknr=2) __find_get_block(blocknr=3)
41 ns per __find_get_block mod=1 41 ns per __find_get_block mod=2 42 ns per __find_get_block mod=3 43 ns per __find_get_block mod=4 45 ns per __find_get_block mod=5 48 ns per __find_get_block mod=6 48 ns per __find_get_block mod=7 49 ns per __find_get_block mod=8 51 ns per __find_get_block mod=9 52 ns per __find_get_block mod=10 54 ns per __find_get_block mod=11 56 ns per __find_get_block mod=12 58 ns per __find_get_block mod=13 60 ns per __find_get_block mod=14 61 ns per __find_get_block mod=15 63 ns per __find_get_block mod=16 138 ns per __find_get_block mod=17 138 ns per __find_get_block mod=18 138 ns per __find_get_block mod=19 <-- results from first patch, when
lookup_bh_lru is a miss
70 ns per __find_get_block_slow mod=1
71 ns per __find_get_block_slow mod=2
71 ns per __find_get_block_slow mod=3
71 ns per __find_get_block_slow mod=4
71 ns per __find_get_block_slow mod=5
72 ns per __find_get_block_slow mod=6
71 ns per __find_get_block_slow mod=7
72 ns per __find_get_block_slow mod=8
71 ns per __find_get_block_slow mod=9
71 ns per __find_get_block_slow mod=10
71 ns per __find_get_block_slow mod=11
71 ns per __find_get_block_slow mod=12
71 ns per __find_get_block_slow mod=13
71 ns per __find_get_block_slow mod=14
71 ns per __find_get_block_slow mod=15
71 ns per __find_get_block_slow mod=16
71 ns per __find_get_block_slow mod=17
72 ns per __find_get_block_slow mod=18
72 ns per __find_get_block_slow mod=19
ls on home directory:
hits: 2 misses: 91
find on a linux-2.6 git tree:
hits: 25453 misses: 51084
make clean on a linux-2.6 git tree:
hits: 247615 misses: 32855
make on a linux-2.6 git tree:
hits: 1410414 misses: 166896
In more detail, where each bucket below indicates which index into per-CPU buffer lookup_bh_lru was found:
hits idx1 idx2 ... idx16
hits 139506 24299 21597 7462 15790 19108 6477 1349 1237 938 845 636 637 523 431 454 misses: 65773
So i think it makes more sense to just disable the cache for isolated CPUs.
Quoted 1 line
Reviewed-by Matthew Wilcox (oracle) <[email protected]>
Thanks.