Skip to contentSkip to conversation

fs/buffer.c: remove per-CPU buffer_head lookup cache

PATCH In review Marcelo Tosatti 4 participants 8 messages Jun 26, 2023 – Jun 27, 2023

Subsystems (6)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fs/buffer.c: remove per-CPU buffer_head lookup cache

For certain types of applications (for example PLC software or RAN processing), upon occurrence of an event, it is necessary to complete a certain task in a maximum amount of time (deadline).

One way to express this requirement is with a pair of numbers, deadline time and execution time, where:

* deadline time: length of time between event and deadline.
* execution time: length of time it takes for processing of event to occur on a particular hardware platform (uninterrupted).

The particular values depend on use-case. For the case where the realtime application executes in a virtualized guest, an IPI which must be serviced in the host will cause the following sequence of events:

1) VM-exit
2) execution of IPI (and function call)
3) VM-entry

Which causes an excess of 50us latency as observed by cyclictest (this violates the latency requirement of vRAN application with 1ms TTI, for example).

invalidate_bh_lrus calls an IPI on each CPU that has non empty per-CPU cache:

        on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

Test program:

#define NRLOOPS 200000
static int __init example_init(void)

{
        ktime_t s, e;
        s64 delta;
        int i, suc;

        bdev = blkdev_get_by_path("/dev/loop0", FMODE_READ, NULL);
        if (IS_ERR(bdev)) {
                printk(KERN_ERR "failed to load /dev/loop0\n");
                return -ENODEV;
        }

        suc = 0;
        delta = 0;
        for (i=0; i < NRLOOPS; i++) {
                struct buffer_head *bh;

                s = ktime_get();
                bh = __find_get_block(bdev, 1, 512);
                e = ktime_get();
                if (bh) {
                                suc++;
                                __brelse(bh);
                }
                delta = delta + ktime_to_ns(ktime_sub(e, s));

        }
        printk(KERN_ERR "%lld ns per __find_get_block (suc=%d)\n", delta/NRLOOPS, suc);

        suc = 0;
        delta = 0;
        for (i=0; i < NRLOOPS; i++) {
                struct buffer_head *bh;

                s = ktime_get();
                bh = __find_get_block_slow(bdev, 1);
                e = ktime_get();
                if (bh) {
                        suc++;
                        __brelse(bh);
                }
                delta = delta + ktime_to_ns(ktime_sub(e, s));
        }
        printk(KERN_ERR "%lld ns per __find_get_block_slow (suc=%d)\n", delta/NRLOOPS, suc);

        return 0;
}

---

 block/bdev.c                |    2 -
 fs/buffer.c                 |  209 +++++-------------------------------------------------------------------------------------------------------------------------------------------------------
 fs/jbd2/revoke.c            |    9 ++----
 fs/mpage.c                  |    3 --
 fs/ocfs2/journal.c          |    3 --
 fs/reiserfs/reiserfs.h      |    2 -
 include/linux/buffer_head.h |   11 ++------
 mm/migrate.c                |   12 +-------
 mm/swap.c                   |    4 --
 9 files changed, 21 insertions(+), 234 deletions(-)
block/bdev.c +0 -2
index 21c63bfef323..dc511024b11f 100644
@@ -72,7 +72,6 @@ static void kill_bdev(struct block_device *bdev)
 	if (mapping_empty(mapping))
 		return;
 
-	invalidate_bh_lrus();
 	truncate_inode_pages(mapping, 0);
 }
 
@@ -82,7 +81,6 @@ void invalidate_bdev(struct block_device *bdev)
 	struct address_space *mapping = bdev->bd_inode->i_mapping;
 
 	if (mapping->nrpages) {
-		invalidate_bh_lrus();
 		lru_add_drain_all();	/* make sure all lru add caches are flushed */
 		invalidate_mapping_pages(mapping, 0, -1);
 	}
fs/buffer.c +8 -201

264 lines not on this page · raw message

fs/jbd2/revoke.c +4 -5
index 4556e4689024..f68b9207737d 100644
@@ -345,7 +345,7 @@ int jbd2_journal_revoke(handle_t *handle, unsigned long long blocknr,
 	bh = bh_in;
 
 	if (!bh) {
-		bh = __find_get_block(bdev, blocknr, journal->j_blocksize);
+		bh = __find_get_block(bdev, blocknr);
 		if (bh)
 			BUFFER_TRACE(bh, "found on hash");
 	}
@@ -355,7 +355,7 @@ int jbd2_journal_revoke(handle_t *handle, unsigned long long blocknr,
 
 		/* If there is a different buffer_head lying around in
 		 * memory anywhere... */
-		bh2 = __find_get_block(bdev, blocknr, journal->j_blocksize);
+		bh2 = __find_get_block(bdev, blocknr);
 		if (bh2) {
 			/* ... and it has RevokeValid status... */
 			if (bh2 != bh && buffer_revokevalid(bh2))
@@ -466,7 +466,7 @@ int jbd2_journal_cancel_revoke(handle_t *handle, struct journal_head *jh)
 	 * state machine will get very upset later on. */
 	if (need_cancel) {
 		struct buffer_head *bh2;
-		bh2 = __find_get_block(bh->b_bdev, bh->b_blocknr, bh->b_size);
+		bh2 = __find_get_block(bh->b_bdev, bh->b_blocknr);
 		if (bh2) {
 			if (bh2 != bh)
 				clear_buffer_revoked(bh2);
@@ -496,8 +496,7 @@ void jbd2_clear_buffer_revoked_flags(journal_t *journal)
 			struct buffer_head *bh;
 			record = (struct jbd2_revoke_record_s *)list_entry;
 			bh = __find_get_block(journal->j_fs_dev,
-					      record->blocknr,
-					      journal->j_blocksize);
+					      record->blocknr);
 			if (bh) {
 				clear_buffer_revoked(bh);
 				__brelse(bh);
fs/mpage.c +1 -2
index 242e213ee064..e50d30a009ce 100644
@@ -634,8 +634,7 @@ static int __mpage_writepage(struct folio *folio, struct writeback_control *wbc,
 	if (boundary || (first_unmapped != blocks_per_page)) {
 		bio = mpage_bio_submit_write(bio);
 		if (boundary_block) {
-			write_boundary_block(boundary_bdev,
-					boundary_block, 1 << blkbits);
+			write_boundary_block(boundary_bdev, boundary_block);
 		}
 	} else {
 		mpd->last_block_in_bio = blocks[blocks_per_page - 1];
fs/ocfs2/journal.c +1 -2
index 25d8072ccfce..12be1471c9aa 100644
@@ -1212,8 +1212,7 @@ static int ocfs2_force_read_journal(struct inode *inode)
 		}
 
 		for (i = 0; i < p_blocks; i++, p_blkno++) {
-			bh = __find_get_block(osb->sb->s_bdev, p_blkno,
-					osb->sb->s_blocksize);
+			bh = __find_get_block(osb->sb->s_bdev, p_blkno);
 			/* block not cached. */
 			if (!bh)
 				continue;
fs/reiserfs/reiserfs.h +1 -1
index 1bccf6a2e908..19708f600bce 100644
@@ -2810,7 +2810,7 @@ struct reiserfs_journal_header {
 #define journal_hash(t,sb,block) ((t)[_jhashfn((sb),(block)) & JBH_HASH_MASK])
 
 /* We need these to make journal.c code more readable */
-#define journal_find_get_block(s, block) __find_get_block(SB_JOURNAL(s)->j_dev_bd, block, s->s_blocksize)
+#define journal_find_get_block(s, block) __find_get_block(SB_JOURNAL(s)->j_dev_bd, block)
 #define journal_getblk(s, block) __getblk(SB_JOURNAL(s)->j_dev_bd, block, s->s_blocksize)
 #define journal_bread(s, block) __bread(SB_JOURNAL(s)->j_dev_bd, block, s->s_blocksize)
 
include/linux/buffer_head.h +3 -8
index 1520793c72da..084a9d5f53d3 100644
@@ -227,8 +227,7 @@ static inline void clean_bdev_bh_alias(struct buffer_head *bh)
 void mark_buffer_async_write(struct buffer_head *bh);
 void __wait_on_buffer(struct buffer_head *);
 wait_queue_head_t *bh_waitq_head(struct buffer_head *bh);
-struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block,
-			unsigned size);
+struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block);
 struct buffer_head *__getblk_gfp(struct block_device *bdev, sector_t block,
 				  unsigned size, gfp_t gfp);
 void __brelse(struct buffer_head *);
@@ -236,9 +235,6 @@ void __bforget(struct buffer_head *);
 void __breadahead(struct block_device *, sector_t block, unsigned int size);
 struct buffer_head *__bread_gfp(struct block_device *,
 				sector_t block, unsigned size, gfp_t gfp);
-void invalidate_bh_lrus(void);
-void invalidate_bh_lrus_cpu(void);
-bool has_bh_in_lru(int cpu, void *dummy);
 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
 void free_buffer_head(struct buffer_head * bh);
 void unlock_buffer(struct buffer_head *bh);
@@ -247,8 +243,7 @@ int sync_dirty_buffer(struct buffer_head *bh);
 int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
 void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags);
 void submit_bh(blk_opf_t, struct buffer_head *);
-void write_boundary_block(struct block_device *bdev,
-			sector_t bblock, unsigned blocksize);
+void write_boundary_block(struct block_device *bdev, sector_t bblock);
 int bh_uptodate_or_lock(struct buffer_head *bh);
 int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait);
 void __bh_read_batch(int nr, struct buffer_head *bhs[],
@@ -375,7 +370,7 @@ sb_getblk_gfp(struct super_block *sb, sector_t block, gfp_t gfp)
 static inline struct buffer_head *
 sb_find_get_block(struct super_block *sb, sector_t block)
 {
-	return __find_get_block(sb->s_bdev, block, sb->s_blocksize);
+	return __find_get_block(sb->s_bdev, block);
 }
 
 static inline void
mm/migrate.c +2 -10
index 01cac26a3127..ceecd95cfd49 100644
@@ -747,9 +747,7 @@ static int __buffer_migrate_folio(struct address_space *mapping,
 
 	if (check_refs) {
 		bool busy;
-		bool invalidated = false;
 
-recheck_buffers:
 		busy = false;
 		spin_lock(&mapping->private_lock);
 		bh = head;
@@ -761,14 +759,8 @@ static int __buffer_migrate_folio(struct address_space *mapping,
 			bh = bh->b_this_page;
 		} while (bh != head);
 		if (busy) {
-			if (invalidated) {
-				rc = -EAGAIN;
-				goto unlock_buffers;
-			}
-			spin_unlock(&mapping->private_lock);
-			invalidate_bh_lrus();
-			invalidated = true;
-			goto recheck_buffers;
+			rc = -EAGAIN;
+			goto unlock_buffers;
 		}
 	}
 
mm/swap.c +1 -3
index 423199ee8478..64ce7255ff4d 100644
@@ -765,7 +765,6 @@ static void lru_add_and_bh_lrus_drain(void)
 	local_lock(&cpu_fbatches.lock);
 	lru_add_drain_cpu(smp_processor_id());
 	local_unlock(&cpu_fbatches.lock);
-	invalidate_bh_lrus_cpu();
 	mlock_drain_local();
 }
 
@@ -798,8 +797,7 @@ static bool cpu_needs_drain(unsigned int cpu)
 		folio_batch_count(&fbatches->lru_deactivate) ||
 		folio_batch_count(&fbatches->lru_lazyfree) ||
 		folio_batch_count(&fbatches->activate) ||
-		need_mlock_drain(cpu) ||
-		has_bh_in_lru(cpu, NULL);
+		need_mlock_drain(cpu);
 }
 
 /*
Reviewed-by on v1 1/1
#1 lore raw
Quoted 7 lines · Marcelo Tosatti

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

LOL. That's amazing. I can't even see why it's so expensive. The local_irq_disable(), perhaps? Your test case is the best possible one for lookup_bh_lru() where you're not even doing the copy.

#2 lore raw
Quoted 4 lines · Matthew Wilcox
Quoted 3 lines · Marcelo Tosatti

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

Quoted 8 lines
Quoted 4 lines

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

LOL. That's amazing. I can't even see why it's so expensive. The local_irq_disable(), perhaps? Your test case is the best possible one for lookup_bh_lru() where you're not even doing the copy.

I think it's even simpler than that.

i.e. the lookaside cache is being missed, so it's a pure cost and the code is always having to call __find_get_block_slow() anyway. Peeking at 16 buffers to not find a match is just as expensive as walking 3-4 tree levels in an Xarray to find the buffer in the first place....

IMO, this is an example of how lookaside caches are only a benefit if the working set of items largely fits in the lookaside cache and the cache lookup itself is much, much slower than a lookaside cache miss.

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#3 lore raw
Quoted 10 lines · Dave Chinner
Quoted 9 lines · Matthew Wilcox
Quoted 7 lines · Marcelo Tosatti

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

Quoted 8 lines
Quoted 3 lines

LOL. That's amazing. I can't even see why it's so expensive. The local_irq_disable(), perhaps? Your test case is the best possible one for lookup_bh_lru() where you're not even doing the copy.

I think it's even simpler than that.

i.e. the lookaside cache is being missed, so it's a pure cost and the code is always having to call __find_get_block_slow() anyway.

How does that happen?

__find_get_block(struct block_device *bdev, sector_t block, unsigned size)

{
        struct buffer_head *bh = lookup_bh_lru(bdev, block, size);

        if (bh == NULL) {
                /* __find_get_block_slow will mark the page accessed */
                bh = __find_get_block_slow(bdev, block);
                if (bh)
                        bh_lru_install(bh);

The second (and all subsequent) calls to __find_get_block() should find the BH in the LRU.

Quoted 4 lines

IMO, this is an example of how lookaside caches are only a benefit if the working set of items largely fits in the lookaside cache and the cache lookup itself is much, much slower than a lookaside cache miss.

But the test code he posted always asks for the same buffer each time. So it should find it in the lookaside cache?

#4 lore raw
Quoted 35 lines · Matthew Wilcox
Quoted 10 lines · Dave Chinner
Quoted 9 lines · Matthew Wilcox
Quoted 7 lines · Marcelo Tosatti

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

Quoted 8 lines
Quoted 3 lines

LOL. That's amazing. I can't even see why it's so expensive. The local_irq_disable(), perhaps? Your test case is the best possible one for lookup_bh_lru() where you're not even doing the copy.

I think it's even simpler than that.

i.e. the lookaside cache is being missed, so it's a pure cost and the code is always having to call __find_get_block_slow() anyway.

How does that happen?

__find_get_block(struct block_device *bdev, sector_t block, unsigned size)

{
        struct buffer_head *bh = lookup_bh_lru(bdev, block, size);

        if (bh == NULL) {
                /* __find_get_block_slow will mark the page accessed */
                bh = __find_get_block_slow(bdev, block);
                if (bh)
                        bh_lru_install(bh);

The second (and all subsequent) calls to __find_get_block() should find the BH in the LRU.

Quoted 7 lines
Quoted 4 lines

IMO, this is an example of how lookaside caches are only a benefit if the working set of items largely fits in the lookaside cache and the cache lookup itself is much, much slower than a lookaside cache miss.

But the test code he posted always asks for the same buffer each time. So it should find it in the lookaside cache?

Oh.

	for (i = 0; ....) {
		bh = __find_get_block(bdev, 1, 512);

That's a '1' being passed to __find_get_block, not 'i'.

/me goes and gets more coffee.

Maybe it's CONFIG_PREEMPT_RT=y doing something to the locks that isn't obvious here...

-Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#5 lore raw
In reply to Matthew Wilcox
Quoted 4 lines · Matthew Wilcox
Quoted 3 lines · Marcelo Tosatti

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

Quoted 8 lines
Quoted 4 lines

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

LOL. That's amazing. I can't even see why it's so expensive. The local_irq_disable(), perhaps? Your test case is the best possible one for lookup_bh_lru() where you're not even doing the copy.

Oops, that was due to incorrect buffer size being looked up versus installed size.

About 15ns is due to irq disablement.
6ns due to checking preempt is disabled (from __this_cpu_read).

So the actual numbers for the single block lookup are (searching for the same block number repeatedly):

42 ns per __find_get_block
68 ns per __find_get_block_slow

And increases linearly as the test increases the number of blocks which are searched for:

	say mod 3 is

	__find_get_block(blocknr=1)
	__find_get_block(blocknr=2)
	__find_get_block(blocknr=3)
41 ns per __find_get_block  mod=1
41 ns per __find_get_block  mod=2
42 ns per __find_get_block  mod=3
43 ns per __find_get_block  mod=4
45 ns per __find_get_block  mod=5
48 ns per __find_get_block  mod=6
48 ns per __find_get_block  mod=7
49 ns per __find_get_block  mod=8
51 ns per __find_get_block  mod=9
52 ns per __find_get_block  mod=10
54 ns per __find_get_block  mod=11
56 ns per __find_get_block  mod=12
58 ns per __find_get_block  mod=13
60 ns per __find_get_block  mod=14
61 ns per __find_get_block  mod=15
63 ns per __find_get_block  mod=16
138 ns per __find_get_block  mod=17
138 ns per __find_get_block  mod=18
138 ns per __find_get_block  mod=19		<-- results from first patch, when

lookup_bh_lru is a miss
70 ns per __find_get_block_slow mod=1
71 ns per __find_get_block_slow mod=2
71 ns per __find_get_block_slow mod=3
71 ns per __find_get_block_slow mod=4
71 ns per __find_get_block_slow mod=5
72 ns per __find_get_block_slow mod=6
71 ns per __find_get_block_slow mod=7
72 ns per __find_get_block_slow mod=8
71 ns per __find_get_block_slow mod=9
71 ns per __find_get_block_slow mod=10
71 ns per __find_get_block_slow mod=11
71 ns per __find_get_block_slow mod=12
71 ns per __find_get_block_slow mod=13
71 ns per __find_get_block_slow mod=14
71 ns per __find_get_block_slow mod=15
71 ns per __find_get_block_slow mod=16
71 ns per __find_get_block_slow mod=17
72 ns per __find_get_block_slow mod=18
72 ns per __find_get_block_slow mod=19

ls on home directory:
hits: 2 misses: 91

find on a linux-2.6 git tree:
hits: 25453 misses: 51084

make clean on a linux-2.6 git tree:
hits: 247615 misses: 32855

make on a linux-2.6 git tree:
hits: 1410414 misses: 166896

In more detail, where each bucket below indicates which index into per-CPU buffer lookup_bh_lru was found:

hits idx1 idx2 ... idx16
hits 139506 24299 21597 7462 15790 19108 6477 1349 1237 938 845 636 637 523 431 454 misses: 65773

So i think it makes more sense to just disable the cache for isolated CPUs.

Quoted 1 line

Thanks.

#6 lore raw
In reply to Marcelo Tosatti
Quoted 47 lines · Marcelo Tosatti

Upon closer investigation, it was found that in current codebase, lookup_bh_lru is slower than __find_get_block_slow:

114 ns per __find_get_block
68 ns per __find_get_block_slow

So remove the per-CPU buffer_head caching.

Test program:

#define NRLOOPS 200000
static int __init example_init(void)

{
        ktime_t s, e;
        s64 delta;
        int i, suc;

        bdev = blkdev_get_by_path("/dev/loop0", FMODE_READ, NULL);
        if (IS_ERR(bdev)) {
                printk(KERN_ERR "failed to load /dev/loop0\n");
                return -ENODEV;
        }

        suc = 0;
        delta = 0;
        for (i=0; i < NRLOOPS; i++) {
                struct buffer_head *bh;

                s = ktime_get();
                bh = __find_get_block(bdev, 1, 512);
                e = ktime_get();
                if (bh) {
                                suc++;
                                __brelse(bh);
                }
                delta = delta + ktime_to_ns(ktime_sub(e, s));

        }
        printk(KERN_ERR "%lld ns per __find_get_block (suc=%d)\n", delta/NRLOOPS, suc);

        suc = 0;
        delta = 0;
        for (i=0; i < NRLOOPS; i++) {
                struct buffer_head *bh;

                s = ktime_get();
                bh = __find_get_block_slow(bdev, 1);
Quoted 8 lines
                e = ktime_get();
                if (bh) {
                        suc++;
                        __brelse(bh);
                }
                delta = delta + ktime_to_ns(ktime_sub(e, s));
        }
        printk(KERN_ERR "%lld ns per __find_get_block_slow (suc=%d)\n", delta/NRLOOPS, suc);

It occurs to me that this is close to being the best-case scenario for page-cache lookup as well as for lru lookup. Can you re-run it with block 4UL * 1024 * 1024 * 1024 instead of block 1?