Skip to contentSkip to conversation

xfs: dump cache size with sysfs

PATCH In review Wengang Wang 2 participants 4 messages May 7, 2025 – May 13, 2025 XFS filesystem

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xfs: dump cache size with sysfs

This patch dumps buffer cache size per mount.

Counters for buffer size 1 BB to 257 BBs are stored in struct xfs_mount. The last one is for size >= 257.
When dumping the cache size, we try to dump for each exact buffer size if the output memory buffer (PAGE SIZE) is enough.

Buffer_size Count Total_size
size1 count1
size2 count2
....

Quoted 1 line

=size_last count_size_last total_size_in_BBs

For each size, we
1 dump it the count is not zero
2 dump two fields: size of the buffers and the count of those buffers
so that the total size can be calculated by size * count. We don't dump the total size to save space.
3 for the last size (if count is not zero) we dump three fields:
the buffer size prefixed by ">=", count of those buffers and the total size.

If the space is enough to store about exact output, we are good.

In case the PAGE SIZE is not enough for above format, we then change to the following size range based format:

Buffer_size Count Total_size
<=1 count total /* for buffers with size 1 to 1 */
<=2 count total /* for buffers with size 2 to 2 */
<=4 count total /* for buffers with size 3 to 4 */
....

<=256   count   total    /* for buffers with size 129 to 256 */
>=257   count   total	 /* for buffers with size >= 257 */

For each line, we
1 dump three fields: size range, count of those buffers and total size of
them
2 dump it no matter count if zero or not

---
 fs/xfs/xfs_buf.c   |  36 +++++++++++++-
 fs/xfs/xfs_mount.h |  20 ++++++++
 fs/xfs/xfs_super.c |  36 ++++++++++++--
 fs/xfs/xfs_sysfs.c | 118 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 203 insertions(+), 7 deletions(-)
fs/xfs/xfs_buf.c +34 -2
index 1a2b3f06fa71..9b0aae04745b 100644
@@ -99,6 +99,28 @@ xfs_buf_free_callback(
 	kmem_cache_free(xfs_buf_cache, bp);
 }
 
+static void xfs_cache_count_inc(struct xfs_mount *mp, int bbs)
+{
+	int idx = bbs - 1;
+
+	if (idx > XFS_CACHECOUNT_SIZE_MAX_IDX) {
+		idx = XFS_CACHECOUNT_SIZE_MAX_IDX;
+		percpu_counter_add(&mp->m_cache_cts.last_total_bbs, bbs);
+	}
+	percpu_counter_inc(&mp->m_cache_cts.counters[idx]);
+}
+
+static void xfs_cache_count_dec(struct xfs_mount *mp, int bbs)
+{
+	int idx = bbs - 1;
+
+	if (idx > XFS_CACHECOUNT_SIZE_MAX_IDX) {
+		idx = XFS_CACHECOUNT_SIZE_MAX_IDX;
+		percpu_counter_sub(&mp->m_cache_cts.last_total_bbs, bbs);
+	}
+	percpu_counter_dec(&mp->m_cache_cts.counters[idx]);
+}
+
 static void
 xfs_buf_free(
 	struct xfs_buf		*bp)
@@ -113,6 +135,8 @@ xfs_buf_free(
 	if (!xfs_buftarg_is_mem(bp->b_target) && size >= PAGE_SIZE)
 		mm_account_reclaimed_pages(howmany(size, PAGE_SHIFT));
 
+	xfs_cache_count_dec(bp->b_mount, bp->b_length);
+
 	if (is_vmalloc_addr(bp->b_addr))
 		vfree(bp->b_addr);
 	else if (bp->b_flags & _XBF_KMEM)
@@ -196,8 +220,14 @@ xfs_buf_alloc_backing_mem(
 	 * allocation for all power of two sizes, which matches most of the
 	 * smaller than PAGE_SIZE buffers used by XFS.
 	 */
-	if (size < PAGE_SIZE && is_power_of_2(size))
-		return xfs_buf_alloc_kmem(bp, size, gfp_mask);
+	if (size < PAGE_SIZE && is_power_of_2(size)) {
+		int	ret;
+
+		ret = xfs_buf_alloc_kmem(bp, size, gfp_mask);
+		if (ret == 0)
+			xfs_cache_count_inc(bp->b_mount, bp->b_length);
+		return ret;
+	}
 
 	/*
 	 * Don't bother with the retry loop for single PAGE allocations: vmalloc
@@ -231,6 +261,7 @@ xfs_buf_alloc_backing_mem(
 		goto fallback;
 	}
 	bp->b_addr = folio_address(folio);
+	xfs_cache_count_inc(bp->b_mount, bp->b_length);
 	trace_xfs_buf_backing_folio(bp, _RET_IP_);
 	return 0;
 
@@ -245,6 +276,7 @@ xfs_buf_alloc_backing_mem(
 		memalloc_retry_wait(gfp_mask);
 	}
 
+	xfs_cache_count_inc(bp->b_mount, bp->b_length);
 	trace_xfs_buf_backing_vmalloc(bp, _RET_IP_);
 	return 0;
 }
fs/xfs/xfs_mount.h +20 -0
index e5192c12e7ac..9141d0122fc9 100644
@@ -135,6 +135,21 @@ struct xfs_freecounter {
 	uint64_t		res_saved;
 };
 
+#define XFS_CACHECOUNT_SIZE_MAX_IDX	256
+struct xfs_cachecounters {
+	/*
+	 * index to the array is the size of buffers in BBs minus 1.
+	 * elements are the numbers of buffers for each size.
+	 * last element is for exact size and biger sizes
+	 */
+	struct percpu_counter counters[XFS_CACHECOUNT_SIZE_MAX_IDX + 1];
+
+	/* total number of BBs for last element */
+	struct percpu_counter last_total_bbs;
+
+	struct xfs_kobj	kobj;
+};
+
 /*
  * The struct xfsmount layout is optimised to separate read-mostly variables
  * from variables that are frequently modified. We put the read-mostly variables
@@ -276,6 +291,11 @@ typedef struct xfs_mount {
 	 */
 	atomic64_t		m_allocbt_blks;
 
+	/*
+	 * Count number of buffers and totol size of them per size.
+	 */
+	struct xfs_cachecounters	m_cache_cts;
+
 	struct xfs_groups	m_groups[XG_TYPE_MAX];
 	struct delayed_work	m_reclaim_work;	/* background inode reclaim */
 	struct xfs_zone_info	*m_zone_info;	/* zone allocator information */
fs/xfs/xfs_super.c +31 -5
index 3be041647ec1..30bdb9382786 100644
@@ -1121,7 +1121,7 @@ xfs_init_percpu_counters(
 	struct xfs_mount	*mp)
 {
 	int			error;
-	int			i;
+	int			i, j;
 
 	error = percpu_counter_init(&mp->m_icount, 0, GFP_KERNEL);
 	if (error)
@@ -1142,15 +1142,38 @@ xfs_init_percpu_counters(
 	for (i = 0; i < XC_FREE_NR; i++) {
 		error = percpu_counter_init(&mp->m_free[i].count, 0,
 				GFP_KERNEL);
-		if (error)
+		if (error) {
+			j = i;
 			goto free_freecounters;
+		}
+	}
+
+	error = percpu_counter_init(&mp->m_cache_cts.last_total_bbs,
+				    0, GFP_KERNEL);
+	if (error) {
+		j = XC_FREE_NR;
+		goto free_freecounters;
+	}
+
+	for (i = 0; i <= XFS_CACHECOUNT_SIZE_MAX_IDX; i++) {
+		error = percpu_counter_init(&mp->m_cache_cts.counters[i],
+					    0, GFP_KERNEL);
+		if (error) {
+			j = i;
+			goto free_cache_cts;
+		}
 	}
 
 	return 0;
 
+free_cache_cts:
+	percpu_counter_destroy(&mp->m_cache_cts.last_total_bbs);
+	while (--j >= 0)
+		percpu_counter_destroy(&mp->m_cache_cts.counters[j]);
+	j = XC_FREE_NR;
 free_freecounters:
-	while (--i >= 0)
-		percpu_counter_destroy(&mp->m_free[i].count);
+	while (--j >= 0)
+		percpu_counter_destroy(&mp->m_free[j].count);
 	percpu_counter_destroy(&mp->m_delalloc_rtextents);
 free_delalloc:
 	percpu_counter_destroy(&mp->m_delalloc_blks);
@@ -1177,8 +1200,11 @@ static void
 xfs_destroy_percpu_counters(
 	struct xfs_mount	*mp)
 {
-	enum xfs_free_counter	i;
+	int	i;
 
+	percpu_counter_destroy(&mp->m_cache_cts.last_total_bbs);
+	for (i = 0; i <= XFS_CACHECOUNT_SIZE_MAX_IDX; i++)
+		percpu_counter_destroy(&mp->m_cache_cts.counters[i]);
 	for (i = 0; i < XC_FREE_NR; i++)
 		percpu_counter_destroy(&mp->m_free[i].count);
 	percpu_counter_destroy(&mp->m_icount);
fs/xfs/xfs_sysfs.c +118 -0

138 lines not on this page · raw message

Signature · 1 line
2.39.5 (Apple Git-154)
#1 lore raw

This is an awful lot of code. Why can't this be done in eBPF using the existing tracepoints?

Btw, it would be nice to collect such scripts in say xfsprogs.

#2 lore raw

Hi Christoph,

Quoted 5 lines · Christoph Hellwig

This is an awful lot of code. Why can't this be done in eBPF using the existing tracepoints?

Thanks for the idea of eBPF.
As I understand it, tracepoints are fired only when code got run say when xfs_buf is allocated and released. With eBPF, we are able to track the allocation/release, but I have some things to consider:

1. We have to keep the script running all the time since system boot up to get the counting? If the eBPF script gets killed somehow, we can no longer can get correct info again until next kernel rebooting?
2. eBPF script is not guaranteed to be ready before changes in buffer cache? So the eBPF output might not be what it really is in kernel?
3. to run ePBF, it requires extra packages to be installed. But those packages could be not available on customer’s production systems, right?
3. Also, it looks much complex than a single ‘cat /sys/fs/xfs/<dev>/cache/size’.

What’s the concern to add the syses code? Though the patch looks big (but not really very big), It’s simple and direct.

Thanks,
Wengang

Quoted 2 lines

Btw, it would be nice to collect such scripts in say xfsprogs.

#3 lore raw
Quoted 2 lines · Wengang Wang

What’s the concern to add the syses code? Though the patch looks big (but not really very big), It’s simple and direct.

It adds rather special cased stats for something that does not seem like a killer use case. Especially as we should have a generic way to collect stats form tracepoints, be that using bpf or normal kernel code.