Skip to contentSkip to conversation

vmalloc buffers POC

PATCH New Christoph Hellwig 1 participant 5 messages Jan 22, 2024 XFS filesystem

lore
Share & tools
lore
#0 lore raw

Hi Dave,

this is my old vmalloc buffers POC. I just dusted it off and did a quick rebased. It survives a 4k quick xfstests run but otherwise this should be treated as not very trustworth. Maybe it will be useful for your folio work.

Diffstat:

 libxfs/xfs_ialloc.c    |    2 
 libxfs/xfs_inode_buf.c |    2 
 scrub/inode_repair.c   |    2 
 xfs_buf.c              |  267 ++++++++-----------------------------------------
 xfs_buf.h              |   22 ++--
 xfs_buf_item_recover.c |    7 -
 xfs_inode.c            |    2 
 7 files changed, 65 insertions(+), 239 deletions(-)
#1 lore raw

PATCH 1/4 xfs: kill XBF_UNMAPPED

Unmapped buffer access is a pain, so kill it. Note that this means we now pay the vmap overhead even for inode cluster buffers, but that will be fixed by switching to large folios.

---
 fs/xfs/libxfs/xfs_ialloc.c    |  2 +-
 fs/xfs/libxfs/xfs_inode_buf.c |  2 +-
 fs/xfs/scrub/inode_repair.c   |  2 +-
 fs/xfs/xfs_buf.c              | 45 +----------------------------------
 fs/xfs/xfs_buf.h              | 16 +++++++++----
 fs/xfs/xfs_buf_item_recover.c |  7 +-----
 fs/xfs/xfs_inode.c            |  2 +-
 7 files changed, 17 insertions(+), 59 deletions(-)
fs/xfs/libxfs/xfs_ialloc.c +1 -1
index 2361a22035b0c0..b7aaade726f2ee 100644
@@ -358,7 +358,7 @@ xfs_ialloc_inode_init(
 				(j * M_IGEO(mp)->blocks_per_cluster));
 		error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d,
 				mp->m_bsize * M_IGEO(mp)->blocks_per_cluster,
-				XBF_UNMAPPED, &fbuf);
+				0, &fbuf);
 		if (error)
 			return error;
 
fs/xfs/libxfs/xfs_inode_buf.c +1 -1
index 137a65bda95dc1..e649e0261552be 100644
@@ -133,7 +133,7 @@ xfs_imap_to_bp(
 	struct xfs_buf		**bpp)
 {
 	return xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, imap->im_blkno,
-				   imap->im_len, XBF_UNMAPPED, bpp,
+				   imap->im_len, 0, bpp,
 				   &xfs_inode_buf_ops);
 }
 
fs/xfs/scrub/inode_repair.c +1 -1
index 0ca62d59f84ad1..faae6460acea44 100644
@@ -1095,7 +1095,7 @@ xrep_dinode_core(
 
 	/* Read the inode cluster buffer. */
 	error = xfs_trans_read_buf(sc->mp, sc->tp, sc->mp->m_ddev_targp,
-			ri->imap.im_blkno, ri->imap.im_len, XBF_UNMAPPED, &bp,
+			ri->imap.im_blkno, ri->imap.im_len, 0, &bp,
 			NULL);
 	if (error)
 		return error;
fs/xfs/xfs_buf.c +1 -44

63 lines not on this page · raw message

fs/xfs/xfs_buf.h +11 -5
index b470de08a46ca8..379d58c4cb9a27 100644
@@ -51,7 +51,6 @@ struct xfs_buf;
 #define XBF_LIVESCAN	 (1u << 28)
 #define XBF_INCORE	 (1u << 29)/* lookup only, return if found in cache */
 #define XBF_TRYLOCK	 (1u << 30)/* lock requested, but do not wait */
-#define XBF_UNMAPPED	 (1u << 31)/* do not map the buffer */
 
 
 typedef unsigned int xfs_buf_flags_t;
@@ -74,8 +73,7 @@ typedef unsigned int xfs_buf_flags_t;
 	/* The following interface flags should never be set */ \
 	{ XBF_LIVESCAN,		"LIVESCAN" }, \
 	{ XBF_INCORE,		"INCORE" }, \
-	{ XBF_TRYLOCK,		"TRYLOCK" }, \
-	{ XBF_UNMAPPED,		"UNMAPPED" }
+	{ XBF_TRYLOCK,		"TRYLOCK" }
 
 /*
  * Internal state flags.
@@ -308,12 +306,20 @@ extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error,
 #define xfs_buf_ioerror(bp, err) __xfs_buf_ioerror((bp), (err), __this_address)
 extern void xfs_buf_ioerror_alert(struct xfs_buf *bp, xfs_failaddr_t fa);
 void xfs_buf_ioend_fail(struct xfs_buf *);
-void xfs_buf_zero(struct xfs_buf *bp, size_t boff, size_t bsize);
 void __xfs_buf_mark_corrupt(struct xfs_buf *bp, xfs_failaddr_t fa);
 #define xfs_buf_mark_corrupt(bp) __xfs_buf_mark_corrupt((bp), __this_address)
 
 /* Buffer Utility Routines */
-extern void *xfs_buf_offset(struct xfs_buf *, size_t);
+static inline void *xfs_buf_offset(struct xfs_buf *bp, size_t offset)
+{
+	return bp->b_addr + offset;
+}
+
+static inline void xfs_buf_zero(struct xfs_buf *bp, size_t boff, size_t bsize)
+{
+	memset(bp->b_addr + boff, 0, bsize);
+}
+
 extern void xfs_buf_stale(struct xfs_buf *bp);
 
 /* Delayed Write Buffer Routines */
fs/xfs/xfs_buf_item_recover.c +1 -6
index 43167f543afc33..3b1e2dab3ababc 100644
@@ -891,7 +891,6 @@ xlog_recover_buf_commit_pass2(
 	struct xfs_mount		*mp = log->l_mp;
 	struct xfs_buf			*bp;
 	int				error;
-	uint				buf_flags;
 	xfs_lsn_t			lsn;
 
 	/*
@@ -911,12 +910,8 @@ xlog_recover_buf_commit_pass2(
 
 	trace_xfs_log_recover_buf_recover(log, buf_f);
 
-	buf_flags = 0;
-	if (buf_f->blf_flags & XFS_BLF_INODE_BUF)
-		buf_flags |= XBF_UNMAPPED;
-
 	error = xfs_buf_read(mp->m_ddev_targp, buf_f->blf_blkno, buf_f->blf_len,
-			  buf_flags, &bp, NULL);
+			  0, &bp, NULL);
 	if (error)
 		return error;
 
fs/xfs/xfs_inode.c +1 -1
index 1fd94958aa97aa..db52cd086c4ada 100644
@@ -2301,7 +2301,7 @@ xfs_ifree_cluster(
 		 */
 		error = xfs_trans_get_buf(tp, mp->m_ddev_targp, blkno,
 				mp->m_bsize * igeo->blocks_per_cluster,
-				XBF_UNMAPPED, &bp);
+				0, &bp);
 		if (error)
 			return error;
 
Signature · 1 line
2.39.2
#2 lore raw

PATCH 2/4 xfs: map buffers in xfs_buf_alloc_pages

In reply to Christoph Hellwig

With the concept of unmapped buffer gone, there is no reason to not vmap the buffer pages directly in xfs_buf_alloc_pages.

---
 fs/xfs/xfs_buf.c | 37 ++++++-------------------------------
 1 file changed, 6 insertions(+), 31 deletions(-)
fs/xfs/xfs_buf.c +6 -31
index 792759afdc73be..1bfd89dbb78d79 100644
@@ -406,18 +406,7 @@ xfs_buf_alloc_pages(
 		XFS_STATS_INC(bp->b_mount, xb_page_retries);
 		memalloc_retry_wait(gfp_mask);
 	}
-	return 0;
-}
 
-/*
- *	Map buffer into kernel address-space if necessary.
- */
-STATIC int
-_xfs_buf_map_pages(
-	struct xfs_buf		*bp,
-	xfs_buf_flags_t		flags)
-{
-	ASSERT(bp->b_flags & _XBF_PAGES);
 	if (bp->b_page_count == 1) {
 		/* A single page buffer is always mappable */
 		bp->b_addr = page_address(bp->b_pages[0]);
@@ -443,8 +432,13 @@ _xfs_buf_map_pages(
 		} while (retried++ <= 1);
 		memalloc_nofs_restore(nofs_flag);
 
-		if (!bp->b_addr)
+		if (!bp->b_addr) {
+			xfs_warn_ratelimited(bp->b_target->bt_mount,
+				"%s: failed to map %u pages", __func__,
+				bp->b_page_count);
+			xfs_buf_free_pages(bp);
 			return -ENOMEM;
+		}
 	}
 
 	return 0;
@@ -718,18 +712,6 @@ xfs_buf_get_map(
 		xfs_perag_put(pag);
 	}
 
-	/* We do not hold a perag reference anymore. */
-	if (!bp->b_addr) {
-		error = _xfs_buf_map_pages(bp, flags);
-		if (unlikely(error)) {
-			xfs_warn_ratelimited(btp->bt_mount,
-				"%s: failed to map %u pages", __func__,
-				bp->b_page_count);
-			xfs_buf_relse(bp);
-			return error;
-		}
-	}
-
 	/*
 	 * Clear b_error if this is a lookup from a caller that doesn't expect
 	 * valid data to be found in the buffer.
@@ -959,13 +941,6 @@ xfs_buf_get_uncached(
 	if (error)
 		goto fail_free_buf;
 
-	error = _xfs_buf_map_pages(bp, 0);
-	if (unlikely(error)) {
-		xfs_warn(target->bt_mount,
-			"%s: failed to map pages", __func__);
-		goto fail_free_buf;
-	}
-
 	trace_xfs_buf_get_uncached(bp, _RET_IP_);
 	*bpp = bp;
 	return 0;
Signature · 1 line
2.39.2
#3 lore raw

PATCH 3/4 xfs: walk b_addr for buffer I/O

In reply to Christoph Hellwig

Instead of walking the page array just walk the kernel virtual address in ->b_addr. This prepares for using vmalloc for buffers and removing the b_pages array.

---
 fs/xfs/xfs_buf.c | 93 ++++++++++++++----------------------------------
 fs/xfs/xfs_buf.h |  2 --
 2 files changed, 26 insertions(+), 69 deletions(-)
fs/xfs/xfs_buf.c +26 -67
index 1bfd89dbb78d79..ddd917bed22e34 100644
@@ -343,7 +343,6 @@ xfs_buf_alloc_kmem(
 		bp->b_addr = NULL;
 		return -ENOMEM;
 	}
-	bp->b_offset = offset_in_page(bp->b_addr);
 	bp->b_pages = bp->b_page_array;
 	bp->b_pages[0] = kmem_to_page(bp->b_addr);
 	bp->b_page_count = 1;
@@ -1426,79 +1425,47 @@ xfs_buf_bio_end_io(
 static void
 xfs_buf_ioapply_map(
 	struct xfs_buf	*bp,
-	int		map,
-	int		*buf_offset,
-	int		*count,
+	unsigned int	map,
+	unsigned int	*buf_offset,
 	blk_opf_t	op)
 {
-	int		page_index;
-	unsigned int	total_nr_pages = bp->b_page_count;
-	int		nr_pages;
 	struct bio	*bio;
-	sector_t	sector =  bp->b_maps[map].bm_bn;
 	int		size;
-	int		offset;
-
-	/* skip the pages in the buffer before the start offset */
-	page_index = 0;
-	offset = *buf_offset;
-	while (offset >= PAGE_SIZE) {
-		page_index++;
-		offset -= PAGE_SIZE;
-	}
 
 	/*
 	 * Limit the IO size to the length of the current vector, and update the
 	 * remaining IO count for the next time around.
 	 */
-	size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
-	*count -= size;
-	*buf_offset += size;
+	size = min_t(unsigned int, BBTOB(bp->b_maps[map].bm_len),
+			BBTOB(bp->b_length) - *buf_offset);
+	if (WARN_ON_ONCE(bp->b_page_count > BIO_MAX_VECS)) {
+		xfs_buf_ioerror(bp, -EIO);
+		return;
+	}
 
-next_chunk:
 	atomic_inc(&bp->b_io_remaining);
-	nr_pages = bio_max_segs(total_nr_pages);
 
-	bio = bio_alloc(bp->b_target->bt_bdev, nr_pages, op, GFP_NOIO);
-	bio->bi_iter.bi_sector = sector;
+	bio = bio_alloc(bp->b_target->bt_bdev, bp->b_page_count, op, GFP_NOIO);
+	bio->bi_iter.bi_sector = bp->b_maps[map].bm_bn;
 	bio->bi_end_io = xfs_buf_bio_end_io;
 	bio->bi_private = bp;
 
-	for (; size && nr_pages; nr_pages--, page_index++) {
-		int	rbytes, nbytes = PAGE_SIZE - offset;
+	do {
+		void		*data = bp->b_addr + *buf_offset;
+		struct page	*page = kmem_to_page(data);
+		unsigned int	off = offset_in_page(data);
+		unsigned int	len = min_t(unsigned, size, PAGE_SIZE - off);
 
-		if (nbytes > size)
-			nbytes = size;
+		__bio_add_page(bio, page, len, off);
+		size -= len;
+		*buf_offset += len;
+	} while (size);
 
-		rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
-				      offset);
-		if (rbytes < nbytes)
-			break;
-
-		offset = 0;
-		sector += BTOBB(nbytes);
-		size -= nbytes;
-		total_nr_pages--;
+	if (xfs_buf_is_vmapped(bp)) {
+		flush_kernel_vmap_range(bp->b_addr,
+					xfs_buf_vmap_len(bp));
 	}
-
-	if (likely(bio->bi_iter.bi_size)) {
-		if (xfs_buf_is_vmapped(bp)) {
-			flush_kernel_vmap_range(bp->b_addr,
-						xfs_buf_vmap_len(bp));
-		}
-		submit_bio(bio);
-		if (size)
-			goto next_chunk;
-	} else {
-		/*
-		 * This is guaranteed not to be the last io reference count
-		 * because the caller (xfs_buf_submit) holds a count itself.
-		 */
-		atomic_dec(&bp->b_io_remaining);
-		xfs_buf_ioerror(bp, -EIO);
-		bio_put(bio);
-	}
-
+	submit_bio(bio);
 }
 
 STATIC void
@@ -1507,8 +1474,7 @@ _xfs_buf_ioapply(
 {
 	struct blk_plug	plug;
 	blk_opf_t	op;
-	int		offset;
-	int		size;
+	unsigned int	offset = 0;
 	int		i;
 
 	/*
@@ -1564,16 +1530,9 @@ _xfs_buf_ioapply(
 	 * _xfs_buf_ioapply_vec() will modify them appropriately for each
 	 * subsequent call.
 	 */
-	offset = bp->b_offset;
-	size = BBTOB(bp->b_length);
 	blk_start_plug(&plug);
-	for (i = 0; i < bp->b_map_count; i++) {
-		xfs_buf_ioapply_map(bp, i, &offset, &size, op);
-		if (bp->b_error)
-			break;
-		if (size <= 0)
-			break;	/* all done */
-	}
+	for (i = 0; i < bp->b_map_count; i++)
+		xfs_buf_ioapply_map(bp, i, &offset, op);
 	blk_finish_plug(&plug);
 }
 
fs/xfs/xfs_buf.h +0 -2
index 379d58c4cb9a27..2116fed2b53026 100644
@@ -186,8 +186,6 @@ struct xfs_buf {
 	atomic_t		b_pin_count;	/* pin count */
 	atomic_t		b_io_remaining;	/* #outstanding I/O requests */
 	unsigned int		b_page_count;	/* size of page array */
-	unsigned int		b_offset;	/* page offset of b_addr,
-						   only for _XBF_KMEM buffers */
 	int			b_error;	/* error code on I/O */
 
 	/*
Signature · 1 line
2.39.2
#4 lore raw

PATCH 4/4 xfs: use vmalloc for multi-page buffers

In reply to Christoph Hellwig

Instead of allocating the pages manually using the bulk page allocator and then using vm_map_page just use vmalloc to allocate the entire buffer - vmalloc will use the bulk allocator internally if it fits.

With this the b_pages array can go away as well as nothing uses it.

---
 fs/xfs/xfs_buf.c | 96 +++++++++---------------------------------------
 fs/xfs/xfs_buf.h |  4 --
 2 files changed, 18 insertions(+), 82 deletions(-)
fs/xfs/xfs_buf.c +18 -78
index ddd917bed22e34..6c2c4c809cc55c 100644
@@ -197,7 +197,7 @@ xfs_buf_get_maps(
 }
 
 /*
- *	Frees b_pages if it was allocated.
+ *	Frees b_maps if it was allocated.
  */
 static void
 xfs_buf_free_maps(
@@ -271,29 +271,6 @@ _xfs_buf_alloc(
 	return 0;
 }
 
-static void
-xfs_buf_free_pages(
-	struct xfs_buf	*bp)
-{
-	uint		i;
-
-	ASSERT(bp->b_flags & _XBF_PAGES);
-
-	if (xfs_buf_is_vmapped(bp))
-		vm_unmap_ram(bp->b_addr, bp->b_page_count);
-
-	for (i = 0; i < bp->b_page_count; i++) {
-		if (bp->b_pages[i])
-			__free_page(bp->b_pages[i]);
-	}
-	mm_account_reclaimed_pages(bp->b_page_count);
-
-	if (bp->b_pages != bp->b_page_array)
-		kmem_free(bp->b_pages);
-	bp->b_pages = NULL;
-	bp->b_flags &= ~_XBF_PAGES;
-}
-
 static void
 xfs_buf_free_callback(
 	struct callback_head	*cb)
@@ -312,10 +289,15 @@ xfs_buf_free(
 
 	ASSERT(list_empty(&bp->b_lru));
 
+	if (!(bp->b_flags & _XBF_KMEM))
+		mm_account_reclaimed_pages(bp->b_page_count);
+
 	if (bp->b_flags & _XBF_PAGES)
-		xfs_buf_free_pages(bp);
-	else if (bp->b_flags & _XBF_KMEM)
-		kmem_free(bp->b_addr);
+		put_page(virt_to_page(bp->b_addr));
+	else
+		kvfree(bp->b_addr);
+
+	bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
 
 	call_rcu(&bp->b_rcu, xfs_buf_free_callback);
 }
@@ -343,8 +325,6 @@ xfs_buf_alloc_kmem(
 		bp->b_addr = NULL;
 		return -ENOMEM;
 	}
-	bp->b_pages = bp->b_page_array;
-	bp->b_pages[0] = kmem_to_page(bp->b_addr);
 	bp->b_page_count = 1;
 	bp->b_flags |= _XBF_KMEM;
 	return 0;
@@ -356,7 +336,6 @@ xfs_buf_alloc_pages(
 	xfs_buf_flags_t	flags)
 {
 	gfp_t		gfp_mask = __GFP_NOWARN;
-	long		filled = 0;
 
 	if (flags & XBF_READ_AHEAD)
 		gfp_mask |= __GFP_NORETRY;
@@ -365,56 +344,24 @@ xfs_buf_alloc_pages(
 
 	/* Make sure that we have a page list */
 	bp->b_page_count = DIV_ROUND_UP(BBTOB(bp->b_length), PAGE_SIZE);
-	if (bp->b_page_count <= XB_PAGES) {
-		bp->b_pages = bp->b_page_array;
-	} else {
-		bp->b_pages = kzalloc(sizeof(struct page *) * bp->b_page_count,
-					gfp_mask);
-		if (!bp->b_pages)
-			return -ENOMEM;
-	}
-	bp->b_flags |= _XBF_PAGES;
 
 	/* Assure zeroed buffer for non-read cases. */
 	if (!(flags & XBF_READ))
 		gfp_mask |= __GFP_ZERO;
 
-	/*
-	 * Bulk filling of pages can take multiple calls. Not filling the entire
-	 * array is not an allocation failure, so don't back off if we get at
-	 * least one extra page.
-	 */
-	for (;;) {
-		long	last = filled;
-
-		filled = alloc_pages_bulk_array(gfp_mask, bp->b_page_count,
-						bp->b_pages);
-		if (filled == bp->b_page_count) {
-			XFS_STATS_INC(bp->b_mount, xb_page_found);
-			break;
-		}
-
-		if (filled != last)
-			continue;
+	if (bp->b_page_count == 1) {
+		struct page *page;
 
-		if (flags & XBF_READ_AHEAD) {
-			xfs_buf_free_pages(bp);
+		page = alloc_page(gfp_mask);
+		if (!page)
 			return -ENOMEM;
-		}
-
-		XFS_STATS_INC(bp->b_mount, xb_page_retries);
-		memalloc_retry_wait(gfp_mask);
-	}
-
-	if (bp->b_page_count == 1) {
-		/* A single page buffer is always mappable */
-		bp->b_addr = page_address(bp->b_pages[0]);
+		bp->b_addr = page_address(page);
+		bp->b_flags |= _XBF_PAGES;
 	} else {
-		int retried = 0;
 		unsigned nofs_flag;
 
 		/*
-		 * vm_map_ram() will allocate auxiliary structures (e.g.
+		 * vmalloc() will allocate auxiliary structures (e.g.
 		 * pagetables) with GFP_KERNEL, yet we are likely to be under
 		 * GFP_NOFS context here. Hence we need to tell memory reclaim
 		 * that we are in such a context via PF_MEMALLOC_NOFS to prevent
@@ -422,20 +369,13 @@ xfs_buf_alloc_pages(
 		 * potentially deadlocking.
 		 */
 		nofs_flag = memalloc_nofs_save();
-		do {
-			bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
-						-1);
-			if (bp->b_addr)
-				break;
-			vm_unmap_aliases();
-		} while (retried++ <= 1);
+		bp->b_addr = __vmalloc(BBTOB(bp->b_length), gfp_mask);
 		memalloc_nofs_restore(nofs_flag);
 
 		if (!bp->b_addr) {
 			xfs_warn_ratelimited(bp->b_target->bt_mount,
-				"%s: failed to map %u pages", __func__,
+				"%s: failed to allocate %u pages", __func__,
 				bp->b_page_count);
-			xfs_buf_free_pages(bp);
 			return -ENOMEM;
 		}
 	}
fs/xfs/xfs_buf.h +0 -4
index 2116fed2b53026..b114bfa1b07fb6 100644
@@ -114,8 +114,6 @@ typedef struct xfs_buftarg {
 	struct ratelimit_state	bt_ioerror_rl;
 } xfs_buftarg_t;
 
-#define XB_PAGES	2
-
 struct xfs_buf_map {
 	xfs_daddr_t		bm_bn;	/* block number for I/O */
 	int			bm_len;	/* size of I/O */
@@ -178,8 +176,6 @@ struct xfs_buf {
 	struct xfs_buf_log_item	*b_log_item;
 	struct list_head	b_li_list;	/* Log items list head */
 	struct xfs_trans	*b_transp;
-	struct page		**b_pages;	/* array of page pointers */
-	struct page		*b_page_array[XB_PAGES]; /* inline pages */
 	struct xfs_buf_map	*b_maps;	/* compound buffer map */
 	struct xfs_buf_map	__b_map;	/* inline compound buffer map */
 	int			b_map_count;
Signature · 1 line
2.39.2