fs: range-based admission for generation-drained DIO

122 lines not on this page · raw message

index 683c1841e640..c862b9a01c78 100644
@@ -20,7 +20,7 @@ xfs_can_free_cowblocks(struct xfs_inode *ip)
 	if ((inode_state_read_once(inode) & I_DIRTY_PAGES) ||
 	    mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) ||
 	    mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
-	    atomic_read(&inode->i_dio_count))
+	    !inode_dio_finished(inode))
 		return false;
 	return true;
 }

116 lines not on this page · raw message

index 50d3c92709c8..cb6d6722c18e 100644
@@ -91,7 +91,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		return generic_file_read_iter(iocb, to);
 	}
 
-	ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin);
+	ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin, 0);
 	if (ret == -ENOTBLK)
 		ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);
 	inode_unlock_shared(inode);

index ad041ac3252a..fffe6702c42f 100644
@@ -149,7 +149,8 @@ ssize_t iomap_dio_complete(struct iomap_dio *dio)
 	    !(dio->flags & IOMAP_DIO_NO_INVALIDATE))
 		kiocb_invalidate_post_direct_write(iocb, dio->size);
 
-	inode_dio_end(file_inode(iocb->ki_filp));
+	inode_dio_end_generation(file_inode(iocb->ki_filp),
+				dio->flags & IOMAP_DIO_GENERATION);
 
 	if (ret > 0) {
 		iocb->ki_pos += ret;

@@ -690,6 +691,12 @@ static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
  * the correct total number of bytes transferred; this is essential for
  * completing partial requests asynchronously.
  *
+ * When @dio_flags includes IOMAP_DIO_ACCOUNTED, iomap takes ownership of the
+ * caller's registration from inode_dio_try_begin() on entry, with its saved
+ * generation token supplied in IOMAP_DIO_GENERATION. Iomap releases the
+ * registration on completion or any early error or NULL return, including
+ * -ENOTBLK. Callers retrying the request must acquire a fresh registration.
+ *
  * Returns -ENOTBLK In case of a page invalidation invalidation failure for
  * writes.  The callers needs to fall back to buffered I/O in this case.
  */

@@ -708,6 +715,8 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 	};
 	bool wait_for_completion =
 		is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT);
+	bool accounted = dio_flags & IOMAP_DIO_ACCOUNTED;
+	bool generation = dio_flags & IOMAP_DIO_GENERATION;
 	struct blk_plug plug;
 	struct iomap_dio *dio;
 	loff_t ret = 0;

@@ -715,11 +724,13 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 	trace_iomap_dio_rw_begin(iocb, iter, dio_flags, done_before);
 
 	if (!iomi.len)
-		return NULL;
+		goto out_account;
 
 	dio = kmalloc_obj(*dio);
-	if (!dio)
-		return ERR_PTR(-ENOMEM);
+	if (!dio) {
+		ret = -ENOMEM;
+		goto out_account;
+	}
 
 	dio->iocb = iocb;
 	atomic_set(&dio->ref, 1);

@@ -816,7 +827,10 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 			goto out_free_dio;
 	}
 
-	inode_dio_begin(inode);
+	if (!accounted)
+		generation = inode_dio_begin_generation(inode);
+	if (generation)
+		dio->flags |= IOMAP_DIO_GENERATION;
 
 	blk_start_plug(&plug);
 	while ((ret = iomap_iter(&iomi, ops)) > 0) {

@@ -901,6 +915,9 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 
 out_free_dio:
 	kfree(dio);
+out_account:
+	if (accounted)
+		inode_dio_end_generation(inode, generation);
 	if (ret)
 		return ERR_PTR(ret);
 	return NULL;

@@ -943,6 +960,7 @@ static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr)
 	struct bio *bio = &sr->bio;
 	struct kiocb *iocb = sr->iocb;
 	struct inode *inode = file_inode(iocb->ki_filp);
+	bool generation = sr->dio_flags & IOMAP_DIO_GENERATION;
 	ssize_t ret;
 
 	if (unlikely(bio->bi_status)) {

@@ -962,7 +980,7 @@ static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr)
 		bio_release_pages(bio, false);
 		bio_put(bio);
 	}
-	inode_dio_end(inode);
+	inode_dio_end_generation(inode, generation);
 	trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret);
 	return ret;
 }

@@ -980,7 +998,7 @@ static void iomap_dio_simple_end_io(struct bio *bio)
 }
 
 ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter,
-		struct iomap_iter *iomi)
+		struct iomap_iter *iomi, bool generation)
 {
 	gfp_t gfp = (iomi->flags & IOMAP_NOWAIT) ? GFP_NOWAIT : GFP_KERNEL;
 	struct iomap_dio_simple *sr;

@@ -1019,7 +1037,7 @@ ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter,
 	}
 	sr = container_of(bio, struct iomap_dio_simple, bio);
 	sr->iocb = iocb;
-	sr->dio_flags = 0;
+	sr->dio_flags = generation ? IOMAP_DIO_GENERATION : 0;
 
 	bio->bi_iter.bi_sector = iomap_sector(&iomi->iomap, iomi->pos);
 	bio->bi_ioprio = iocb->ki_ioprio;

@@ -1064,7 +1082,7 @@ ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter,
 out_bio_put:
 	bio_put(bio);
 out_dio_end:
-	inode_dio_end(iomi->inode);
+	inode_dio_end_generation(iomi->inode, generation);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(__iomap_dio_read_simple);

index 7d0c8a197575..124a037cfa9f 100644
@@ -253,7 +253,7 @@ xfs_file_dio_read(
 		ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops,
 				&xfs_dio_read_bounce_ops, 0, NULL, 0);
 	} else {
-		ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin);
+		ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin, 0);
 		if (ret == -ENOTBLK)
 			ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL,
 					0, NULL, 0);

99 lines not on this page · raw message

index 7b027547b305..b31d5ac5bb96 100644
@@ -23,6 +23,9 @@
 #include <linux/rw_hint.h>
 #include <linux/seq_file.h>
 #include <linux/debugfs.h>
+#if IS_ENABLED(CONFIG_DIO_DRAIN_KUNIT_TEST)
+#include <kunit/static_stub.h>
+#endif
 #include <trace/events/writeback.h>
 #define CREATE_TRACE_POINTS
 #include <trace/events/timestamp.h>

@@ -2783,6 +2786,16 @@ bool inode_dio_finished(const struct inode *inode)
 }
 EXPORT_SYMBOL(inode_dio_finished);
 
+#if IS_ENABLED(CONFIG_DIO_DRAIN_KUNIT_TEST)
+bool inode_dio_begin_generation(struct inode *inode)
+{
+	KUNIT_STATIC_STUB_REDIRECT(inode_dio_begin_generation, inode);
+	return atomic_fetch_inc(&inode->i_dio_count) &
+		INODE_DIO_GENERATION;
+}
+EXPORT_SYMBOL_GPL(inode_dio_begin_generation);
+#endif
+
 /*
  * Allocated on the first draining operation, never on ordinary DIO. The
  * filesystem owns the pointer and retains the allocation until inode reclaim.

index fffe6702c42f..2c840358abc8 100644
@@ -13,6 +13,9 @@
 #include <linux/init.h>
 #include "internal.h"
 #include "trace.h"
+#if IS_ENABLED(CONFIG_DIO_DRAIN_KUNIT_TEST)
+#include <kunit/static_stub.h>
+#endif
 
 #include "../internal.h"
 

@@ -674,6 +677,21 @@ static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
 	}
 }
 
+#if IS_ENABLED(CONFIG_DIO_DRAIN_KUNIT_TEST)
+/* Fault injection is scoped to the task running the ownership test. */
+struct iomap_dio *iomap_dio_alloc(void)
+{
+	KUNIT_STATIC_STUB_REDIRECT(iomap_dio_alloc);
+	return kmalloc_obj(struct iomap_dio);
+}
+EXPORT_SYMBOL_GPL(iomap_dio_alloc);
+#else
+static inline struct iomap_dio *iomap_dio_alloc(void)
+{
+	return kmalloc_obj(struct iomap_dio);
+}
+#endif
+
 /*
  * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO
  * is being issued as AIO or not.  This allows us to optimise pure data writes

@@ -726,7 +744,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 	if (!iomi.len)
 		goto out_account;
 
-	dio = kmalloc_obj(*dio);
+	dio = iomap_dio_alloc();
 	if (!dio) {
 		ret = -ENOMEM;
 		goto out_account;

893 lines not on this page · raw message

index 016f621345af..3eb62a40ea21 100644
@@ -3222,11 +3222,16 @@ static inline bool inode_dio_drain_overlap(struct inode *inode,
 	return range.active && last >= range.start && pos <= range.last;
 }
 
+/* Test builds need a single function address for the registration stub. */
+#if IS_ENABLED(CONFIG_DIO_DRAIN_KUNIT_TEST)
+bool inode_dio_begin_generation(struct inode *inode);
+#else
 static inline bool inode_dio_begin_generation(struct inode *inode)
 {
 	return atomic_fetch_inc(&inode->i_dio_count) &
 		INODE_DIO_GENERATION;
 }
+#endif
 
 static inline void inode_dio_end_generation(struct inode *inode, bool generation)
 {

index 611c37c19f82..f6652f5d8ac5 100644
@@ -723,6 +723,10 @@ struct iomap_dio_ops {
 ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 		const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
 		unsigned int dio_flags, void *private, size_t done_before);
+#if IS_ENABLED(CONFIG_DIO_DRAIN_KUNIT_TEST)
+struct iomap_dio *iomap_dio_alloc(void);
+#endif
+
 struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 		const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
 		unsigned int dio_flags, void *private, size_t done_before);

190 lines not on this page · raw message

new file mode 100644
index 000000000000..178b668580d8
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __XFS_DIO_TEST_H__
+#define __XFS_DIO_TEST_H__
+
+struct inode;
+struct dentry;
+
+enum xfs_dio_test_event {
+	XFS_DIO_TEST_BATCH_LEADER,
+	XFS_DIO_TEST_BATCH_FOLLOWER,
+	XFS_DIO_TEST_BATCH_COLLECT,
+	XFS_DIO_TEST_BATCH_ZERO,
+	XFS_DIO_TEST_WAIT_UNLOCKED,
+	XFS_DIO_TEST_WRITE_CHECKS,
+	XFS_DIO_TEST_WRITE_ADMIT,
+	XFS_DIO_TEST_WRITE_REJECT,
+	XFS_DIO_TEST_UNALIGNED_RECHECK,
+	XFS_DIO_TEST_REMAP_ACTIVE,
+	XFS_DIO_TEST_REMAP_UPGRADE,
+	XFS_DIO_TEST_NR_EVENTS,
+};
+
+#ifdef CONFIG_XFS_DIO_TEST
+void xfs_dio_test_init(struct dentry *parent);
+void xfs_dio_test_event(struct inode *inode, enum xfs_dio_test_event event,
+		unsigned long value);
+#else
+static inline void xfs_dio_test_init(struct dentry *parent) { }
+static inline void xfs_dio_test_event(struct inode *inode,
+		enum xfs_dio_test_event event, unsigned long value) { }
+#endif
+#endif

266 lines not on this page · raw message

index de8be344e987..e26df6894618 100644
@@ -110,6 +110,7 @@ xfs_inode_alloc(
 	ip->i_mount = mp;
 	memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
 	ip->i_cowfp = NULL;
+	ip->i_dio_drain = NULL;
 	memset(&ip->i_af, 0, sizeof(ip->i_af));
 	ip->i_af.if_format = XFS_DINODE_FMT_EXTENTS;
 	memset(&ip->i_df, 0, sizeof(ip->i_df));

@@ -156,6 +157,7 @@ xfs_inode_free_callback(
 		xfs_inode_item_destroy(ip);
 	}
 
+	kfree(ip->i_dio_drain);
 	kmem_cache_free(xfs_inode_cache, ip);
 }
 

index 75eff68c1e39..04907a7b431f 100644
@@ -41,6 +41,7 @@ typedef struct xfs_inode {
 	struct xfs_inode_log_item *i_itemp;	/* logging information */
 	struct rw_semaphore	i_lock;		/* inode lock */
 	atomic_t		i_pincount;	/* inode pin count */
+	struct inode_dio_drain	*i_dio_drain; /* lazily allocated DIO drain */
 	struct llist_node	i_gclist;	/* deferred inactivation list */
 
 	/*

index d307ec97fac8..dec2139c0c6f 100644
@@ -1278,6 +1278,14 @@ xfs_vn_tmpfile(
 	return finish_open_simple(file, err);
 }
 
+static void
+xfs_vn_dio_drain_complete(
+	struct inode		*inode)
+{
+	/* The generation cutover orders i_dio_drain initialization before this callback. */
+	inode_dio_drain_complete(inode, READ_ONCE(XFS_I(inode)->i_dio_drain));
+}
+
 static const struct inode_operations xfs_inode_operations = {
 	.get_inode_acl		= xfs_get_acl,
 	.set_acl		= xfs_set_acl,

@@ -1289,6 +1297,7 @@ static const struct inode_operations xfs_inode_operations = {
 	.sync_lazytime		= xfs_vn_sync_lazytime,
 	.fileattr_get		= xfs_fileattr_get,
 	.fileattr_set		= xfs_fileattr_set,
+	.dio_drain_complete	= xfs_vn_dio_drain_complete,
 };
 
 static const struct inode_operations xfs_dir_inode_operations = {

index 104542b10d9e..3e1b7d628477 100644
@@ -12,6 +12,7 @@
 #include "xfs_mount.h"
 #include "xfs_defer.h"
 #include "xfs_inode.h"
+#include "xfs_dio_test.h"
 #include "xfs_trans.h"
 #include "xfs_bmap.h"
 #include "xfs_bmap_util.h"

@@ -1763,6 +1764,7 @@ xfs_reflink_remap_prep(
 	xfs_iflags_set(src, XFS_IREMAPPING);
 	if (inode_in != inode_out)
 		xfs_ilock_demote(src, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL);
+	xfs_dio_test_event(inode_in, XFS_DIO_TEST_REMAP_ACTIVE, 0);
 
 	return 0;
 out_unlock:

index 5a06132aa384..a7e84ddd145b 100644
@@ -12,6 +12,7 @@
 #include "xfs_sb.h"
 #include "xfs_mount.h"
 #include "xfs_inode.h"
+#include "xfs_dio_test.h"
 #include "xfs_btree.h"
 #include "xfs_bmap.h"
 #include "xfs_alloc.h"

@@ -2677,6 +2678,7 @@ init_xfs_fs(void)
 		goto out_cleanup_procfs;
 
 	xfs_debugfs = xfs_debugfs_mkdir("xfs", NULL);
+	xfs_dio_test_init(xfs_debugfs);
 
 	xfs_kset = kset_create_and_add("xfs", NULL, fs_kobj);
 	if (!xfs_kset) {

index 865aec3ae5ab..30759c6f94d3 100644
@@ -889,8 +889,8 @@ xfs_free_file_space(
 		return 0;
 
 	/*
-	 * Now AIO and DIO has drained we flush and (if necessary) invalidate
-	 * the cached range over the first operation we are about to run.
+	 * The caller has excluded and drained DIO over the affected range.
+	 * Flush and invalidate the cached range before removing its mappings.
 	 */
 	error = xfs_flush_unmap_range(ip, offset, len);
 	if (error)

@@ -917,6 +917,11 @@ xfs_free_file_space(
 		}
 	}
 
+	/* An aligned data-device punch needs no partial-block zeroing. */
+	if (!XFS_IS_REALTIME_INODE(ip) &&
+	    !((offset | len) & mp->m_blockmask))
+		return 0;
+
 	/*
 	 * Now that we've unmap all full blocks we'll have to zero out any
 	 * partial block at the beginning and/or end.  xfs_zero_range is smart

100 lines not on this page · raw message

index 05a14da28031..b8119abab30a 100644
@@ -118,6 +118,16 @@ xfs_lock_flags_assert(
  *
  * i_rwsem -> invalidate_lock -> page_lock -> i_ilock
  *
+ * DIO range drainers add an outer mutex:
+ *
+ * i_dio_drain->lock -> i_rwsem -> invalidate_lock -> page_lock -> i_ilock
+ *
+ * Never wait for the drain mutex while holding IOLOCK: its owner may drop
+ * shared IOLOCK and reacquire it exclusively during write checks. A waiter
+ * holding shared IOLOCK would prevent that upgrade and deadlock both drainers.
+ * The mutex stays held through exclusion removal, including extent freeing
+ * and any synchronous log force. DIO admission and completion do not take it.
+ *
  * mmap_lock locking order:
  *
  * i_rwsem -> page lock -> mmap_lock

380 lines not on this page · raw message

371 lines not on this page · raw message

391 lines not on this page · raw message

index ad83328b7350..53b3d6266e03 100644
@@ -453,6 +453,55 @@ xfs_file_splice_read(
 	return ret;
 }
 
+/*
+ * Serialize zeroing with I/O to all cache folios that can cover the old EOF
+ * block and gap, not unrelated requests outside those folios. Clear the
+ * exclusion before submitting this write, which can share the final folio.
+ * Drain the finite cohort admitted before the generation cutover, including
+ * COW end-I/O remapping. New unrelated requests can proceed under shared IOLOCK.
+ */
+static int
+xfs_file_write_zero_eof_range(
+	struct kiocb		*iocb,
+	loff_t			isize,
+	unsigned int		*iolock,
+	struct inode_dio_drain	*drain)
+{
+	struct inode		*inode = file_inode(iocb->ki_filp);
+	struct xfs_inode	*ip = XFS_I(inode);
+	loff_t			granularity = max_t(loff_t, i_blocksize(inode),
+					mapping_max_folio_size(inode->i_mapping));
+	loff_t			start = round_down(isize, granularity);
+	loff_t			last = (iocb->ki_pos - 1) | (granularity - 1);
+	int			error = 0;
+
+	error = kiocb_modified(iocb);
+	if (error)
+		return error;
+	if (*iolock == XFS_IOLOCK_EXCL) {
+		xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
+		*iolock = XFS_IOLOCK_SHARED;
+	}
+	/* Folio invalidation is wider than zeroing, even for sub-page blocks. */
+	error = inode_dio_drain_start(inode, drain, start, last);
+	if (error)
+		return error;
+	inode_dio_drain_wait(inode, drain);
+	spin_lock(&ip->i_flags_lock);
+	isize = i_size_read(inode);
+	spin_unlock(&ip->i_flags_lock);
+	if (iocb->ki_pos > isize) {
+		trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize);
+		xfs_ilock(ip, XFS_MMAPLOCK_EXCL);
+		error = iomap_zero_range(inode, isize, iocb->ki_pos - isize,
+				NULL, &xfs_buffered_write_iomap_ops,
+				&xfs_iomap_write_ops, NULL);
+		xfs_iunlock(ip, XFS_MMAPLOCK_EXCL);
+	}
+	inode_dio_drain_finish(inode, drain);
+	return error;
+}
+
 /*
  * Take care of zeroing post-EOF blocks when they might exist.
  *

@@ -467,20 +516,17 @@ xfs_file_write_zero_eof(
 	unsigned int		*iolock,
 	size_t			count,
 	bool			*drained_dio,
-	struct xfs_zone_alloc_ctx *ac)
+	struct xfs_zone_alloc_ctx *ac,
+	struct inode_dio_drain	*drain)
 {
 	struct xfs_inode	*ip = XFS_I(iocb->ki_filp->f_mapping->host);
 	loff_t			isize;
 	int			error;
 
 	/*
-	 * We need to serialise against EOF updates that occur in IO completions
-	 * here. We want to make sure that nobody is changing the size while
-	 * we do this check until we have placed an IO barrier (i.e. hold
-	 * XFS_IOLOCK_EXCL) that prevents new IO from being dispatched.  The
-	 * spinlock effectively forms a memory barrier once we have
-	 * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and
-	 * hence be able to correctly determine if we need to run zeroing.
+	 * Pair with EOF updates in DIO completion. The IOLOCK excludes truncate,
+	 * so EOF can only move forwards. Recheck after establishing either a
+	 * range exclusion or the exclusive IOLOCK and draining the relevant I/O.
 	 */
 	spin_lock(&ip->i_flags_lock);
 	isize = i_size_read(VFS_I(ip));

@@ -493,6 +539,9 @@ xfs_file_write_zero_eof(
 	if (iocb->ki_flags & IOCB_NOWAIT)
 		return -EAGAIN;
 
+	if (drain)
+		return xfs_file_write_zero_eof_range(iocb, isize, iolock, drain);
+
 	if (!*drained_dio) {
 		/*
 		 * If zeroing is needed and we are currently holding the iolock

@@ -543,23 +592,49 @@ xfs_file_write_checks(
 	struct xfs_zone_alloc_ctx *ac)
 {
 	struct inode		*inode = iocb->ki_filp->f_mapping->host;
+	struct inode_dio_drain	*drain = NULL;
 	size_t			count = iov_iter_count(from);
 	bool			drained_dio = false;
 	ssize_t			error;
 
+	/*
+	 * Another drainer may need IOLOCK_EXCL during write checks. Do not pin
+	 * shared IOLOCK while waiting for its drain mutex.
+	 */
+	if (*iolock == XFS_IOLOCK_SHARED &&
+	    (iocb->ki_flags & IOCB_DIRECT) &&
+	    !(iocb->ki_flags & IOCB_ATOMIC) &&
+	    !XFS_IS_REALTIME_INODE(XFS_I(inode)) &&
+	    IS_ALIGNED(iocb->ki_pos | count, i_blocksize(inode)) &&
+	    iocb->ki_pos > i_size_read(inode)) {
+		if (iocb->ki_flags & IOCB_NOWAIT)
+			return -EAGAIN;
+		xfs_iunlock(XFS_I(inode), *iolock);
+		drain = inode_dio_drain_get(&XFS_I(inode)->i_dio_drain);
+		if (IS_ERR(drain)) {
+			xfs_ilock(XFS_I(inode), *iolock);
+			return PTR_ERR(drain);
+		}
+		mutex_lock(&drain->lock);
+		error = xfs_ilock_iocb_for_write(iocb, iolock);
+		if (error) {
+			*iolock = 0;
+			goto out;
+		}
+	}
 restart:
 	/* Normalize append position and enforce limits before range admission. */
 	error = generic_write_checks(iocb, from);
 	xfs_dio_test_event(inode, XFS_DIO_TEST_WRITE_CHECKS,
 			iov_iter_count(from));
 	if (error <= 0)
-		return error;
+		goto out;
 
 	if (iocb->ki_flags & IOCB_DIRECT) {
 		error = xfs_dio_range_wait(iocb, iov_iter_count(from), iolock,
 				true);
 		if (error < 0)
-			return error;
+			goto out;
 		if (error) {
 			iov_iter_reexpand(from, count);
 			/* EOF and limits can change while IOLOCK is dropped. */

@@ -576,7 +651,7 @@ xfs_file_write_checks(
 	}
 
 	if (error)
-		return error;
+		goto out;
 
 	/*
 	 * For changing security info in file_remove_privs() we need i_rwsem

@@ -588,7 +663,7 @@ xfs_file_write_checks(
 		error = xfs_ilock_iocb(iocb, *iolock);
 		if (error) {
 			*iolock = 0;
-			return error;
+			goto out;
 		}
 		goto restart;
 	}

@@ -605,14 +680,18 @@ xfs_file_write_checks(
 	 */
 	if (iocb->ki_pos > i_size_read(inode)) {
 		error = xfs_file_write_zero_eof(iocb, from, iolock, count,
-				&drained_dio, ac);
+				&drained_dio, ac, drain);
 		if (error == 1)
 			goto restart;
 		if (error)
-			return error;
+			goto out;
 	}
 
-	return kiocb_modified(iocb);
+	error = kiocb_modified(iocb);
+out:
+	if (drain)
+		mutex_unlock(&drain->lock);
+	return error;
 }
 
 static ssize_t

96 lines not on this page · raw message