Skip to contentSkip to conversation

nfs: fix -ENOSPC DIO write regression

PATCH In review Jeff Layton 3 participants 7 messages Jul 22, 2022 – Aug 4, 2022

Subsystems (2)
lore
Share & tools
lore

Jeff Layton

author maintainer
#0 lore raw

cover nfs: fix -ENOSPC DIO write regression

Boyang reported that xfstest generic/476 would never complete when run against a filesystem that was "too small".

What I found was that we would end up trying to issue a large DIO write that would come back short. The kernel would then follow up and try to write out the rest and get back -ENOSPC. It would then try to issue a commit, which would then try to reissue the writes, and around it would go.

This patchset seems to fix it. Unfortunately, I'm not positive which patch _broke_ this as it seems to have happened quite some time ago.

Jeff Layton (3):
nfs: add new nfs_direct_req tracepoint events
nfs: always check dreq->error after a commit
nfs: only issue commit in DIO codepath if we have uncommitted data

 fs/nfs/direct.c         | 50 +++++++++--------------------
 fs/nfs/internal.h       | 33 ++++++++++++++++++++
 fs/nfs/nfstrace.h       | 69 +++++++++++++++++++++++++++++++++++++++++
 fs/nfs/write.c          | 48 +++++++++++++++++-----------
 include/linux/nfs_xdr.h |  1 +
 5 files changed, 148 insertions(+), 53 deletions(-)
Signature · 1 line
2.36.1
author maintainer
#1 lore raw

PATCH 1/3 nfs: add new nfs_direct_req tracepoint events

Add some new tracepoints to the DIO write code.

---
 fs/nfs/direct.c   | 45 +++++++++----------------------
 fs/nfs/internal.h | 33 +++++++++++++++++++++++
 fs/nfs/nfstrace.h | 69 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 114 insertions(+), 33 deletions(-)
fs/nfs/direct.c +12 -33
index 4eb2a8380a28..ad40e81857ee 100644
@@ -60,44 +60,12 @@
 #include "iostat.h"
 #include "pnfs.h"
 #include "fscache.h"
+#include "nfstrace.h"
 
 #define NFSDBG_FACILITY		NFSDBG_VFS
 
 static struct kmem_cache *nfs_direct_cachep;
 
-struct nfs_direct_req {
-	struct kref		kref;		/* release manager */
-
-	/* I/O parameters */
-	struct nfs_open_context	*ctx;		/* file open context info */
-	struct nfs_lock_context *l_ctx;		/* Lock context info */
-	struct kiocb *		iocb;		/* controlling i/o request */
-	struct inode *		inode;		/* target file of i/o */
-
-	/* completion state */
-	atomic_t		io_count;	/* i/os we're waiting for */
-	spinlock_t		lock;		/* protect completion state */
-
-	loff_t			io_start;	/* Start offset for I/O */
-	ssize_t			count,		/* bytes actually processed */
-				max_count,	/* max expected count */
-				bytes_left,	/* bytes left to be sent */
-				error;		/* any reported error */
-	struct completion	completion;	/* wait for i/o completion */
-
-	/* commit state */
-	struct nfs_mds_commit_info mds_cinfo;	/* Storage for cinfo */
-	struct pnfs_ds_commit_info ds_cinfo;	/* Storage for cinfo */
-	struct work_struct	work;
-	int			flags;
-	/* for write */
-#define NFS_ODIRECT_DO_COMMIT		(1)	/* an unstable reply was received */
-#define NFS_ODIRECT_RESCHED_WRITES	(2)	/* write verification failed */
-	/* for read */
-#define NFS_ODIRECT_SHOULD_DIRTY	(3)	/* dirty user-space page after read */
-#define NFS_ODIRECT_DONE		INT_MAX	/* write verification failed */
-};
-
 static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
 static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
 static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
@@ -595,6 +563,8 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)
 	struct nfs_page *req;
 	int status = data->task.tk_status;
 
+	trace_nfs_direct_commit_complete(dreq);
+
 	if (status < 0) {
 		/* Errors in commit are fatal */
 		dreq->error = status;
@@ -631,6 +601,8 @@ static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
 {
 	struct nfs_direct_req *dreq = cinfo->dreq;
 
+	trace_nfs_direct_resched_write(dreq);
+
 	spin_lock(&dreq->lock);
 	if (dreq->flags != NFS_ODIRECT_DONE)
 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
@@ -695,6 +667,7 @@ static void nfs_direct_write_schedule_work(struct work_struct *work)
 
 static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
 {
+	trace_nfs_direct_write_complete(dreq);
 	queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */
 }
 
@@ -705,6 +678,8 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
 	struct nfs_page *req = nfs_list_entry(hdr->pages.next);
 	int flags = NFS_ODIRECT_DONE;
 
+	trace_nfs_direct_write_completion(dreq);
+
 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
 
 	spin_lock(&dreq->lock);
@@ -759,6 +734,8 @@ static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
 {
 	struct nfs_direct_req *dreq = hdr->dreq;
 
+	trace_nfs_direct_write_reschedule_io(dreq);
+
 	spin_lock(&dreq->lock);
 	if (dreq->error == 0) {
 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
@@ -799,6 +776,8 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
 	size_t requested_bytes = 0;
 	size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
 
+	trace_nfs_direct_write_schedule_iovec(dreq);
+
 	nfs_pageio_init_write(&desc, inode, ioflags, false,
 			      &nfs_direct_write_completion_ops);
 	desc.pg_dreq = dreq;
fs/nfs/internal.h +33 -0
index 8f8cd6e2d4db..24a48ba1ca7e 100644
@@ -859,3 +859,36 @@ static inline void nfs_set_port(struct sockaddr *sap, int *port,
 
 	rpc_set_port(sap, *port);
 }
+
+struct nfs_direct_req {
+	struct kref		kref;		/* release manager */
+
+	/* I/O parameters */
+	struct nfs_open_context	*ctx;		/* file open context info */
+	struct nfs_lock_context *l_ctx;		/* Lock context info */
+	struct kiocb *		iocb;		/* controlling i/o request */
+	struct inode *		inode;		/* target file of i/o */
+
+	/* completion state */
+	atomic_t		io_count;	/* i/os we're waiting for */
+	spinlock_t		lock;		/* protect completion state */
+
+	loff_t			io_start;	/* Start offset for I/O */
+	ssize_t			count,		/* bytes actually processed */
+				max_count,	/* max expected count */
+				bytes_left,	/* bytes left to be sent */
+				error;		/* any reported error */
+	struct completion	completion;	/* wait for i/o completion */
+
+	/* commit state */
+	struct nfs_mds_commit_info mds_cinfo;	/* Storage for cinfo */
+	struct pnfs_ds_commit_info ds_cinfo;	/* Storage for cinfo */
+	struct work_struct	work;
+	int			flags;
+	/* for write */
+#define NFS_ODIRECT_DO_COMMIT		(1)	/* an unstable reply was received */
+#define NFS_ODIRECT_RESCHED_WRITES	(2)	/* write verification failed */
+	/* for read */
+#define NFS_ODIRECT_SHOULD_DIRTY	(3)	/* dirty user-space page after read */
+#define NFS_ODIRECT_DONE		INT_MAX	/* write verification failed */
+};
fs/nfs/nfstrace.h +69 -0

75 lines not on this page · raw message

Signature · 1 line
2.36.1
author maintainer
#2 lore raw

PATCH 2/3 nfs: always check dreq->error after a commit

In reply to Jeff Layton

When the client gets back a short DIO write, it will then attempt to issue another write to finish the DIO request. If that write then fails (as is often the case in an -ENOSPC situation), then we still may need to issue a COMMIT if the earlier short write was unstable. If that COMMIT then succeeds, then we don't want the client to reschedule the write requests, and to instead just return a short write. Otherwise, we can end up looping over the same DIO write forever.

Always consult dreq->error after a successful RPC, even when the flag state is not NFS_ODIRECT_DONE.

---
 fs/nfs/direct.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
fs/nfs/direct.c +2 -1
index ad40e81857ee..a47d13296194 100644
@@ -571,8 +571,9 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)
 		dreq->max_count = 0;
 		dreq->count = 0;
 		dreq->flags = NFS_ODIRECT_DONE;
-	} else if (dreq->flags == NFS_ODIRECT_DONE)
+	} else {
 		status = dreq->error;
+	}
 
 	nfs_init_cinfo_from_dreq(&cinfo, dreq);
 
Signature · 1 line
2.36.1
author maintainer
#3 lore raw

PATCH 3/3 nfs: only issue commit in DIO codepath if we have uncommitted data

In reply to Jeff Layton

Currently, we try to determine whether to issue a commit based on nfs_write_need_commit which looks at the current verifier. In the case where we got a short write and then tried to follow it up with one that failed, the verifier can't be trusted.

What we really want to know is whether the pgio request had any successful writes that came back as UNSTABLE. Add a new flag to the pgio request, and use that to indicate that we've had a successful unstable write. Only issue a commit if that flag is set.

---
 fs/nfs/direct.c         |  2 +-
 fs/nfs/write.c          | 48 +++++++++++++++++++++++++----------------
 include/linux/nfs_xdr.h |  1 +
 3 files changed, 32 insertions(+), 19 deletions(-)
fs/nfs/direct.c +1 -1
index a47d13296194..86df66bb14c5 100644
@@ -690,7 +690,7 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
 	}
 
 	nfs_direct_count_bytes(dreq, hdr);
-	if (hdr->good_bytes != 0 && nfs_write_need_commit(hdr)) {
+	if (test_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags)) {
 		if (!dreq->flags)
 			dreq->flags = NFS_ODIRECT_DO_COMMIT;
 		flags = dreq->flags;
fs/nfs/write.c +30 -18

55 lines not on this page · raw message

include/linux/nfs_xdr.h +1 -0
index 0e3aa0f5f324..e86cf6642d21 100644
@@ -1600,6 +1600,7 @@ enum {
 	NFS_IOHDR_STAT,
 	NFS_IOHDR_RESEND_PNFS,
 	NFS_IOHDR_RESEND_MDS,
+	NFS_IOHDR_UNSTABLE_WRITES,
 };
 
 struct nfs_io_completion;
Signature · 1 line
2.36.1
#4 lore raw
In reply to Jeff Layton
Quoted 21 lines · Jeff Layton

Boyang reported that xfstest generic/476 would never complete when run
against a filesystem that was "too small".

What I found was that we would end up trying to issue a large DIO write
that would come back short. The kernel would then follow up and try to
write out the rest and get back -ENOSPC. It would then try to issue a commit, which would then try to reissue the writes, and around it would
go.

This patchset seems to fix it. Unfortunately, I'm not positive which patch _broke_ this as it seems to have happened quite some time ago.

Jeff Layton (3):
  nfs: add new nfs_direct_req tracepoint events
  nfs: always check dreq->error after a commit
  nfs: only issue commit in DIO codepath if we have uncommitted data

Quoted 8 lines

 fs/nfs/direct.c         | 50 +++++++++-------------------- fs/nfs/internal.h       | 33 ++++++++++++++++++++ fs/nfs/nfstrace.h       | 69
+++++++++++++++++++++++++++++++++++++++++
 fs/nfs/write.c          | 48 +++++++++++++++++----------- include/linux/nfs_xdr.h |  1 +

 5 files changed, 148 insertions(+), 53 deletions(-)

With this series applied, I'm seeing things like xfstests generic/013 looping forever.

Signature · 3 lines
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]
#5 lore raw
Quoted 35 lines · Trond Myklebust
Quoted 25 lines · Jeff Layton

Boyang reported that xfstest generic/476 would never complete when run
against a filesystem that was "too small".

What I found was that we would end up trying to issue a large DIO write
that would come back short. The kernel would then follow up and try to
write out the rest and get back -ENOSPC. It would then try to issue a
commit, which would then try to reissue the writes, and around it would
go.

This patchset seems to fix it. Unfortunately, I'm not positive which
patch _broke_ this as it seems to have happened quite some time ago.

Jeff Layton (3):
  nfs: add new nfs_direct_req tracepoint events
  nfs: always check dreq->error after a commit
  nfs: only issue commit in DIO codepath if we have uncommitted data

Quoted 8 lines

 fs/nfs/direct.c         | 50 +++++++++-------------------- fs/nfs/internal.h       | 33 ++++++++++++++++++++ fs/nfs/nfstrace.h       | 69
+++++++++++++++++++++++++++++++++++++++++
 fs/nfs/write.c          | 48 +++++++++++++++++----------- include/linux/nfs_xdr.h |  1 +

 5 files changed, 148 insertions(+), 53 deletions(-)
Quoted 3 lines

With this series applied, I'm seeing things like xfstests generic/013 looping forever.

Sorry, false alarm... That turned out to be due to an interesting readahead config issue.

Signature · 3 lines
Trond Myklebust
Linux NFS client maintainer, Hammerspace
[email protected]
#6 lore raw
In reply to Jeff Layton

Hi Jeff,

Thanks for fixing this! I have run some tests against this patchset for days, and the results are all good. generic/476 would complete within 30 mins typically. These tests are:

For verifying generic/476:
xfstests-multihost-nfsv3-over-ext4
xfstests-multihost-nfsv3-over-feature-ext4
xfstests-multihost-nfsv3-over-feature-xfs
xfstests-multihost-nfsv3-over-xfs
xfstests-multihost-nfsv4.0-over-ext4
xfstests-multihost-nfsv4.0-over-feature-ext4
xfstests-multihost-nfsv4.0-over-feature-xfs
xfstests-multihost-nfsv4.0-over-xfs
xfstests-multihost-nfsv4.1-over-ext4
xfstests-multihost-nfsv4.1-over-feature-ext4
xfstests-multihost-nfsv4.1-over-feature-xfs
xfstests-multihost-nfsv4.1-over-xfs
xfstests-multihost-nfsv4.2-over-ext4
xfstests-multihost-nfsv4.2-over-feature-ext4
xfstests-multihost-nfsv4.2-over-feature-xfs
xfstests-multihost-nfsv4.2-over-xfs
xfstests-localhost-nfsv3
xfstests-localhost-nfsv4.0
xfstests-localhost-nfsv4.1
xfstests-localhost-nfsv4.2

Regression tests:
ltp-nfsv{3,4.0,4.1,4.2}
pjd-test: nfs
nfs-connectathon
nfs-sanity-check

All tests were run on x86_64, aarch64, ppc64le, and s390x (part, due to some config issues).

Hope this helps.

Thanks,
Boyang

Quoted 28 lines · Jeff Layton

Boyang reported that xfstest generic/476 would never complete when run against a filesystem that was "too small".

What I found was that we would end up trying to issue a large DIO write that would come back short. The kernel would then follow up and try to write out the rest and get back -ENOSPC. It would then try to issue a commit, which would then try to reissue the writes, and around it would go.

This patchset seems to fix it. Unfortunately, I'm not positive which patch _broke_ this as it seems to have happened quite some time ago.

Jeff Layton (3):
nfs: add new nfs_direct_req tracepoint events
nfs: always check dreq->error after a commit
nfs: only issue commit in DIO codepath if we have uncommitted data

 fs/nfs/direct.c         | 50 +++++++++--------------------
 fs/nfs/internal.h       | 33 ++++++++++++++++++++
 fs/nfs/nfstrace.h       | 69 +++++++++++++++++++++++++++++++++++++++++
 fs/nfs/write.c          | 48 +++++++++++++++++-----------
 include/linux/nfs_xdr.h |  1 +
 5 files changed, 148 insertions(+), 53 deletions(-)

--
2.36.1