nfs: fix -ENOSPC DIO write regression
Subsystems (2)
Jeff Layton
maintainercover nfs: fix -ENOSPC DIO write regression
Boyang reported that xfstest generic/476 would never complete when run against a filesystem that was "too small".
What I found was that we would end up trying to issue a large DIO write that would come back short. The kernel would then follow up and try to write out the rest and get back -ENOSPC. It would then try to issue a commit, which would then try to reissue the writes, and around it would go.
This patchset seems to fix it. Unfortunately, I'm not positive which patch _broke_ this as it seems to have happened quite some time ago.
Jeff Layton (3):
nfs: add new nfs_direct_req tracepoint events
nfs: always check dreq->error after a commit
nfs: only issue commit in DIO codepath if we have uncommitted data
fs/nfs/direct.c | 50 +++++++++-------------------- fs/nfs/internal.h | 33 ++++++++++++++++++++ fs/nfs/nfstrace.h | 69 +++++++++++++++++++++++++++++++++++++++++ fs/nfs/write.c | 48 +++++++++++++++++----------- include/linux/nfs_xdr.h | 1 + 5 files changed, 148 insertions(+), 53 deletions(-)
Signature · 1 line
2.36.1
Jeff Layton
maintainerPATCH 1/3 nfs: add new nfs_direct_req tracepoint events
Add some new tracepoints to the DIO write code.
--- fs/nfs/direct.c | 45 +++++++++---------------------- fs/nfs/internal.h | 33 +++++++++++++++++++++++ fs/nfs/nfstrace.h | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 33 deletions(-)
fs/nfs/direct.c +12 -33
@@ -60,44 +60,12 @@#include "iostat.h" #include "pnfs.h" #include "fscache.h" +#include "nfstrace.h" #define NFSDBG_FACILITY NFSDBG_VFSstatic struct kmem_cache *nfs_direct_cachep; -struct nfs_direct_req { -struct kref kref;/* release manager */ - -/* I/O parameters */ -struct nfs_open_context *ctx;/* file open context info */ -struct nfs_lock_context *l_ctx;/* Lock context info */ -struct kiocb * iocb;/* controlling i/o request */ -struct inode * inode;/* target file of i/o */ - -/* completion state */ -atomic_t io_count;/* i/os we're waiting for */ -spinlock_t lock;/* protect completion state */ - -loff_t io_start;/* Start offset for I/O */ -ssize_t count,/* bytes actually processed */ - max_count,/* max expected count */ - bytes_left,/* bytes left to be sent */ - error;/* any reported error */ -struct completion completion;/* wait for i/o completion */ - -/* commit state */ -struct nfs_mds_commit_info mds_cinfo;/* Storage for cinfo */ -struct pnfs_ds_commit_info ds_cinfo;/* Storage for cinfo */ -struct work_struct work; -int flags; -/* for write */ -#define NFS_ODIRECT_DO_COMMIT (1)/* an unstable reply was received */ -#define NFS_ODIRECT_RESCHED_WRITES (2)/* write verification failed */ -/* for read */ -#define NFS_ODIRECT_SHOULD_DIRTY (3)/* dirty user-space page after read */ -#define NFS_ODIRECT_DONE INT_MAX/* write verification failed */ -}; -static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;static void nfs_direct_write_complete (struct nfs_direct_req *dreq);
@@ -595,6 +563,8 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)struct nfs_page *req;int status = data->task.tk_status; +trace_nfs_direct_commit_complete (dreq); +if (status <0 ) {/* Errors in commit are fatal */ dreq->error = status;
@@ -631,6 +601,8 @@ static void nfs_direct_resched_write(struct nfs_commit_info *cinfo, {struct nfs_direct_req *dreq = cinfo->dreq; +trace_nfs_direct_resched_write (dreq); +spin_lock (&dreq->lock);if (dreq->flags !=NFS_ODIRECT_DONE ) dreq->flags =NFS_ODIRECT_RESCHED_WRITES ;
@@ -695,6 +667,7 @@ static void nfs_direct_write_schedule_work(struct work_struct *work)static void nfs_direct_write_complete (struct nfs_direct_req *dreq) { +trace_nfs_direct_write_complete (dreq);queue_work (nfsiod_workqueue, &dreq->work);/* Calls nfs_direct_write_schedule_work */ }
@@ -705,6 +678,8 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)struct nfs_page *req =nfs_list_entry (hdr->pages.next);int flags =NFS_ODIRECT_DONE ; +trace_nfs_direct_write_completion (dreq); +nfs_init_cinfo_from_dreq (&cinfo, dreq);spin_lock (&dreq->lock);
@@ -759,6 +734,8 @@ static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr) {struct nfs_direct_req *dreq = hdr->dreq; +trace_nfs_direct_write_reschedule_io (dreq); +spin_lock (&dreq->lock);if (dreq->error ==0 ) { dreq->flags =NFS_ODIRECT_RESCHED_WRITES ;
@@ -799,6 +776,8 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,size_t requested_bytes =0 ;size_t wsize =max_t (size_t,NFS_SERVER (inode)->wsize,PAGE_SIZE ); +trace_nfs_direct_write_schedule_iovec (dreq); +nfs_pageio_init_write (&desc, inode, ioflags, false, &nfs_direct_write_completion_ops); desc.pg_dreq = dreq;
fs/nfs/internal.h +33 -0
@@ -859,3 +859,36 @@ static inline void nfs_set_port(struct sockaddr *sap, int *port,rpc_set_port (sap, *port); } + +struct nfs_direct_req { +struct kref kref;/* release manager */ + +/* I/O parameters */ +struct nfs_open_context *ctx;/* file open context info */ +struct nfs_lock_context *l_ctx;/* Lock context info */ +struct kiocb * iocb;/* controlling i/o request */ +struct inode * inode;/* target file of i/o */ + +/* completion state */ +atomic_t io_count;/* i/os we're waiting for */ +spinlock_t lock;/* protect completion state */ + +loff_t io_start;/* Start offset for I/O */ +ssize_t count,/* bytes actually processed */ + max_count,/* max expected count */ + bytes_left,/* bytes left to be sent */ + error;/* any reported error */ +struct completion completion;/* wait for i/o completion */ + +/* commit state */ +struct nfs_mds_commit_info mds_cinfo;/* Storage for cinfo */ +struct pnfs_ds_commit_info ds_cinfo;/* Storage for cinfo */ +struct work_struct work; +int flags; +/* for write */ +#define NFS_ODIRECT_DO_COMMIT (1)/* an unstable reply was received */ +#define NFS_ODIRECT_RESCHED_WRITES (2)/* write verification failed */ +/* for read */ +#define NFS_ODIRECT_SHOULD_DIRTY (3)/* dirty user-space page after read */ +#define NFS_ODIRECT_DONE INT_MAX/* write verification failed */ +};
fs/nfs/nfstrace.h +69 -0
75 lines not on this page · raw message
Signature · 1 line
2.36.1
Jeff Layton
maintainerPATCH 2/3 nfs: always check dreq->error after a commit
In reply to Jeff LaytonWhen the client gets back a short DIO write, it will then attempt to issue another write to finish the DIO request. If that write then fails (as is often the case in an -ENOSPC situation), then we still may need to issue a COMMIT if the earlier short write was unstable. If that COMMIT then succeeds, then we don't want the client to reschedule the write requests, and to instead just return a short write. Otherwise, we can end up looping over the same DIO write forever.
Always consult dreq->error after a successful RPC, even when the flag state is not NFS_ODIRECT_DONE.
--- fs/nfs/direct.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
fs/nfs/direct.c +2 -1
@@ -571,8 +571,9 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) dreq->max_count =0 ; dreq->count =0 ; dreq->flags =NFS_ODIRECT_DONE ; - }else if (dreq->flags ==NFS_ODIRECT_DONE ) + }else { status = dreq->error; + }nfs_init_cinfo_from_dreq (&cinfo, dreq);
Signature · 1 line
2.36.1
Jeff Layton
maintainerPATCH 3/3 nfs: only issue commit in DIO codepath if we have uncommitted data
In reply to Jeff LaytonCurrently, we try to determine whether to issue a commit based on nfs_write_need_commit which looks at the current verifier. In the case where we got a short write and then tried to follow it up with one that failed, the verifier can't be trusted.
What we really want to know is whether the pgio request had any successful writes that came back as UNSTABLE. Add a new flag to the pgio request, and use that to indicate that we've had a successful unstable write. Only issue a commit if that flag is set.
--- fs/nfs/direct.c | 2 +- fs/nfs/write.c | 48 +++++++++++++++++++++++++---------------- include/linux/nfs_xdr.h | 1 + 3 files changed, 32 insertions(+), 19 deletions(-)
fs/nfs/direct.c +1 -1
@@ -690,7 +690,7 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) }nfs_direct_count_bytes (dreq, hdr); -if (hdr->good_bytes !=0 &&nfs_write_need_commit (hdr)) { +if (test_bit (NFS_IOHDR_UNSTABLE_WRITES , &hdr->flags)) {if (!dreq->flags) dreq->flags =NFS_ODIRECT_DO_COMMIT ; flags = dreq->flags;
fs/nfs/write.c +30 -18
55 lines not on this page · raw message
include/linux/nfs_xdr.h +1 -0
@@ -1600,6 +1600,7 @@ enum {NFS_IOHDR_STAT ,NFS_IOHDR_RESEND_PNFS ,NFS_IOHDR_RESEND_MDS , + NFS_IOHDR_UNSTABLE_WRITES, };struct nfs_io_completion ;
Signature · 1 line
2.36.1
Quoted 21 lines · Jeff Layton
Boyang reported that xfstest generic/476 would never complete when run
against a filesystem that was "too small".What I found was that we would end up trying to issue a large DIO write
that would come back short. The kernel would then follow up and try to
write out the rest and get back-ENOSPC. It would then try to issue a commit, which would then try to reissue the writes, and around it would
go.This patchset seems to fix it. Unfortunately, I'm not positive which patch _broke_ this as it seems to have happened quite some time ago.
Jeff Layton (3):
nfs: add newnfs_direct_reqtracepoint events
nfs: always checkdreq->errorafter a commit
nfs: only issue commit in DIO codepath if we have uncommitted data
Quoted 8 lines
fs/nfs/direct.c | 50 +++++++++-------------------- fs/nfs/internal.h | 33 ++++++++++++++++++++ fs/nfs/nfstrace.h | 69
+++++++++++++++++++++++++++++++++++++++++
fs/nfs/write.c | 48 +++++++++++++++++-----------include/linux/nfs_xdr.h| 1 +5 files changed, 148 insertions(+), 53 deletions(-)
With this series applied, I'm seeing things like xfstests generic/013 looping forever.
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, Hammerspace [email protected]
Quoted 35 lines · Trond Myklebust
Quoted 25 lines · Jeff Layton
Boyang reported that xfstest generic/476 would never complete when run
against a filesystem that was "too small".What I found was that we would end up trying to issue a large DIO write
that would come back short. The kernel would then follow up and try to
write out the rest and get back-ENOSPC. It would then try to issue a
commit, which would then try to reissue the writes, and around it would
go.This patchset seems to fix it. Unfortunately, I'm not positive which
patch _broke_ this as it seems to have happened quite some time ago.Jeff Layton (3):
nfs: add newnfs_direct_reqtracepoint events
nfs: always checkdreq->errorafter a commit
nfs: only issue commit in DIO codepath if we have uncommitted dataQuoted 8 lines
fs/nfs/direct.c | 50 +++++++++-------------------- fs/nfs/internal.h | 33 ++++++++++++++++++++ fs/nfs/nfstrace.h | 69
+++++++++++++++++++++++++++++++++++++++++
fs/nfs/write.c | 48 +++++++++++++++++-----------include/linux/nfs_xdr.h| 1 +5 files changed, 148 insertions(+), 53 deletions(-)
Quoted 3 lines
With this series applied, I'm seeing things like xfstests generic/013 looping forever.
Sorry, false alarm... That turned out to be due to an interesting readahead config issue.
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, Hammerspace [email protected]
Hi Jeff,
Thanks for fixing this! I have run some tests against this patchset for days, and the results are all good. generic/476 would complete within 30 mins typically. These tests are:
For verifying generic/476:
xfstests-multihost-nfsv3-over-ext4
xfstests-multihost-nfsv3-over-feature-ext4
xfstests-multihost-nfsv3-over-feature-xfs
xfstests-multihost-nfsv3-over-xfs
xfstests-multihost-nfsv4.0-over-ext4
xfstests-multihost-nfsv4.0-over-feature-ext4
xfstests-multihost-nfsv4.0-over-feature-xfs
xfstests-multihost-nfsv4.0-over-xfs
xfstests-multihost-nfsv4.1-over-ext4
xfstests-multihost-nfsv4.1-over-feature-ext4
xfstests-multihost-nfsv4.1-over-feature-xfs
xfstests-multihost-nfsv4.1-over-xfs
xfstests-multihost-nfsv4.2-over-ext4
xfstests-multihost-nfsv4.2-over-feature-ext4
xfstests-multihost-nfsv4.2-over-feature-xfs
xfstests-multihost-nfsv4.2-over-xfs
xfstests-localhost-nfsv3
xfstests-localhost-nfsv4.0
xfstests-localhost-nfsv4.1
xfstests-localhost-nfsv4.2
Regression tests:
ltp-nfsv{3,4.0,4.1,4.2}
pjd-test: nfs
nfs-connectathon
nfs-sanity-check
All tests were run on x86_64, aarch64, ppc64le, and s390x (part, due to some config issues).
Hope this helps.
Thanks,
Boyang
Quoted 28 lines · Jeff Layton
Boyang reported that xfstest generic/476 would never complete when run against a filesystem that was "too small".
What I found was that we would end up trying to issue a large DIO write that would come back short. The kernel would then follow up and try to write out the rest and get back
-ENOSPC. It would then try to issue a commit, which would then try to reissue the writes, and around it would go.This patchset seems to fix it. Unfortunately, I'm not positive which patch _broke_ this as it seems to have happened quite some time ago.
Jeff Layton (3):
nfs: add newnfs_direct_reqtracepoint events
nfs: always checkdreq->errorafter a commit
nfs: only issue commit in DIO codepath if we have uncommitted datafs/nfs/direct.c | 50 +++++++++-------------------- fs/nfs/internal.h | 33 ++++++++++++++++++++ fs/nfs/nfstrace.h | 69 +++++++++++++++++++++++++++++++++++++++++ fs/nfs/write.c | 48 +++++++++++++++++----------- include/linux/nfs_xdr.h | 1 + 5 files changed, 148 insertions(+), 53 deletions(-)--
2.36.1