pNFS: recoalesce pages when ld read/write fails

PATCH In review Peng Tao 1 participant 4 messages Aug 6, 2011 – Aug 8, 2011

Subsystems (2)

NFS client NFS server

lore

Share & tools

lore

Peng Tao

author

2011-08-06 15:52 UTC #0 lore raw

cover pNFS: recoalesce pages when ld read/write fails

Hi,

I start to look at the error handling for ld_read/write_done. The two patches try to address the error handling for read/write pagelist failures, via nfs readpage/writepage interface. Please see if this method is proper. Thanks!

Cheers,
Tao

Peng Tao

author

2011-08-06 15:53 UTC #1 lore raw

PATCH 1/2 pNFS: deal with ld write failure

For pnfs pagelist write failure, we need to pg_recoalesce and resend IO to mds.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/internal.h |    4 ++++
 fs/nfs/pnfs.c     |   35 ++++++++++++++++++++++++++++++++---
 fs/nfs/write.c    |   21 ++++++++++++++-------
 3 files changed, 50 insertions(+), 10 deletions(-)

fs/nfs/internal.h +4 -0

index ab12913..62f183d 100644
@@ -305,6 +305,10 @@ extern void nfs_readdata_release(struct nfs_read_data *rdata);
 /* write.c */
 extern int nfs_generic_flush(struct nfs_pageio_descriptor *desc,
 		struct list_head *head);
+extern int do_nfs_writepage(struct page *page, struct writeback_control *wbc,
+		struct nfs_pageio_descriptor *pgio);
+extern void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
+		struct inode *inode, int ioflags);
 extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio);
 extern void nfs_writedata_release(struct nfs_write_data *wdata);
 extern void nfs_commit_free(struct nfs_write_data *p);

fs/nfs/pnfs.c +32 -3

index e550e88..08aba45 100644
@@ -1172,6 +1172,13 @@ int
 pnfs_ld_write_done(struct nfs_write_data *data)
 {
 	int status;
+	struct nfs_pageio_descriptor pgio;
+	struct writeback_control wbc = {
+		.sync_mode = WB_SYNC_ALL,
+		.range_start = data->mds_offset,
+		.nr_to_write = data->npages,
+		.range_end = LLONG_MAX,
+	};
 
 	if (!data->pnfs_error) {
 		pnfs_set_layoutcommit(data);

@@ -1180,11 +1187,33 @@ pnfs_ld_write_done(struct nfs_write_data *data)
 		return 0;
 	}
 
+	put_lseg(data->lseg);
+	data->lseg = NULL;
 	dprintk("%s: pnfs_error=%d, retry via MDS\n", __func__,
 		data->pnfs_error);
-	status = nfs_initiate_write(data, NFS_CLIENT(data->inode),
-				    data->mds_ops, NFS_FILE_SYNC);
-	return status ? : -EAGAIN;
+	nfs_pageio_init_write_mds(&pgio, data->inode, FLUSH_STABLE);
+	pgio.pg_recoalesce = 1;
+	while (!list_empty(&data->pages)) {
+		struct nfs_page *req = nfs_list_entry(data->pages.next);
+		struct page *page = req->wb_page;
+
+		nfs_list_remove_request(req);
+		nfs_clear_page_tag_locked(req);
+
+		end_page_writeback(page);
+
+		lock_page(page);
+		status = do_nfs_writepage(page, &wbc, &pgio);
+		if (status) {
+			/* FIXME: is this enough?? */
+			set_page_dirty(page);
+		}
+		unlock_page(page);
+	}
+	nfs_pageio_complete(&pgio);
+	nfs_writedata_release(data);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pnfs_ld_write_done);

fs/nfs/write.c +14 -7

index b39b37f..0ccdf98 100644
@@ -285,14 +285,9 @@ out:
 	return ret;
 }
 
-static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
+int do_nfs_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
 {
-	struct inode *inode = page->mapping->host;
 	int ret;
-
-	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
-	nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
-
 	nfs_pageio_cond_complete(pgio, page->index);
 	ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
 	if (ret == -EAGAIN) {

@@ -301,6 +296,17 @@ static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, st
 	}
 	return ret;
 }
+EXPORT_SYMBOL_GPL(do_nfs_writepage);
+
+static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
+{
+	struct inode *inode = page->mapping->host;
+
+	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
+	nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
+
+	return do_nfs_writepage(page, wbc, pgio);
+}
 
 /*
  * Write an mmapped page to the server.

@@ -1051,12 +1057,13 @@ static const struct nfs_pageio_ops nfs_pageio_write_ops = {
 	.pg_doio = nfs_generic_pg_writepages,
 };
 
-static void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
+void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
 				  struct inode *inode, int ioflags)
 {
 	nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops,
 				NFS_SERVER(inode)->wsize, ioflags);
 }
+EXPORT_SYMBOL_GPL(nfs_pageio_init_write_mds);
 
 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
 {

Signature · 1 line

1.7.1.262.g5ef3d

Peng Tao

author

2011-08-06 15:53 UTC #2 lore raw

PATCH 2/2 pNFS: deal with ld read failure

In reply to Peng Tao

For pnfs pagelist read failure, we need to pg_recoalesce and resend IO to mds.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/internal.h |    2 ++
 fs/nfs/pnfs.c     |   18 ++++++++++++++----
 fs/nfs/read.c     |    3 ++-
 3 files changed, 18 insertions(+), 5 deletions(-)

fs/nfs/internal.h +2 -0

index 62f183d..78b662e 100644
@@ -307,6 +307,8 @@ extern int nfs_generic_flush(struct nfs_pageio_descriptor *desc,
 		struct list_head *head);
 extern int do_nfs_writepage(struct page *page, struct writeback_control *wbc,
 		struct nfs_pageio_descriptor *pgio);
+extern void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
+		struct inode *inode);
 extern void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
 		struct inode *inode, int ioflags);
 extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio);

fs/nfs/pnfs.c +14 -4

index 08aba45..66fc854 100644
@@ -1300,7 +1300,7 @@ EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
 int
 pnfs_ld_read_done(struct nfs_read_data *data)
 {
-	int status;
+	struct nfs_pageio_descriptor pgio;
 
 	if (!data->pnfs_error) {
 		__nfs4_read_done_cb(data);

@@ -1309,11 +1309,21 @@ pnfs_ld_read_done(struct nfs_read_data *data)
 		return 0;
 	}
 
+	put_lseg(data->lseg);
+	data->lseg = NULL;
 	dprintk("%s: pnfs_error=%d, retry via MDS\n", __func__,
 		data->pnfs_error);
-	status = nfs_initiate_read(data, NFS_CLIENT(data->inode),
-				   data->mds_ops);
-	return status ? : -EAGAIN;
+	nfs_pageio_init_read_mds(&pgio, data->inode);
+	pgio.pg_recoalesce = 1;
+	while (!list_empty(&data->pages)) {
+		struct nfs_page *req = nfs_list_entry(data->pages.next);
+
+		nfs_pageio_add_request(&pgio, req);
+	}
+	nfs_pageio_complete(&pgio);
+	nfs_readdata_release(data);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pnfs_ld_read_done);

fs/nfs/read.c +2 -1

index 2171c04..2484131 100644
@@ -112,12 +112,13 @@ static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data)
 	}
 }
 
-static void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
+void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
 		struct inode *inode)
 {
 	nfs_pageio_init(pgio, inode, &nfs_pageio_read_ops,
 			NFS_SERVER(inode)->rsize, 0);
 }
+EXPORT_SYMBOL_GPL(nfs_pageio_init_read_mds);
 
 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
 {

Signature · 1 line

1.7.1.262.g5ef3d

Peng Tao

author

2011-08-08 14:33 UTC #3 lore raw

In reply to Peng Tao

Quoted 6 lines · Peng Tao

Hi,
I start to look at the error handling for ld_read/write_done. The two patches try to address the error handling for read/write pagelist failures, via nfs readpage/writepage interface. Please see if this method is proper. Thanks!

FWIW, this is just what we need for recoalescing pages when IO fails. Currently ld_read/write_done is called in default workqueue, which is not really a good place for IO path code. Trond mentioned that nfsiod is not an ideal place for it either. So do we need to create a pnfs private workqueue for this? Any other suggestions?

Signature · 2 lines

Thanks,
-Bergwolf