Skip to contentSkip to conversation

pnfs-obj: Fixes left-out from 3.5 for the 3.6 merge window

PATCH In review Boaz Harrosh 1 participant 6 messages Aug 2, 2012

Subsystems (2)
NFS client NFS server
lore
Share & tools
lore

Boaz Harrosh

author
#0 lore raw

cover pnfs-obj: Fixes left-out from 3.5 for the 3.6 merge window

Hi Trond

These fixes where for the 3.5 Kernel, Sorry I forgot about them. In any way they should be pushed for 3.6 fixes.
(Should we pursue Stable@ for them)

This is a re-post, except the 3rd one which is a new BUG fix.

list of patches:
[PATCH 1/3] NFS41: add pg_layout_private to nfs_pageio_descriptor [PATCH 2/3] pnfs-obj: Better IO pattern in case of unaligned offset [PATCH 3/3] pnfs: Don't BUG on info received from Server

Thanks
Boaz

author
#1 lore raw

PATCH 1/3 NFS41: add pg_layout_private to nfs_pageio_descriptor

To allow layout driver to pass private information around pg_init/pg_doio.

Signed-off-by Peng Tao <[email protected]>
Signed-off-by Boaz Harrosh <[email protected]>
---
 fs/nfs/pagelist.c        | 2 ++
 include/linux/nfs_page.h | 1 +
 include/linux/nfs_xdr.h  | 1 +
 3 files changed, 4 insertions(+)
fs/nfs/pagelist.c +2 -0
index aed913c..342ca5e 100644
@@ -49,6 +49,7 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
 	hdr->io_start = req_offset(hdr->req);
 	hdr->good_bytes = desc->pg_count;
 	hdr->dreq = desc->pg_dreq;
+	hdr->layout_private = desc->pg_layout_private;
 	hdr->release = release;
 	hdr->completion_ops = desc->pg_completion_ops;
 	if (hdr->completion_ops->init_hdr)

@@ -267,6 +268,7 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
 	desc->pg_error = 0;
 	desc->pg_lseg = NULL;
 	desc->pg_dreq = NULL;
+	desc->pg_layout_private = NULL;
 }
 
 /**
include/linux/nfs_page.h +1 -0
index 88d166b..63093b1 100644
@@ -69,6 +69,7 @@ struct nfs_pageio_descriptor {
 	const struct nfs_pgio_completion_ops *pg_completion_ops;
 	struct pnfs_layout_segment *pg_lseg;
 	struct nfs_direct_req	*pg_dreq;
+	void			*pg_layout_private;
 };
 
 #define NFS_WBACK_BUSY(req)	(test_bit(PG_BUSY,&(req)->wb_flags))
include/linux/nfs_xdr.h +1 -0
index 8aadd90..58023cd 100644
@@ -1244,6 +1244,7 @@ struct nfs_pgio_header {
 	void (*release) (struct nfs_pgio_header *hdr);
 	const struct nfs_pgio_completion_ops *completion_ops;
 	struct nfs_direct_req	*dreq;
+	void			*layout_private;
 	spinlock_t		lock;
 	/* fields protected by lock */
 	int			pnfs_error;
Signature · 1 line
1.7.10.2.677.gb6bc67f
author
#2 lore raw

PATCH 1/3 NFS41: add pg_layout_private to nfs_pageio_descriptor

In reply to Boaz Harrosh

From: Peng Tao <[email protected]>

To allow layout driver to pass private information around pg_init/pg_doio.

Signed-off-by Peng Tao <[email protected]>
Signed-off-by Boaz Harrosh <[email protected]>
---
 fs/nfs/pagelist.c        | 2 ++
 include/linux/nfs_page.h | 1 +
 include/linux/nfs_xdr.h  | 1 +
 3 files changed, 4 insertions(+)
fs/nfs/pagelist.c +2 -0
index aed913c..342ca5e 100644
@@ -49,6 +49,7 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
 	hdr->io_start = req_offset(hdr->req);
 	hdr->good_bytes = desc->pg_count;
 	hdr->dreq = desc->pg_dreq;
+	hdr->layout_private = desc->pg_layout_private;
 	hdr->release = release;
 	hdr->completion_ops = desc->pg_completion_ops;
 	if (hdr->completion_ops->init_hdr)

@@ -267,6 +268,7 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
 	desc->pg_error = 0;
 	desc->pg_lseg = NULL;
 	desc->pg_dreq = NULL;
+	desc->pg_layout_private = NULL;
 }
 
 /**
include/linux/nfs_page.h +1 -0
index 88d166b..63093b1 100644
@@ -69,6 +69,7 @@ struct nfs_pageio_descriptor {
 	const struct nfs_pgio_completion_ops *pg_completion_ops;
 	struct pnfs_layout_segment *pg_lseg;
 	struct nfs_direct_req	*pg_dreq;
+	void			*pg_layout_private;
 };
 
 #define NFS_WBACK_BUSY(req)	(test_bit(PG_BUSY,&(req)->wb_flags))
include/linux/nfs_xdr.h +1 -0
index 8aadd90..58023cd 100644
@@ -1244,6 +1244,7 @@ struct nfs_pgio_header {
 	void (*release) (struct nfs_pgio_header *hdr);
 	const struct nfs_pgio_completion_ops *completion_ops;
 	struct nfs_direct_req	*dreq;
+	void			*layout_private;
 	spinlock_t		lock;
 	/* fields protected by lock */
 	int			pnfs_error;
Signature · 1 line
1.7.10.2.677.gb6bc67f
author
#3 lore raw

Re: [PATCH 1/3 repost] NFS41: add pg_layout_private to nfs_pageio_descriptor

Quoted 2 lines · Boaz Harrosh

From: Peng Tao <[email protected]>

Trond Hi

I forgot the From: Peng ... on the first post please use this one

Thanks
Boaz

Quoted 55 lines

To allow layout driver to pass private information around pg_init/pg_doio.

Signed-off-by Peng Tao <[email protected]>
Signed-off-by Boaz Harrosh <[email protected]>
---
 fs/nfs/pagelist.c        | 2 ++
 include/linux/nfs_page.h | 1 +
 include/linux/nfs_xdr.h  | 1 +
 3 files changed, 4 insertions(+)
fs/nfs/pagelist.c +2 -0
index aed913c..342ca5e 100644
@@ -49,6 +49,7 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
 	hdr->io_start = req_offset(hdr->req);
 	hdr->good_bytes = desc->pg_count;
 	hdr->dreq = desc->pg_dreq;
+	hdr->layout_private = desc->pg_layout_private;
 	hdr->release = release;
 	hdr->completion_ops = desc->pg_completion_ops;
 	if (hdr->completion_ops->init_hdr)

@@ -267,6 +268,7 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
 	desc->pg_error = 0;
 	desc->pg_lseg = NULL;
 	desc->pg_dreq = NULL;
+	desc->pg_layout_private = NULL;
 }
 
 /**
include/linux/nfs_page.h +1 -0
index 88d166b..63093b1 100644
@@ -69,6 +69,7 @@ struct nfs_pageio_descriptor {
 	const struct nfs_pgio_completion_ops *pg_completion_ops;
 	struct pnfs_layout_segment *pg_lseg;
 	struct nfs_direct_req	*pg_dreq;
+	void			*pg_layout_private;
 };
 
 #define NFS_WBACK_BUSY(req)	(test_bit(PG_BUSY,&(req)->wb_flags))
include/linux/nfs_xdr.h +1 -0
index 8aadd90..58023cd 100644
@@ -1244,6 +1244,7 @@ struct nfs_pgio_header {
 	void (*release) (struct nfs_pgio_header *hdr);
 	const struct nfs_pgio_completion_ops *completion_ops;
 	struct nfs_direct_req	*dreq;
+	void			*layout_private;
 	spinlock_t		lock;
 	/* fields protected by lock */
 	int			pnfs_error;
author
#4 lore raw

PATCH 2/3 pnfs-obj: Better IO pattern in case of unaligned offset

In reply to Boaz Harrosh

Depending on layout and ARCH, ORE has some limits on max IO sizes which is communicated on (what else) ore_layout->max_io_length, which is always stripe aligned.
This was considered as the pg_test boundary for splitting and starting a new IO.

But in the case of a long IO where the start offset is not aligned what would happen is that both end of IO[N] and start of IO[N+1] would be unaligned, causing each IO boundary parity unit to be calculated and written twice.

So what we do in this patch is split the very start of an unaligned IO, up to a stripe boundary, and then next IO's can continue fully aligned til the end.

We might be sacrificing the case where the full unaligned IO would fit within a single max_io_length, but the sacrifice is well worth the elimination of double calculation and parity units IO. Actually the sacrificing is marginal and is almost unmeasurable.

TODO:
If we know the total expected linear segment that will be received, at pg_init, we could use that information in many places:
1. blocks-layout get_layout write segment size
2. Better mds-threshold
3. In above situation for a better clean split

I will do this in future submission.

Signed-off-by Boaz Harrosh <[email protected]>
---
 fs/nfs/objlayout/objio_osd.c | 55 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 52 insertions(+), 3 deletions(-)
fs/nfs/objlayout/objio_osd.c +52 -3
index f50d3e8..ea6d111 100644
@@ -570,17 +570,66 @@ static bool objio_pg_test(struct nfs_pageio_descriptor *pgio,
 		return false;
 
 	return pgio->pg_count + req->wb_bytes <=
-			OBJIO_LSEG(pgio->pg_lseg)->layout.max_io_length;
+			(unsigned long)pgio->pg_layout_private;
+}
+
+void objio_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
+{
+	pnfs_generic_pg_init_read(pgio, req);
+	if (unlikely(pgio->pg_lseg == NULL))
+		return; /* Not pNFS */
+
+	pgio->pg_layout_private = (void *)
+				OBJIO_LSEG(pgio->pg_lseg)->layout.max_io_length;
+}
+
+static bool aligned_on_raid_stripe(u64 offset, struct ore_layout *layout,
+				   unsigned long *stripe_end)
+{
+	u32 stripe_off;
+	unsigned stripe_size;
+
+	if (layout->raid_algorithm == PNFS_OSD_RAID_0)
+		return true;
+
+	stripe_size = layout->stripe_unit *
+				(layout->group_width - layout->parity);
+
+	div_u64_rem(offset, stripe_size, &stripe_off);
+	if (!stripe_off)
+		return true;
+
+	*stripe_end = stripe_size - stripe_off;
+	return false;
+}
+
+void objio_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
+{
+	unsigned long stripe_end = 0;
+
+	pnfs_generic_pg_init_write(pgio, req);
+	if (unlikely(pgio->pg_lseg == NULL))
+		return; /* Not pNFS */
+
+	if (req->wb_offset ||
+	    !aligned_on_raid_stripe(req->wb_index * PAGE_SIZE,
+			       &OBJIO_LSEG(pgio->pg_lseg)->layout,
+			       &stripe_end)) {
+		pgio->pg_layout_private = (void *)stripe_end;
+	} else {
+		pgio->pg_layout_private = (void *)
+				OBJIO_LSEG(pgio->pg_lseg)->layout.max_io_length;
+	}
 }
 
 static const struct nfs_pageio_ops objio_pg_read_ops = {
-	.pg_init = pnfs_generic_pg_init_read,
+	.pg_init = objio_init_read,
 	.pg_test = objio_pg_test,
 	.pg_doio = pnfs_generic_pg_readpages,
 };
 
 static const struct nfs_pageio_ops objio_pg_write_ops = {
-	.pg_init = pnfs_generic_pg_init_write,
+	.pg_init = objio_init_write,
 	.pg_test = objio_pg_test,
 	.pg_doio = pnfs_generic_pg_writepages,
 };
Signature · 1 line
1.7.10.2.677.gb6bc67f
author
#5 lore raw

PATCH 3/3 pnfs: Don't BUG on info received from Server

In reply to Boaz Harrosh

In nfs4_layoutreturn_done() there is a BUG_ON on a Server returned member, when received in what the client thinks is an impossible situation.

[The server returned ! lrp->res.lrs_present, but client
still had more segments]

This BUG really hit me with the Linux pnfs Server, but regardless of who is at fault here, Server or client, client must not crash, even on a buggy Server.

Signed-off-by Boaz Harrosh <[email protected]>
---
 fs/nfs/nfs4proc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
fs/nfs/nfs4proc.c +1 -1
index 15fc7e4..e25b686 100644
@@ -6249,7 +6249,7 @@ static void nfs4_layoutreturn_done(struct rpc_task *task, void *calldata)
 		if (lrp->res.lrs_present) {
 			pnfs_set_layout_stateid(lo, &lrp->res.stateid, true);
 		} else
-			BUG_ON(!list_empty(&lo->plh_segs));
+			WARN_ON(!list_empty(&lo->plh_segs));
 	}
 	lo->plh_block_lgets--;
 	spin_unlock(&lo->plh_inode->i_lock);
Signature · 1 line
1.7.10.2.677.gb6bc67f