nfs41: allow layoutget at pnfs_do_multiple_writes

This gives LD option not to ask for layout in pg_init.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/pnfs.c |   46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 46 insertions(+), 0 deletions(-)

fs/nfs/pnfs.c +0 -0

index 734e670..c8dc0b1 100644

fs/nfs/pnfs.c +1 -0

 	desc->pg_lseg = NULL;
 	while (!list_empty(head)) {

@@ -1262,7 +1263,29 @@ pnfs_do_multiple_writes(struct nfs_pageio_descriptor *desc, struct list_head *he
 		data = list_entry(head->next, struct nfs_write_data, list);
 		list_del_init(&data->list);
 
+		if (!has_lseg) {

fs/nfs/pnfs.c +13 -0

 	desc->pg_lseg = NULL;
 	while (!list_empty(head)) {

@@ -1358,7 +1382,29 @@ pnfs_do_multiple_reads(struct nfs_pageio_descriptor *desc, struct list_head *hea
 		data = list_entry(head->next, struct nfs_read_data, list);
 		list_del_init(&data->list);
 
+		if (!has_lseg) {
+			struct nfs_page *req = nfs_list_entry(data->pages.next);
+			__u64 length = data->npages << PAGE_CACHE_SHIFT;
+
+			lseg = pnfs_update_layout(desc->pg_inode,
+						  req->wb_context,
+						  req_offset(req),
+						  length,
+						  IOMODE_READ,
+						  GFP_KERNEL);
+			if (!lseg || length > lseg->pls_range.length) {
+				put_lseg(lseg);
+				lseg = NULL;

fs/nfs/pnfs.c +0 -0

 		if (trypnfs == PNFS_NOT_ATTEMPTED)
 			pnfs_read_through_mds(desc, data);
 	}

Asking for layout in pg_init will always make client ask for only 4KB layout in every layoutget. This way, client drops the IO size information that is meaningful for MDS in handing out layout.

In stead, if layout is not find in cache, do not send layoutget at once. Wait until before issuing IO in pnfs_do_multiple_reads/writes because that is where we know the real size of current IO. By telling the real IO size to MDS, MDS will have a better chance to give proper layout.

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/blocklayout/blocklayout.c |   54 ++++++++++++++++++++++++++++++++++++-
 1 files changed, 52 insertions(+), 2 deletions(-)

fs/nfs/blocklayout/blocklayout.c +3 -0

index 48cfac3..fd585fe 100644
@@ -39,6 +39,7 @@
 #include <linux/prefetch.h>
 
 #include "blocklayout.h"
+#include "../internal.h"
 
 #define NFSDBG_FACILITY	NFSDBG_PNFS_LD
 

@@ -990,14 +991,63 @@ bl_clear_layoutdriver(struct nfs_server *server)
 	return 0;
 }
 
+/* While RFC doesn't limit maximum size of layout, we better limit it ourself. */
+#define PNFSBLK_MAXRSIZE (0x1<<22)

fs/nfs/blocklayout/blocklayout.c +18 -0

+	if (!lo || test_bit(lo_fail_bit(IOMODE_READ), &lo->plh_flags)) {
+		spin_unlock(&ino->i_lock);
+		nfs_pageio_reset_read_mds(pgio);
+		return;
+	}
+
+	pgio->pg_bsize = PNFSBLK_MAXRSIZE;
+	pgio->pg_lseg = pnfs_find_get_layout_locked(ino,
+						req_offset(req),
+						req->wb_bytes,
+						IOMODE_READ);
+	spin_unlock(&ino->i_lock);
+}
+
+static void
+bl_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
+{
+	struct inode *ino = pgio->pg_inode;

fs/nfs/blocklayout/blocklayout.c +11 -2

+
+	pgio->pg_bsize = PNFSBLK_MAXWSIZE;
+	pgio->pg_lseg = pnfs_find_get_layout_locked(ino,
+						req_offset(req),
+						req->wb_bytes,
+						IOMODE_RW);
+	spin_unlock(&ino->i_lock);
+}
+
 static const struct nfs_pageio_ops bl_pg_read_ops = {
-	.pg_init = pnfs_generic_pg_init_read,
+	.pg_init = bl_pg_init_read,
 	.pg_test = pnfs_generic_pg_test,
 	.pg_doio = pnfs_generic_pg_readpages,
 };
 
 static const struct nfs_pageio_ops bl_pg_write_ops = {
-	.pg_init = pnfs_generic_pg_init_write,
+	.pg_init = bl_pg_init_write,
 	.pg_test = pnfs_generic_pg_test,
 	.pg_doio = pnfs_generic_pg_writepages,
 };

Quoted 4 lines · Peng Tao

Asking for layout in pg_init will always make client ask for only 4KB layout in every layoutget. This way, client drops the IO size information that is meaningful for MDS in handing out layout.

Quoted 4 lines

In stead, if layout is not find in cache, do not send layoutget at once. Wait until before issuing IO in pnfs_do_multiple_reads/writes because that is where we know the real size of current IO. By telling the real IO size to MDS, MDS will have a better chance to give proper layout.

Why can't you just split pnfs_update_layout() into 2 sub-functions instead of duplicating it in private block code?

Then call layoutget in your pg_doio() callback instead of adding a redundant pnfs_update_layout to pnfs_do_multiple_reads/pnfs_do_multiple_writes...

Quoted 23 lines

Signed-off-by Peng Tao <[email protected]>

---
 fs/nfs/blocklayout/blocklayout.c |   54 ++++++++++++++++++++++++++++++++++++-

 1 files changed, 52 insertions(+), 2 deletions(-)

fs/nfs/blocklayout/blocklayout.c +1 -0

index 48cfac3..fd585fe 100644
@@ -39,6 +39,7 @@
  #include <linux/prefetch.h>
 
  #include "blocklayout.h"
+#include "../internal.h"
 
  #define NFSDBG_FACILITY      NFSDBG_PNFS_LD
 

@@ -990,14 +991,63 @@ bl_clear_layoutdriver(struct nfs_server *server)
       return 0;
  }
 

fs/nfs/blocklayout/blocklayout.c bl_clear_layoutdriver(struct nfs_server *server) · line 1005

Quoted 12 lines

fs/nfs/blocklayout/blocklayout.c +12 -0

+/* While RFC doesn't limit maximum size of layout, we better limit it ourself. */
+#define PNFSBLK_MAXRSIZE (0x1<<22)
+#define PNFSBLK_MAXWSIZE (0x1<<21)
+static void
+bl_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
+{
+     struct inode *ino = pgio->pg_inode;
+     struct pnfs_layout_hdr *lo;
+
+     BUG_ON(pgio->pg_lseg != NULL);
+     spin_lock(&ino->i_lock);
+     lo = pnfs_find_alloc_layout(ino, req->wb_context, GFP_KERNEL);

This has never been tested... It contains all sorts of bugs from recursive attempts to take the ino->i_lock, to sleep-under-spinlock...

Quoted 19 lines

fs/nfs/blocklayout/blocklayout.c +18 -0

+     if (!lo || test_bit(lo_fail_bit(IOMODE_READ), &lo->plh_flags)) {
+             spin_unlock(&ino->i_lock);
+             nfs_pageio_reset_read_mds(pgio);
+             return;
+     }
+
+     pgio->pg_bsize = PNFSBLK_MAXRSIZE;
+     pgio->pg_lseg = pnfs_find_get_layout_locked(ino,
+                                             req_offset(req),
+                                             req->wb_bytes,
+                                             IOMODE_READ);
+     spin_unlock(&ino->i_lock);
+}
+
+static void
+bl_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
+{
+     struct inode *ino = pgio->pg_inode;

fs/nfs/blocklayout/blocklayout.c bl_clear_layoutdriver(struct nfs_server *server) · line 1033

Quoted 10 lines

fs/nfs/blocklayout/blocklayout.c +10 -0

+     struct pnfs_layout_hdr *lo;
+
+     BUG_ON(pgio->pg_lseg != NULL);
+     spin_lock(&ino->i_lock);
+     lo = pnfs_find_alloc_layout(ino, req->wb_context, GFP_NOFS);
+     if (!lo || test_bit(lo_fail_bit(IOMODE_RW), &lo->plh_flags)) {
+             spin_unlock(&ino->i_lock);
+             nfs_pageio_reset_write_mds(pgio);
+             return;
+     }

Ditto...

Quoted 12 lines

fs/nfs/blocklayout/blocklayout.c +10 -1

+
+     pgio->pg_bsize = PNFSBLK_MAXWSIZE;
+     pgio->pg_lseg = pnfs_find_get_layout_locked(ino,
+                                             req_offset(req),
+                                             req->wb_bytes,
+                                             IOMODE_RW);
+     spin_unlock(&ino->i_lock);
+}
+
  static const struct nfs_pageio_ops bl_pg_read_ops = {
-     .pg_init = pnfs_generic_pg_init_read,
+     .pg_init = bl_pg_init_read,

fs/nfs/blocklayout/blocklayout.c bl_clear_layoutdriver(struct nfs_server *server) · line 1053

Quoted 10 lines

fs/nfs/blocklayout/blocklayout.c +1 -1

       .pg_test = pnfs_generic_pg_test,
       .pg_doio = pnfs_generic_pg_readpages,
  };
 
  static const struct nfs_pageio_ops bl_pg_write_ops = {
-     .pg_init = pnfs_generic_pg_init_write,
+     .pg_init = bl_pg_init_write,
       .pg_test = pnfs_generic_pg_test,
       .pg_doio = pnfs_generic_pg_writepages,
  };

--
Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

Quoted 5 lines · Trond Myklebust

Quoted 4 lines · Peng Tao

Asking for layout in pg_init will always make client ask for only 4KB layout in every layoutget. This way, client drops the IO size information that is meaningful for MDS in handing out layout.

Quoted 7 lines

Quoted 4 lines

In stead, if layout is not find in cache, do not send layoutget at once. Wait until before issuing IO in pnfs_do_multiple_reads/writes because that is where we know the real size of current IO. By telling the real IO size to MDS, MDS will have a better chance to give proper layout.

Why can't you just split pnfs_update_layout() into 2 sub-functions instead of duplicating it in private block code?

Because I wanted to differentiate between no layout header and no cached lseg, where the pnfs_update_layout() interface is not enough to tell the difference. Of course I can put these all into generic layer. I will update the patchset to do it.

Quoted 1 line

Quoted 3 lines

Then call layoutget in your pg_doio() callback instead of adding a redundant pnfs_update_layout to pnfs_do_multiple_reads/pnfs_do_multiple_writes...

I have considered it before but using private pg_doio() means we will have as much duplication of pnfs_generic_pg_read/writepages.

Issuing layoutget at .pg_init will drop the IO size information and ask for 4KB layout every time. However, the IO size information is very valuable for MDS to determine how much layout it should return to client.

The patchset try to allow LD not to send layoutget at .pg_init but instead at pnfs_do_multiple_writes. So that real IO size is preserved and sent to MDS.

Tests against a server that does not aggressively pre-allocate layout, shows that the IO size informantion is really useful to block layout MDS.

The generic pnfs layer changes are trival to file layout and object as long as they still send layoutget at .pg_init.

iozone cmd:
./iozone -r 1m -s 4G -w -W -c -t 10 -i 0 -F /mnt/iozone.data.1 /mnt/iozone.data.2 /mnt/iozone.data.3 /mnt/iozone.data.4 /mnt/iozone.data.5 /mnt/iozone.data.6 /mnt/iozone.data.7 /mnt/iozone.data.8 /mnt/iozone.data.9 /mnt/iozone.data.10

Befor patch: around 12MB/s throughput
After patch: around 72MB/s throughput

Peng Tao (4):
nfsv41: export pnfs_find_alloc_layout
nfsv41: add and export pnfs_find_get_layout_locked
nfsv41: get lseg before issue LD IO if pgio doesn't carry lseg
pnfsblock: do ask for layout in pg_init

 fs/nfs/blocklayout/blocklayout.c |   54 ++++++++++++++++++++++++++-
 fs/nfs/pnfs.c                    |   74 +++++++++++++++++++++++++++++++++++++-
 fs/nfs/pnfs.h                    |    9 +++++
 3 files changed, 134 insertions(+), 3 deletions(-)

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

But just do the above and you'll see that it is perfect.

BTW don't limit the lo_segment size by the max_io_size. This is why you have .bg_test to signal when IO is maxed out.

- The read segments should be as big as possible (i_size long)
- The Write segments should ideally be as big as the Application wants to write to. (Amount of dirty pages at time of nfs-write-out is a very good first approximation).

So I guess it is: I hate these patches, to much mess, too little goodness.

Thank
Boaz

Quoted 10 lines · Boaz Harrosh

But just do the above and you'll see that it is perfect.

BTW don't limit the lo_segment size by the max_io_size. This is why you have .bg_test to signal when IO is maxed out.

- The read segments should be as big as possible (i_size long)
- The Write segments should ideally be as big as the Application wants to write to. (Amount of dirty pages at time of nfs-write-out is a very good first approximation).

Quoted 5 lines

So I guess it is: I hate these patches, to much mess, too little goodness.

Thank
Boaz

Ho and one more thing.

Also Files when they will support segments and servers that request segments, like the CEPH server, will very much enjoy the above, .i.e: Tell me the amount you know you want to write.

Quoted 3 lines

Also Files when they will support segments and servers that request segments, like the CEPH server, will very much enjoy the above, .i.e: Tell me the amount you know you want to write.

Why would we want to add segment support to the pNFS files client??? Segments are a nuisance that multiply the amount of unnecessary chitchat between the client and the MDS without providing any tangible benefits...

Quoted 9 lines · Trond Myklebust

Quoted 3 lines

Also Files when they will support segments and servers that request segments, like the CEPH server, will very much enjoy the above, .i.e: Tell me the amount you know you want to write.

Why would we want to add segment support to the pNFS files client??? Segments are a nuisance that multiply the amount of unnecessary chitchat between the client and the MDS without providing any tangible benefits...

Your kidding right?

One: it is mandated by the Standard, This is not an option. So a perfectly
Standard complaint server is not Supported by Linux because we don't see the point.

Two: There are already file-layout servers out there (multiple) which are
waiting for the Linux files-layout segment support, because the underline FS requires Segments and now they do not work with the Linux client. These are CEPH and GPFS and more.

Quoted 13 lines · Boaz Harrosh

Quoted 9 lines · Trond Myklebust

Quoted 3 lines

Also Files when they will support segments and servers that request segments, like the CEPH server, will very much enjoy the above, .i.e: Tell me the amount you know you want to write.

Why would we want to add segment support to the pNFS files client??? Segments are a nuisance that multiply the amount of unnecessary chitchat between the client and the MDS without providing any tangible benefits...

Your kidding right?

Quoted 3 lines

One: it is mandated by the Standard, This is not an option. So a perfectly
Standard complaint server is not Supported by Linux because we don't see the point.

Bollocks.. Nothing is "mandated by the Standard". If the server doesn't give us a full layout, then we fall back to write through MDS. Why dick around with crap that SLOWS YOU DOWN.

Quoted 4 lines

Two: There are already file-layout servers out there (multiple) which are
waiting for the Linux files-layout segment support, because the underline FS requires Segments and now they do not work with the Linux client. These are CEPH and GPFS and more.

Then they will have a _long_ wait....

Trond

Quoted 14 lines · Trond Myklebust

Quoted 13 lines · Boaz Harrosh

Quoted 9 lines · Trond Myklebust

Quoted 3 lines

Also Files when they will support segments and servers that request segments, like the CEPH server, will very much enjoy the above, .i.e: Tell me the amount you know you want to write.

Why would we want to add segment support to the pNFS files client??? Segments are a nuisance that multiply the amount of unnecessary chitchat between the client and the MDS without providing any tangible benefits...

Your kidding right?

Quoted 8 lines

Quoted 3 lines

One: it is mandated by the Standard, This is not an option. So a perfectly
Standard complaint server is not Supported by Linux because we don't see the point.

Bollocks.. Nothing is "mandated by the Standard". If the server doesn't give us a full layout, then we fall back to write through MDS. Why dick around with crap that SLOWS YOU DOWN.

NO! MAKE YOU FASTER.

The kind of typologies I'm talking about a single layout get ever 1GB is marginal to the gain I get in deploying 100 of DSs. I have thousands of DSs I want to spread the load evenly. I'm limited by the size of the layout (Device info in the case of files) So I'm limited by the number of DSs I can have in a layout. For large files these few devices become an hot spot all the while the rest of the cluster is idle.

Quoted 7 lines

Quoted 4 lines

Two: There are already file-layout servers out there (multiple) which are
waiting for the Linux files-layout segment support, because the underline FS requires Segments and now they do not work with the Linux client. These are CEPH and GPFS and more.

Then they will have a _long_ wait....

OK, so now I understand. Because when I was talking to Fred before BAT and during It was very very peculiar to me why he is not already done with that simple stuff. Because usually Fred is such a brilliant fast programmer that I admire, and that simple crap?

But now that explains

Quoted 3 lines · Trond Myklebust

Quoted 2 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

Quoted 1 line

Also Files when they will support segments and servers that request

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

like the CEPH server, will very much enjoy the above, .i.e: Tell me

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 1 line

you know you want to write.

Why would we want to add segment support to the pNFS files client??? Segments are a nuisance that multiply the amount of unnecessary

Quoted 7 lines

Quoted 7 lines

Quoted 3 lines

between the client and the MDS without providing any tangible benefits...

Your kidding right?

One: it is mandated by the Standard, This is not an option. So a

Quoted 1 line

Quoted 1 line

Standard complaint server is not Supported by Linux because we

Quoted 3 lines

Quoted 1 line

the point.

Bollocks.. Nothing is "mandated by the Standard". If the server

Quoted 1 line

give us a full layout, then we fall back to write through MDS. Why

Quoted 2 lines

around with crap that SLOWS YOU DOWN.

NO! MAKE YOU FASTER.

The kind of typologies I'm talking about a single layout get ever 1GB is marginal to the gain I get in deploying 100 of DSs. I have thousands of DSs I want to spread the load evenly. I'm limited by the size of the

(Device info in the case of files) So I'm limited by the number of DSs I

have in a layout. For large files these few devices become an hot spot

the while the rest of the cluster is idle.

Quoted 1 line

Quoted 1 line

Two: There are already file-layout servers out there (multiple) which

Quoted 1 line

Quoted 1 line

waiting for the Linux files-layout segment support, because the

Quoted 1 line

Quoted 1 line

FS requires Segments and now they do not work with the Linux

Quoted 4 lines

Quoted 1 line

are CEPH and GPFS and more.

Then they will have a _long_ wait....

OK, so now I understand. Because when I was talking to Fred before BAT

It was very very peculiar to me why he is not already done with that

Because usually Fred is such a brilliant fast programmer that I admire,

crap?

But now that explains

The only 1 cent that I can add to this argument is that I was led to believe by you and others that Linux kernel don't add functionality on the client side that is not supported on the server side. Last time I made this point I was told that it is ok if they are of by a version or two. You made it clear that this is no longer true and that the Linux client and server are now independent of each other. I spent time working on the server side believing that the client and server will progress more or less together, disappointed to find out that it is not the case any more. Marc.

The only 1 cent that I can add to this argument is that I was led to believe by you and others that Linux kernel don't add functionality on

client side that is not supported on the server side. Last time I made this point I was told that it is ok if they are of by a version or two. You made it clear that this is no longer true and that the Linux client and server are now independent of each other. I spent time working on

server side believing that the client and server will progress more or less together, disappointed to find out that it is not the case any

Marc.

You ignored my main point, I was talking about the server side, my point was that there is nothing to build on on the serve side since the pNFS Linux server is not happening.
Marc.

From:
Trond Myklebust <[email protected]>
To:
Marc Eshel <[email protected]>
Cc:
Peng Tao <[email protected]>, [email protected], Boaz Harrosh <[email protected]>, Garth Gibson <[email protected]>, Fred Isaman <[email protected]>, [email protected], Matt Benjamin <[email protected]>
Date:
11/29/2011 04:08 PM
Subject:
Re: [PATCH 0/4] nfs41: allow layoutget at pnfs_do_multiple_writes

Quoted 2 lines · Marc Eshel

The only 1 cent that I can add to this argument is that I was led to believe by you and others that Linux kernel don't add functionality on

the

Quoted 4 lines

client side that is not supported on the server side. Last time I made this point I was told that it is ok if they are of by a version or two. You made it clear that this is no longer true and that the Linux client and server are now independent of each other. I spent time working on

the

Quoted 2 lines

server side believing that the client and server will progress more or less together, disappointed to find out that it is not the case any

more.

Quoted 1 line

Marc.

I don't know how to manage layout segments in a way that meets the pNFS goals of scalability and performance, and neither you nor the pNFS spec have told me how to do this.

If IBM wants a client that implements layout segments, then it is _your_ responsibility to:
A. Convince me that it is an implementable part of the spec. B. Provide me with an implementation that deals all with the
concerns that I have.

IOW: nobody has ever promised IBM that the community would do all your client work for you. EMC, NetApp and Panasas have implemented (most of) the bits that they cared about; any bits that they haven't implemented and that you care about are up to you.

Quoted 4 lines · Marc Eshel

You ignored my main point, I was talking about the server side, my point was that there is nothing to build on on the serve side since the pNFS Linux server is not happening.
Marc.

Sorry. I misunderstood your concern. As far as I know, the main problem there is also one of investment: nobody has stepped up to help Bruce write a pNFS server.

I'm less worried about this now than I was earlier, because other open source efforts are gaining traction (see Ganesha - which is being sponsored by IBM, and projects such as Tigran's java based pNFS server). The other point is that we've developed client test-rigs that don't depend on the availability of a Linux server (newpynfs and the pynfs based proxy).

Cheers
Trond

Quoted 10 lines · Trond Myklebust

Quoted 4 lines · Marc Eshel

You ignored my main point, I was talking about the server side, my point was that there is nothing to build on on the serve side since the pNFS Linux server is not happening.
Marc.

Sorry. I misunderstood your concern. As far as I know, the main problem there is also one of investment: nobody has stepped up to help Bruce write a pNFS server.

I use "OUR" Linux pNFS Server every day, all day. It's there it is written and it kicks us. We've been working on that for 5 Years now.

The only holding factor is that no one wants to do the final step and fight the VFS guys to actually push it into Linus.

But it is there and alive in full open-source glory in Benny's tree.

Quoted 7 lines

I'm less worried about this now than I was earlier, because other open source efforts are gaining traction (see Ganesha - which is being sponsored by IBM, and projects such as Tigran's java based pNFS server). The other point is that we've developed client test-rigs that don't depend on the availability of a Linux server (newpynfs and the pynfs based proxy).

For me I'm testing with the linux Server and it is just there and works well for me. Better than any other solution I have right now.

Quoted 3 lines

Cheers
Trond

Ciao
Heart
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Peng Tao, bhalevy, Boaz Harrosh, Garth Gibson, Fred Isaman, linux- nfs, Matt Benjamin

Quoted 1 line · Marc Eshel

You ignored my main point, I was talking about the server side, my

Quoted 4 lines

was that there is nothing to build on on the serve side since the pNFS

Linux server is not happening.
Marc.

Sorry. I misunderstood your concern. As far as I know, the main problem there is also one of investment: nobody has stepped up to help Bruce write a pNFS server.

I'm less worried about this now than I was earlier, because other open source efforts are gaining traction (see Ganesha - which is being sponsored by IBM, and projects such as Tigran's java based pNFS server). The other point is that we've developed client test-rigs that don't depend on the availability of a Linux server (newpynfs and the pynfs based proxy).

Cheers
Trond

Quoted 20 lines

From:
Trond Myklebust <[email protected]>
To:
Marc Eshel <[email protected]>
Cc:
Peng Tao <[email protected]>, [email protected], Boaz Harrosh <[email protected]>, Garth Gibson <[email protected]>, Fred Isaman <[email protected]>, [email protected], Matt Benjamin <[email protected]>
Date:
11/29/2011 04:08 PM
Subject:
Re: [PATCH 0/4] nfs41: allow layoutget at pnfs_do_multiple_writes

Quoted 3 lines · Marc Eshel

The only 1 cent that I can add to this argument is that I was led to

believe by you and others that Linux kernel don't add functionality

Quoted 2 lines

the

Quoted 1 line

client side that is not supported on the server side. Last time I

Quoted 1 line

Quoted 1 line

this point I was told that it is ok if they are of by a version or

Quoted 1 line

Quoted 1 line

You made it clear that this is no longer true and that the Linux

Quoted 1 line

Quoted 1 line

and server are now independent of each other. I spent time working

Quoted 2 lines

the

Quoted 1 line

server side believing that the client and server will progress more

Quoted 5 lines

Quoted 1 line

less together, disappointed to find out that it is not the case any

more.

Quoted 1 line

Marc.

I don't know how to manage layout segments in a way that meets the

Quoted 1 line

goals of scalability and performance, and neither you nor the pNFS

Quoted 3 lines

have told me how to do this.

If IBM wants a client that implements layout segments, then it is

Quoted 7 lines

responsibility to:
A. Convince me that it is an implementable part of the spec. B. Provide me with an implementation that deals all with the
concerns that I have.

IOW: nobody has ever promised IBM that the community would do all your client work for you. EMC, NetApp and Panasas have implemented (most

Quoted 3 lines

the bits that they cared about; any bits that they haven't implemented and that you care about are up to you.

Signature · 6 lines

Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

Trond Myklebust <[email protected]> wrote on 11/29/2011 04:37:05 PM:

Quoted 6 lines

Peng Tao, bhalevy, Boaz Harrosh, Garth Gibson, Fred Isaman, linux- nfs, Matt Benjamin

Quoted 1 line · Marc Eshel

You ignored my main point, I was talking about the server side, my

point

Quoted 9 lines

Quoted 4 lines

was that there is nothing to build on on the serve side since the pNFS

Linux server is not happening.
Marc.

Sorry. I misunderstood your concern. As far as I know, the main problem there is also one of investment: nobody has stepped up to help Bruce write a pNFS server.

Quoted 6 lines

I'm less worried about this now than I was earlier, because other open source efforts are gaining traction (see Ganesha - which is being sponsored by IBM, and projects such as Tigran's java based pNFS server). The other point is that we've developed client test-rigs that don't depend on the availability of a Linux server (newpynfs and the pynfs based proxy).

You got it backward, Ganesha is getting traction precisely because the Linux kernel server is not happening :)

Quoted 25 lines · Marc Eshel

Trond Myklebust <[email protected]> wrote on 11/29/2011 04:37:05 PM:

Quoted 6 lines

Peng Tao, bhalevy, Boaz Harrosh, Garth Gibson, Fred Isaman, linux- nfs, Matt Benjamin

Quoted 1 line · Marc Eshel

You ignored my main point, I was talking about the server side, my

point

Quoted 15 lines

Quoted 4 lines

was that there is nothing to build on on the serve side since the pNFS

Linux server is not happening.
Marc.

Sorry. I misunderstood your concern. As far as I know, the main problem there is also one of investment: nobody has stepped up to help Bruce write a pNFS server.

I'm less worried about this now than I was earlier, because other open source efforts are gaining traction (see Ganesha - which is being sponsored by IBM, and projects such as Tigran's java based pNFS server). The other point is that we've developed client test-rigs that don't depend on the availability of a Linux server (newpynfs and the pynfs based proxy).

Quoted 2 lines

You got it backward, Ganesha is getting traction precisely because the Linux kernel server is not happening :)

My understanding is the same as Trond's--the reason it's not happening is because nobody is making an effort to merge it. What am I missing?

--b.

Quoted 29 lines · J. Bruce Fields

Quoted 27 lines · Marc Eshel

Trond Myklebust <[email protected]> wrote on 11/29/2011 04:37:05 PM:

Quoted 6 lines

Peng Tao, bhalevy, Boaz Harrosh, Garth Gibson, Fred Isaman, linux- nfs, Matt Benjamin

Quoted 1 line · Marc Eshel

You ignored my main point, I was talking about the server side, my

point

Quoted 9 lines

Quoted 4 lines

was that there is nothing to build on on the serve side since the pNFS

Linux server is not happening.
Marc.

Sorry. I misunderstood your concern. As far as I know, the main problem there is also one of investment: nobody has stepped up to help Bruce write a pNFS server.

Quoted 6 lines

I'm less worried about this now than I was earlier, because other open source efforts are gaining traction (see Ganesha - which is being sponsored by IBM, and projects such as Tigran's java based pNFS server). The other point is that we've developed client test-rigs that don't depend on the availability of a Linux server (newpynfs and the pynfs based proxy).

You got it backward, Ganesha is getting traction precisely because the Linux kernel server is not happening :)

Quoted 3 lines

My understanding is the same as Trond's--the reason it's not happening is because nobody is making an effort to merge it. What am I missing?

Tonian is working on this and we've allocated resources also to help out on your nfs4.1 todo list.

Quoted 7 lines · Benny Halevy

Quoted 3 lines · J. Bruce Fields

My understanding is the same as Trond's--the reason it's not happening is because nobody is making an effort to merge it. What am I missing?

Tonian is working on this and we've allocated resources also to help out on your nfs4.1 todo list.

Great, thanks; let me know how I can help.

Quoted 9 lines · Marc Eshel

The only 1 cent that I can add to this argument is that I was led to believe by you and others that Linux kernel don't add functionality on the client side that is not supported on the server side. Last time I made this point I was told that it is ok if they are of by a version or two. You made it clear that this is no longer true and that the Linux client and server are now independent of each other. I spent time working on the server side believing that the client and server will progress more or less together, disappointed to find out that it is not the case any more. Marc.

I don't know how to manage layout segments in a way that meets the pNFS goals of scalability and performance, and neither you nor the pNFS spec have told me how to do this.

If IBM wants a client that implements layout segments, then it is _your_ responsibility to:
A. Convince me that it is an implementable part of the spec.

B. Provide me with an implementation that deals all with the
concerns that I have.

IOW: nobody has ever promised IBM that the community would do all your client work for you. EMC, NetApp and Panasas have implemented (most of) the bits that they cared about; any bits that they haven't implemented and that you care about are up to you.

Quoted 7 lines · Boaz Harrosh

The kind of typologies I'm talking about a single layout get ever 1GB is marginal to the gain I get in deploying 100 of DSs. I have thousands of DSs I want to spread the load evenly. I'm limited by the size of the layout (Device info in the case of files) So I'm limited by the number of DSs I can have in a layout. For large files these few devices become an hot spot all the while the rest of the cluster is idle.

I call "bullshit" on that whole argument...

You've done sod all so far to address the problem of a client managing

layout segments for a '1000 DS' case. Are you expecting that all pNFS object servers out there are going to do that for you? How do I assume that a generic pNFS files server is going to do the same? As far as I know, the spec is completely moot on the whole subject.

IOW: I'm not even remotely interested in your "everyday problems" if there are no "everyday solutions" that actually fit the generic can of spec worms that the pNFS layout segments open.

Quoted 14 lines

Quoted 8 lines

Quoted 5 lines

Two: There are already file-layout servers out there (multiple) which are
waiting for the Linux files-layout segment support, because the underline FS requires Segments and now they do not work with the Linux client. These are CEPH and GPFS and more.

Then they will have a _long_ wait....

OK, so now I understand. Because when I was talking to Fred before BAT and during It was very very peculiar to me why he is not already done with that simple stuff. Because usually Fred is such a brilliant fast programmer that I admire, and that simple crap?

Quoted 1 line

But now that explains

Yes. It's all a big conspiracy, and we're deliberately holding Fred's genius back in order to disappoint you...

Quoted 9 lines · Trond Myklebust

Quoted 7 lines · Boaz Harrosh

The kind of typologies I'm talking about a single layout get ever 1GB is marginal to the gain I get in deploying 100 of DSs. I have thousands of DSs I want to spread the load evenly. I'm limited by the size of the layout (Device info in the case of files) So I'm limited by the number of DSs I can have in a layout. For large files these few devices become an hot spot all the while the rest of the cluster is idle.

Quoted 3 lines

I call "bullshit" on that whole argument...

You've done sod all so far to address the problem of a client managing

sod? I don't know this word?

Quoted 5 lines

layout segments for a '1000 DS' case. Are you expecting that all pNFS object servers out there are going to do that for you? How do I assume that a generic pNFS files server is going to do the same? As far as I know, the spec is completely moot on the whole subject.

What? The all segments thing is in the Generic part of the spec and is not at all specific or even specified in the objects and blocks RFCs.

There is no layout in the spec, there are only layout_segments. Actually what we call layout_segments, in the spec, it is simply called a layout.

The client asks for a layout (segment) and gets one. An ~0 length one is just a special case. Without layout_get (segment) there is no optional pnfs support.

So we are reading two different specs because to me it clearly says layout - which is a segment.

Because the way I read it the pNFS is optional in 4.1. But if I'm a pNFS client I need to expect layouts (segments)

Quoted 3 lines

IOW: I'm not even remotely interested in your "everyday problems" if there are no "everyday solutions" that actually fit the generic can of spec worms that the pNFS layout segments open.

That I don't understand. What "spec worms that the pNFS layout segments open" Are you seeing. Because it works pretty simple for me. And I don't see the big difference for files. One thing I learned for the past is that when you have concerns I should understand them and start to address them. Because your insights are usually on the Money. If you are concerned then there is something I should fix.

Quoted 9 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

On Tue, 2011-11-29 at 14:58 -0800, Boaz Harrosh wrote:

That I don't understand. What "spec worms that the pNFS layout segments open" Are you seeing. Because it works pretty simple for me. And I don't see the big difference for files. One thing I learned for the past is that when you have concerns I should understand them and start to address them. Because your insights are usually on the Money. If you are concerned then there is something I should fix.

I'm saying that if I need to manage layouts that deal with >1000 DSes, then I presumably need a strategy for ensuring that I return/forget segments that are no longer needed, and I need a strategy for ensuring that I always hold the segments that I do need; otherwise, I could just ask for a full-file layout and deal with the 1000 DSes (which is what we do today)...

My problem is that the spec certainly doesn't give me any guidance as to such a strategy, and I haven't seen anybody else step up to the plate. In fact, I strongly suspect that such a strategy is going to be very application specific.

IOW: I don't accept that a layout-segment based solution is useful without some form of strategy for telling me which segments to keep and which to throw out when I start hitting client resource limits.

I also
haven't seen any strategy out there for setting loga_length (as opposed to loga_minlength) in the LAYOUTGET requests: as far as I know that is going to be heavily application-dependent in the 1000-DS world.

Quoted 11 lines · Trond Myklebust

Quoted 9 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

On Tue, 2011-11-29 at 14:58 -0800, Boaz Harrosh wrote:

That I don't understand. What "spec worms that the pNFS layout segments open" Are you seeing. Because it works pretty simple for me. And I don't see the big difference for files. One thing I learned for the past is that when you have concerns I should understand them and start to address them. Because your insights are usually on the Money. If you are concerned then there is something I should fix.

Quoted 7 lines

I'm saying that if I need to manage layouts that deal with >1000 DSes, then I presumably need a strategy for ensuring that I return/forget segments that are no longer needed, and I need a strategy for ensuring that I always hold the segments that I do need; otherwise, I could just ask for a full-file layout and deal with the 1000 DSes (which is what we do today)...

Thanks for asking because now I can answer you and you will find that I'm one step a head in some of the issues.

1. The 1000 DSes problem is separate from the segments problem. The devices

solution is on the way. The device cache is all but ready to see some
periodic scan that throws 0 used devices. We never got to it because currently every one is testing with up to 10 devices and I'm using upto 128 devices which is just fine. The load is marginal so far. But I promise you it is right here on my to do list. After some more pressed problem.
Lets say one thing this subsystem is the same regardless of if the 1000 devices are refed by 1 segment or by 10 segments. Actually if by 10 then I might get rid of some and free devices.

2. The many segments problem. There are not that many. It's more less a segment for every 2GB so an lo_seg struct for so much IO is not noticeable.

At the upper bound we do not have any problem because Once the system is
out of memory it will start to evict inodes. And on evict we just return them. Also ROC Servers we forget them on close. So so far all our combined testing did not show any real memory pressure caused by that. When shown we can start discarding segs in an LRU fashion. There are all the mechanics to do that, we only need to see the need.

3. The current situation is fine and working and showing great performance for objects and blocks. And it is all in the Generic part so it should just be the same for files. I do not see any difference.

The only BUG I see is the COMMIT and I think we know how to fix that

Quoted 5 lines

My problem is that the spec certainly doesn't give me any guidance as to such a strategy, and I haven't seen anybody else step up to the plate. In fact, I strongly suspect that such a strategy is going to be very application specific.

You never asked. I'm thinking about these things all the time. Currently we are far behind the limits of a running system. I think I'm going to get to these limits before any one else.

My strategy is stated above LRU for devices is almost all there ref-counting and all only the periodic timer needs to be added.
LRU for segments is more work, but is doable. But the segments count are so low that we will not hit that problem for a long time. Before I ship a system that will break that barrier I'll send a fix I promise.

Quoted 3 lines

IOW: I don't accept that a layout-segment based solution is useful without some form of strategy for telling me which segments to keep and which to throw out when I start hitting client resource limits.

LRU. Again there are not more than a few segments per inode. It's not 1000 like devices.

Quoted 5 lines

I also
haven't seen any strategy out there for setting loga_length (as opposed to loga_minlength) in the LAYOUTGET requests: as far as I know that is going to be heavily application-dependent in the 1000-DS world.

Current situation is working for me. But we also are actively working to improve it. What we want is that files-LO can enjoy the same privileges that objects and blocks already have, in exactly the same, simple stupid but working, way.

All your above concerns are true and interesting. I call them a rich man problems. But they are not specific to files-LO they are generic to all of us. Current situation satisfies us for blocks and objects. The file guys out there are jealous.

Quoted 2 lines

1. The 1000 DSes problem is separate from the segments problem. The devices

Errr... That was the problem that you used to justify the need for a full implementation of layout segments in the pNFS files case...

Quoted 10 lines

solution is on the way. The device cache is all but ready to see some
periodic scan that throws 0 used devices. We never got to it because currently every one is testing with up to 10 devices and I'm using upto 128 devices which is just fine. The load is marginal so far. But I promise you it is right here on my to do list. After some more pressed problem.
Lets say one thing this subsystem is the same regardless of if the 1000 devices are refed by 1 segment or by 10 segments. Actually if by 10 then I might get rid of some and free devices.

Quoted 3 lines

2. The many segments problem. There are not that many. It's more less a segment for every 2GB so an lo_seg struct for so much IO is not noticeable.

Where do you get that 2GB number from?

Quoted 6 lines

At the upper bound we do not have any problem because Once the system is
out of memory it will start to evict inodes. And on evict we just return them. Also ROC Servers we forget them on close. So so far all our combined testing did not show any real memory pressure caused by that. When shown we can start discarding segs in an LRU fashion. There are all the mechanics to do that, we only need to see the need.

It's not necessarily that simple: if you are already low on memory, them LAYOUTGET and GETDEVICE will require you to allocate more memory in order to get round to cleaning those dirty pages.
There are plenty of situations where the majority of dirty pages belong to a single file. If that file is one of your 1000 DS-files and it requires you to allocate 1000 new device table entries...

Quoted 5 lines

3. The current situation is fine and working and showing great performance for objects and blocks. And it is all in the Generic part so it should just be the same for files. I do not see any difference.

The only BUG I see is the COMMIT and I think we know how to fix that

I haven't seen any performance numbers for either, so I can't comment.

Quoted 3 lines

LRU. Again there are not more than a few segments per inode. It's not 1000 like devices.

Again, the problem for files shouldn't be the number of segments, it is number of devices.

Quoted 4 lines

All your above concerns are true and interesting. I call them a rich man problems. But they are not specific to files-LO they are generic to all of us. Current situation satisfies us for blocks and objects. The file guys out there are jealous.

I'm not convinced that the problems are the same. objects, and particularly blocks, appear to treat layout segments as a form of byte range lock. There is no reason for a pNFS files server to do so.

Quoted 10 lines · Boaz Harrosh

Quoted 9 lines · Trond Myklebust

Quoted 7 lines · Boaz Harrosh

The kind of typologies I'm talking about a single layout get ever 1GB is marginal to the gain I get in deploying 100 of DSs. I have thousands of DSs I want to spread the load evenly. I'm limited by the size of the layout (Device info in the case of files) So I'm limited by the number of DSs I can have in a layout. For large files these few devices become an hot spot all the while the rest of the cluster is idle.

Quoted 5 lines

Quoted 3 lines

I call "bullshit" on that whole argument...

You've done sod all so far to address the problem of a client managing

sod? I don't know this word?

'sod all' == 'nothing'

it's an English slang...

Quoted 8 lines

Quoted 5 lines

layout segments for a '1000 DS' case. Are you expecting that all pNFS object servers out there are going to do that for you? How do I assume that a generic pNFS files server is going to do the same? As far as I know, the spec is completely moot on the whole subject.

What? The all segments thing is in the Generic part of the spec and is not at all specific or even specified in the objects and blocks RFCs.

..and it doesn't say _anything_ about how a client is supposed to manage them in order to maximise efficiency.

Quoted 17 lines

There is no layout in the spec, there are only layout_segments. Actually what we call layout_segments, in the spec, it is simply called a layout.

The client asks for a layout (segment) and gets one. An ~0 length one is just a special case. Without layout_get (segment) there is no optional pnfs support.

So we are reading two different specs because to me it clearly says layout - which is a segment.

Because the way I read it the pNFS is optional in 4.1. But if I'm a pNFS client I need to expect layouts (segments)

Quoted 3 lines

IOW: I'm not even remotely interested in your "everyday problems" if there are no "everyday solutions" that actually fit the generic can of spec worms that the pNFS layout segments open.

Quoted 6 lines

That I don't understand. What "spec worms that the pNFS layout segments open" Are you seeing. Because it works pretty simple for me. And I don't see the big difference for files. One thing I learned for the past is that when you have concerns I should understand them and start to address them. Because your insights are usually on the Money. If you are concerned then there is something I should fix.

I'm saying that if I need to manage layouts that deal with >1000 DSes, then I presumably need a strategy for ensuring that I return/forget segments that are no longer needed, and I need a strategy for ensuring that I always hold the segments that I do need; otherwise, I could just ask for a full-file layout and deal with the 1000 DSes (which is what we do today)...111

My problem is that the spec certainly doesn't give me any guidance as to such a strategy, and I haven't seen anybody else step up to the plate. In fact, I strongly suspect that such a strategy is going to be very application specific.

IOW: I don't accept that a layout-segment based solution is useful without some form of strategy for telling me which segments to keep and which to throw out when I start hitting client resource limits. I also haven't seen any strategy out there for setting loga_length (as opposed to loga_minlength) in the LAYOUTGET requests: as far as I know that is going to be heavily application-dependent in the 1000-DS world.

Quoted 2 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

Quoted 1 line

Also Files when they will support segments and servers that

request segments,

Quoted 1 line

Quoted 1 line

Quoted 1 line

like the CEPH server, will very much enjoy the above, .i.e: Tell

me the amount

Quoted 3 lines

Quoted 3 lines

Quoted 1 line

you know you want to write.

Why would we want to add segment support to the pNFS files

client???

Quoted 1 line

Quoted 1 line

Segments are a nuisance that multiply the amount of unnecessary

chitchat

Quoted 7 lines

Quoted 3 lines

between the client and the MDS without providing any tangible benefits...

Your kidding right?

One: it is mandated by the Standard, This is not an option. So a

perfectly

Quoted 1 line

Standard complaint server is not Supported by Linux because we

don't see

Quoted 1 line

the point.

Bollocks.. Nothing is "mandated by the Standard". If the server doesn't
give us a full layout, then we fall back to write through MDS. Why dick
around with crap that SLOWS YOU DOWN.

Quoted 1 line

Two: There are already file-layout servers out there (multiple)

which are

Quoted 1 line

waiting for the Linux files-layout segment support, because the

underline

Quoted 1 line

FS requires Segments and now they do not work with the Linux

client. These

Quoted 1 line

are CEPH and GPFS and more.

Then they will have a _long_ wait....

Trond

Signature · 6 lines

Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

That would be pretty disappointing. However, based on previous interactions, my belief would be, the
Linux client will do what can be shown empirically to work better, or more correctly.

Matt

Quoted 5 lines · ----- "Trond Myklebust

Quoted 2 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

Quoted 1 line

Also Files when they will support segments and servers that

request segments,

Quoted 1 line

Quoted 1 line

Quoted 1 line

like the CEPH server, will very much enjoy the above, .i.e:

Tell

Quoted 7 lines

me the amount

Quoted 3 lines

Quoted 3 lines

Quoted 1 line

you know you want to write.

Why would we want to add segment support to the pNFS files

client???

Quoted 1 line

Quoted 1 line

Segments are a nuisance that multiply the amount of unnecessary

chitchat

Quoted 9 lines

Quoted 7 lines

Quoted 3 lines

between the client and the MDS without providing any tangible benefits...

Your kidding right?

One: it is mandated by the Standard, This is not an option. So a

perfectly

Quoted 1 line

Standard complaint server is not Supported by Linux because

we

Quoted 12 lines

don't see

Quoted 1 line

the point.

Bollocks.. Nothing is "mandated by the Standard". If the server doesn't
give us a full layout, then we fall back to write through MDS. Why dick
around with crap that SLOWS YOU DOWN.

Quoted 1 line

Two: There are already file-layout servers out there (multiple)

which are

Quoted 1 line

waiting for the Linux files-layout segment support, because

the

Quoted 9 lines

underline

Quoted 1 line

FS requires Segments and now they do not work with the Linux

client. These

Quoted 1 line

are CEPH and GPFS and more.

Then they will have a _long_ wait....

Trond

Quoted 7 lines

--
Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

--

Matt Benjamin

The Linux Box
206 South Fifth Ave. Suite 150
Ann Arbor, MI 48104

http://linuxbox.com

tel. 734-761-4689
fax. 734-769-8938
cel. 734-216-5309

Let me clarify: there are files based servers, our Ceph on Ganesha server is one, which have file allocation not satisfied by whole-file layouts. I would think that demonstrating this would be sufficient to get support from the Linux client to support appropriate segment management, at any rate, if someone is willing to write and support the required code, or already has. One of those alternatives is certainly the case. By the way, we wrote generic pNFS, pNFS files support for Ganesha and, with a big dose of help from Panasas, are taking it to merge.

Matt

Quoted 75 lines · ----- "Matt W. Benjamin

That would be pretty disappointing. However, based on previous interactions, my belief would be, the
Linux client will do what can be shown empirically to work better, or more correctly.

Matt

Quoted 5 lines · ----- "Trond Myklebust

Quoted 2 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

Quoted 1 line

Also Files when they will support segments and servers that

request segments,

Quoted 1 line

Quoted 1 line

Quoted 1 line

like the CEPH server, will very much enjoy the above, .i.e:

Tell

Quoted 7 lines

me the amount

Quoted 3 lines

Quoted 3 lines

Quoted 1 line

you know you want to write.

Why would we want to add segment support to the pNFS files

client???

Quoted 1 line

Quoted 1 line

Segments are a nuisance that multiply the amount of unnecessary

chitchat

Quoted 9 lines

Quoted 7 lines

Quoted 3 lines

between the client and the MDS without providing any tangible benefits...

Your kidding right?

One: it is mandated by the Standard, This is not an option. So a

perfectly

Quoted 1 line

Standard complaint server is not Supported by Linux because

we

Quoted 12 lines

don't see

Quoted 1 line

the point.

Bollocks.. Nothing is "mandated by the Standard". If the server doesn't
give us a full layout, then we fall back to write through MDS. Why dick
around with crap that SLOWS YOU DOWN.

Quoted 1 line

Two: There are already file-layout servers out there (multiple)

which are

Quoted 1 line

waiting for the Linux files-layout segment support, because

the

Quoted 9 lines

underline

Quoted 1 line

FS requires Segments and now they do not work with the Linux

client. These

Quoted 1 line

are CEPH and GPFS and more.

Then they will have a _long_ wait....

Trond

Quoted 7 lines

--
Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

--

Matt Benjamin

The Linux Box
206 South Fifth Ave. Suite 150
Ann Arbor, MI 48104

http://linuxbox.com

tel. 734-761-4689
fax. 734-769-8938
cel. 734-216-5309

Quoted 1 line · Matt W. Benjamin

Let me clarify: there are files based servers, our Ceph on Ganesha

server is one, which have file allocation not satisfied by whole-file layouts. I would think that demonstrating this would be sufficient to get support from the Linux client to support appropriate segment management, at any rate, if someone is willing to write and support the required code, or already has. One of those alternatives is certainly the case. By the way, we wrote generic pNFS, pNFS files support for Ganesha and, with a big dose of help from Panasas, are taking it to merge.

I really want more than that. Please see the reply that I just sent to
Boaz: I need a client strategy for managing partial layout segments in
the case where holding a whole-file layout is not acceptable. Otherwise,
what we have now should be sufficient...

Trond

Quoted 7 lines

Matt

Quoted 3 lines · ----- "Matt W. Benjamin

That would be pretty disappointing. However, based on previous interactions, my belief would be, the
Linux client will do what can be shown empirically to work better,

or

Quoted 9 lines

Quoted 9 lines

more correctly.

Matt

Quoted 3 lines · ----- "Trond Myklebust

Quoted 2 lines · Boaz Harrosh

Quoted 1 line · Trond Myklebust

Quoted 1 line

Also Files when they will support segments and servers

that

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

request segments,

Quoted 1 line

Quoted 1 line

Quoted 1 line

like the CEPH server, will very much enjoy the above,

.i.e:

Quoted 7 lines

Quoted 7 lines

Tell

Quoted 6 lines

me the amount

Quoted 3 lines

Quoted 3 lines

Quoted 1 line

you know you want to write.

Why would we want to add segment support to the pNFS files

client???

Quoted 1 line

Quoted 1 line

Segments are a nuisance that multiply the amount of

unnecessary

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

chitchat

Quoted 1 line

Quoted 1 line

between the client and the MDS without providing any

tangible

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 2 lines

benefits...

Your kidding right?

One: it is mandated by the Standard, This is not an option. So

a

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

perfectly

Quoted 1 line

Standard complaint server is not Supported by Linux

because

Quoted 7 lines

Quoted 7 lines

we

Quoted 6 lines

don't see

Quoted 1 line

the point.

Bollocks.. Nothing is "mandated by the Standard". If the server doesn't
give us a full layout, then we fall back to write through MDS.

Why

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

dick
around with crap that SLOWS YOU DOWN.

Quoted 1 line

Two: There are already file-layout servers out there

(multiple)

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

which are

Quoted 1 line

waiting for the Linux files-layout segment support,

because

Quoted 3 lines

Quoted 3 lines

the

Quoted 2 lines

underline

Quoted 1 line

FS requires Segments and now they do not work with the

Linux

Quoted 29 lines

Quoted 28 lines

Quoted 7 lines

client. These

Quoted 1 line

are CEPH and GPFS and more.

Then they will have a _long_ wait....

Trond

Quoted 7 lines

--
Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

--

Matt Benjamin

The Linux Box
206 South Fifth Ave. Suite 150
Ann Arbor, MI 48104

http://linuxbox.com

tel. 734-761-4689
fax. 734-769-8938
cel. 734-216-5309

Signature · 6 lines

Trond Myklebust
Linux NFS client maintainer

NetApp
[email protected]
www.netapp.com

The reason Netapp don't see a point is not a factor. I have not seen a netapp server doing a top 100 HPC cluster either.

Quoted 2 lines · Boaz Harrosh

The reason Netapp don't see a point is not a factor. I have not seen a netapp server doing a top 100 HPC cluster either.

BTW: Stop adding this "NetApp" insinuation bullshit...

I _strongly_ resent the fact that you appear to think I'm incapable of thinking for myself...

Quoted 26 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Boaz Harrosh
Sent: Wednesday, November 30, 2011 5:34 AM
To: Peng Tao
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH 0/4] nfs41: allow layoutget at pnfs_do_multiple_writes

Quoted 4 lines · Peng Tao

Issuing layoutget at .pg_init will drop the IO size information and ask for 4KB layout every time. However, the IO size information is very valuable for MDS to determine how much layout it should return to client.

Quoted 9 lines

The patchset try to allow LD not to send layoutget at .pg_init but instead at pnfs_do_multiple_writes. So that real IO size is preserved and sent to MDS.

Tests against a server that does not aggressively pre-allocate layout, shows that the IO size informantion is really useful to block layout MDS.

The generic pnfs layer changes are trival to file layout and object as long as they still send layoutget at .pg_init.

I have a better solution for your problem. Which is a much smaller a change and I think gives you much better heuristics.

Quoted 7 lines

Keep the layout_get exactly where it is, but instead of sending PAGE_SIZE send the amount of dirty pages you have.

If it is a linear write you will be exact on the money with a single lo_get. If it is an heavy random write then you might need more lo_gets and you might be getting some unused segments. But heavy random write is rare and slow anyway. As a first approximation its fine. (We can later fix that as well)

I would say no to the above... For objects/files MDS, it may not hurt much to allocate wasting layout. But for blocklayout server, each layout allocation consumes much more resource than just giving out stripping information like objects/files.

So helping MDS to do the
correct decision is the right thing for client to do.

Quoted 11 lines

The .pg_init is done after .write_pages call from VFS and all the to-be-written pages are already staged to be written. So there should be a way to easily extract that information.

Quoted 2 lines

iozone cmd:
./iozone -r 1m -s 4G -w -W -c -t 10 -i 0 -F /mnt/iozone.data.1 /mnt/iozone.data.2 /mnt/iozone.data.3

/mnt/iozone.data.4 /mnt/iozone.data.5 /mnt/iozone.data.6 /mnt/iozone.data.7 /mnt/iozone.data.8 /mnt/iozone.data.9 /mnt/iozone.data.10

Quoted 1 line

Quoted 8 lines

Quoted 3 lines

Befor patch: around 12MB/s throughput
After patch: around 72MB/s throughput

Yes Yes that stupid Brain dead Server is no indication for anything. The server should know best about optimal sizes and layouts. Please don't give me that stuff again.

Actually the server is already doing layout pre-allocation. It is just that it doesn't know what client really wants so cannot do it too aggressively. That's why I wanted to make client to send the REAL IO size information to server. From performance perspective, dropping IO size information is always a BAD THING(TM) to do.

Quoted 4 lines

BTW don't limit the lo_segment size by the max_io_size. This is why you have .bg_test to signal when IO is maxed out.

Actually lo_segment size is never limited by max_io_size. Server is always entitled to send larger layout than client asks from.

Quoted 7 lines

- The read segments should be as big as possible (i_size long)
- The Write segments should ideally be as big as the Application wants to write to. (Amount of dirty pages at time of nfs-write-out is a very good first approximation).

So I guess it is: I hate these patches, to much mess, too little goodness.

I'm afraid I can't agree with you...

Thanks,
Tao

Quoted 42 lines

-----Original Message-----
From: Boaz Harrosh [mailto:[email protected]]
Sent: Wednesday, November 30, 2011 11:51 AM
To: Peng, Tao
Cc: [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH 0/4] nfs41: allow layoutget at pnfs_do_multiple_writes

Quoted 34 lines · On 11/29/2011 07:16 PM, [email protected] wrote:

Quoted 33 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Boaz Harrosh
Sent: Wednesday, November 30, 2011 5:34 AM
To: Peng Tao
Cc: [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH 0/4] nfs41: allow layoutget at pnfs_do_multiple_writes

Quoted 4 lines · Peng Tao

Issuing layoutget at .pg_init will drop the IO size information and ask for 4KB layout every time. However, the IO size information is very valuable for MDS to determine how much layout it should return to client.

Quoted 9 lines

The patchset try to allow LD not to send layoutget at .pg_init but instead at pnfs_do_multiple_writes. So that real IO size is preserved and sent to MDS.

Tests against a server that does not aggressively pre-allocate layout, shows that the IO size informantion is really useful to block layout MDS.

The generic pnfs layer changes are trival to file layout and object as long as they still send layoutget at .pg_init.

I have a better solution for your problem. Which is a much smaller a change and I think gives you much better heuristics.

Keep the layout_get exactly where it is, but instead of sending PAGE_SIZE send the amount of dirty pages you have.

If it is a linear write you will be exact on the money with a single lo_get. If it is an heavy random write then you might need more lo_gets and you might be getting some unused segments. But heavy random write is rare and slow anyway. As a first approximation its fine. (We can later fix that as well)

Quoted 8 lines

Quoted 4 lines

I would say no to the above... For objects/files MDS, it may not hurt much to allocate wasting layout. But for blocklayout server, each layout allocation consumes much more resource than just giving out stripping information like objects/files.

That's fine, for the linear IO like iozone below my way is just the same as yours. For the random IO I'm not sure how much better will your solution be. Not by much.

As I said, for random IO, there will be much disk space wasting on blocklayout server in your solution. That's why I don't agree with it. Besides, in some cases, server may be put in a hard position to determine if the IO is really linear or in fact random in your solution.

Quoted 4 lines

I want a solution for objects as well. But I cannot use yours because I need a layout before the final request consolidation. Solve my problem too.

I used to look at objects at some time, and as I remember, it need max io size in each lseg to finish .pg_test. Is this the reason you need a layout before the final request consolidation? Does the value vary in different lseg?

Quoted 7 lines

Quoted 2 lines

So helping MDS to do the
correct decision is the right thing for client to do.

I agree. All I'm saying is that there is available information at the time of .pg_init to send that number just fine. Have you looked? it's all there NFS core can tell you how many pages have passed ->write_pages.

It only tells a fake IO size for the number of dirty pages. No one can promise these pages are all continuous. Instead, if we can give a real IO size, why refuse to do it?

Quoted 20 lines

Quoted 8 lines

Quoted 8 lines

The .pg_init is done after .write_pages call from VFS and all the to-be-written pages are already staged to be written. So there should be a way to easily extract that information.

Quoted 2 lines

iozone cmd:
./iozone -r 1m -s 4G -w -W -c -t 10 -i 0 -F /mnt/iozone.data.1 /mnt/iozone.data.2

/mnt/iozone.data.3

Quoted 11 lines

Quoted 11 lines

/mnt/iozone.data.4 /mnt/iozone.data.5 /mnt/iozone.data.6 /mnt/iozone.data.7 /mnt/iozone.data.8 /mnt/iozone.data.9 /mnt/iozone.data.10

Quoted 4 lines

Befor patch: around 12MB/s throughput
After patch: around 72MB/s throughput

Yes Yes that stupid Brain dead Server is no indication for anything. The server should know best about optimal sizes and layouts. Please don't give me that stuff again.

Quoted 9 lines

Quoted 5 lines

Actually the server is already doing layout pre-allocation. It is just that it doesn't know what client really wants so cannot do it too aggressively. That's why I wanted to make client to send the REAL IO size information to server. From performance perspective, dropping IO size information is always a BAD THING(TM) to do.

I totally agree. I want it too. There is a way to do it in pg_init time all the information is there it only needs to be passed to layout_get.

This "all the information is there" is likely to be false, unless you only deal with sequential IO...

Quoted 13 lines

Quoted 6 lines

Quoted 4 lines

BTW don't limit the lo_segment size by the max_io_size. This is why you have .bg_test to signal when IO is maxed out.

Actually lo_segment size is never limited by max_io_size. Server is always entitled to send larger layout than client asks from.

You miss my point. In your last patch you have

(quoted diff) +3 -0

+/* While RFC doesn't limit maximum size of layout, we better limit it ourself. */
+#define PNFSBLK_MAXRSIZE (0x1<<22)
+#define PNFSBLK_MAXWSIZE (0x1<<21)

Quoted 3 lines

I don't know what these number mean but they kind of look like IO limits and not segment limits. If I'm wrong then sorry. What are these numbers?

Yes, these are io size limit. I should just remove the comments that are totally misleading.

Quoted 3 lines

If a client has 1G of dirty pages to write why not get the full layout at once. Where does the 4M limit comes from?

Please note that block layout server cannot just give full file stripping information. When we ask for 1GB layout, most times we get much less anyway. And the value of 2MB comes from our experience with MPFS, to allow the balance between server pressure and client performance.

Also, currently we retry MDS on a per nfs_read/write_data basis. It is much easier to handle 2MB rather than 1GB dirty pages. I notice that it may not be an issue for objects as you have max IO size limit on every lseg.

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

- The read segments should be as big as possible (i_size long)
- The Write segments should ideally be as big as the Application wants to write to. (Amount of dirty pages at time of nfs-write-out is a very good first approximation).

Quoted 8 lines

Quoted 3 lines

Quoted 1 line

So I guess it is: I hate these patches, to much mess, too little goodness.

I'm afraid I can't agree with you...

Sure you do. You did the hard work and now I'm telling you you need to do more work. I'm sorry for that. But I want a solution for me and I think there is a simple solution that will satisfy both of our needs.

Sorry but I don't think your solution is good enough to address blocklayout's concerns. It would be great if we can utilize the same solution. But when we do need, I think it perfectly reasonable to let blocklayout and object layout have different strategy on layoutget, based on the fact that our servers have different behavior on layout allocation. And allowing this kind of difference is exactly what strcut nfs_pageio_ops serves for.

What do you think?

Thanks,
Tao

Quoted 7 lines

Sorry for that. If I had time I would do it. Only I have harder real BUGS to fix on my plate.

If you could look into it It will be very nice. And thank you for working on this so far. Only that current solution is not optimal and I will need to continue on it later, if left as is.

Quoted 5 lines

Quoted 3 lines

Thanks,
Tao

Thanks

N�����r��y���b�X��ǧv�^�)޺{.n�+����{���"��^n�r��z���h����&���G���h�(�階�ݢj"���m�����z�ޖ���f���h���~�mml==