NFSD: Add READ_PLUS support
Subsystems (2)
v3 cover NFSD: Add READ_PLUS support
These patches add server support for the NFS v4.2 operation READ_PLUS.
I tested these patches using xfstests and compared the runtime between NFS v4.1, NFS v4.2 without READ_PLUS, and NFS v4.2 with READ_PLUS. Here are the results:
Test | v4.1 | no READ_PLUS | READ_PLUS
---------------------------------------------------
generic/013 | 135s | 143s | 123s generic/075 | 4s | 11s | 4s generic/091 | 9s | 16s | 15s generic/112 | 4s | 7s | 6s generic/127 | 62s | 117s | 114s generic/213 | [not run] | 1s | 1s generic/214 | [not run] | 0s | 0s generic/228 | [not run] | 1s | 2s generic/236 | 1s | 1s | 1s generic/263 | 4s | 6s | 8s generic/285 | 0s | 1s | 3s generic/315 | [not run] | 2s | 1s
---------------------------------------------------
Total | 3:47.47 | 5:11.85 | 4:43.77
Using the READ_PLUS operation does have an impact on reading sparse files over the network. There is still a big difference between v4.1 and v4.2 runtimes, but this looks to be a result of fallocate() tests that only run over v4.2.
Changes since v2:
- Encode data segments using splice reads whenever possible.
- Combine patches for adding HOLE and DATA segment support.
These patches and the corresponding client changes are available in the [read_plus] branch of
git://git.linux-nfs.org/projects/anna/linux-nfs.git
Questions? Comments? Thoughts?
Anna
Anna Schumaker (3):
NFSD: nfsd4_encode_read{v}() should encode eof and maxcount
NFSD: Add basic READ_PLUS support
NFSD: Add support for encoding multiple segments
fs/nfsd/nfs4proc.c | 16 +++++ fs/nfsd/nfs4xdr.c | 182 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 160 insertions(+), 38 deletions(-)
Signature · 1 line
2.3.3
PATCH v3 1/3 NFSD: nfsd4_encode_read{v}() should encode eof and maxcount
I intend to reuse nfsd4_encode_readv() for READ_PLUS, so I need an alternate way to encode these values. I think it makes sense for nfsd4_encode_read() to handle this in a single place for both splice and readv paths.
--- fs/nfsd/nfs4xdr.c | 59 ++++++++++++++++++++++--------------------------------- 1 file changed, 23 insertions(+), 36 deletions(-)
fs/nfsd/nfs4xdr.c +23 -36
@@ -3286,21 +3286,19 @@ nfsd4_encode_open_downgrade(struct nfsd4_compoundres *resp, __be32 nfserr, strucstatic __be32 nfsd4_encode_splice_read (struct nfsd4_compoundres *resp,struct nfsd4_read *read, -struct file *file,unsigned long maxcount) +struct file *file,unsigned long *maxcount) {struct xdr_stream *xdr = &resp->xdr;struct xdr_buf *buf = xdr->buf; -u32 eof;int space_left;__be32 nfserr; -__be32 *p = xdr->p -2 ;/* Make sure there will be room for padding if needed */ if (xdr->end - xdr->p <1 )return nfserr_resource; nfserr =nfsd_splice_read (read->rd_rqstp, file, - read->rd_offset, &maxcount); + read->rd_offset, maxcount);if (nfserr) {/* * nfsd_splice_actor may have already messed with the
@@ -3311,27 +3309,21 @@ static __be32 nfsd4_encode_splice_read(return nfserr; } - eof = (read->rd_offset + maxcount >= - read->rd_fhp->fh_dentry->d_inode->i_size); - - *(p++) =htonl (eof); - *(p++) =htonl (maxcount); - - buf->page_len = maxcount; - buf->len += maxcount; - xdr->page_ptr += (buf->page_base + maxcount +PAGE_SIZE -1 ) + buf->page_len = *maxcount; + buf->len += *maxcount; + xdr->page_ptr += (buf->page_base + *maxcount +PAGE_SIZE -1 ) /PAGE_SIZE ;/* Use rest of head for padding and remaining ops: */ buf->tail[0 ].iov_base = xdr->p; buf->tail[0 ].iov_len =0 ; xdr->iov = buf->tail; -if (maxcount&3 ) { -int pad =4 - (maxcount&3 ); +if (*maxcount&3 ) { +int pad =4 - (*maxcount&3 ); *(xdr->p++) =0 ; - buf->tail[0 ].iov_base += maxcount&3 ; + buf->tail[0 ].iov_base += *maxcount&3 ; buf->tail[0 ].iov_len = pad; buf->len += pad; }
@@ -3346,21 +3338,19 @@ static __be32 nfsd4_encode_splice_read(static __be32 nfsd4_encode_readv (struct nfsd4_compoundres *resp,struct nfsd4_read *read, -struct file *file,unsigned long maxcount) +struct file *file,unsigned long *maxcount) {struct xdr_stream *xdr = &resp->xdr; -u32 eof;int v; -int starting_len = xdr->buf->len -8 ; +int starting_len = xdr->buf->len;long len;int thislen;__be32 nfserr; -__be32 tmp;__be32 *p;u32 zzz =0 ;int pad; - len = maxcount; + len = *maxcount; v =0 ; thislen =min_t (long, len, ((void *)xdr->end - (void *)xdr->p));
@@ -3383,22 +3373,13 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp, read->rd_vlen = v; nfserr =nfsd_readv (file, read->rd_offset, resp->rqstp->rq_vec, - read->rd_vlen, &maxcount); + read->rd_vlen, maxcount);if (nfserr)return nfserr; -xdr_truncate_encode (xdr, starting_len +8 + ((maxcount+3 )&~3 )); +xdr_truncate_encode (xdr, starting_len + ((*maxcount+3 )&~3 )); - eof = (read->rd_offset + maxcount >= - read->rd_fhp->fh_dentry->d_inode->i_size); - - tmp =htonl (eof); -write_bytes_to_xdr_buf (xdr->buf, starting_len , &tmp,4 ); - tmp =htonl (maxcount); -write_bytes_to_xdr_buf (xdr->buf, starting_len +4 , &tmp,4 ); - - pad = (maxcount&3 ) ?4 - (maxcount&3 ) :0 ; -write_bytes_to_xdr_buf (xdr->buf, starting_len +8 + maxcount, - &zzz, pad); + pad = (*maxcount&3 ) ?4 - (*maxcount&3 ) :0 ; +write_bytes_to_xdr_buf (xdr->buf, starting_len + *maxcount, &zzz, pad);return 0 ; }
@@ -3414,6 +3395,7 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr,struct raparms *ra;__be32 *p;__be32 err; +u32 eof;if (nfserr)return nfserr;
@@ -3441,9 +3423,14 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr, }if (file->f_op->splice_read &&test_bit (RQ_SPLICE_OK , &resp->rqstp->rq_flags)) - err =nfsd4_encode_splice_read (resp, read, file, maxcount); + err =nfsd4_encode_splice_read (resp, read, file, &maxcount);else - err =nfsd4_encode_readv (resp, read, file, maxcount); + err =nfsd4_encode_readv (resp, read, file, &maxcount); + + eof = (read->rd_offset + maxcount >= + read->rd_fhp->fh_dentry->d_inode->i_size); + *p++ =cpu_to_be32 (eof); + *p++ =cpu_to_be32 (maxcount);if (!read->rd_filp)nfsd_put_tmp_read_open (file, ra);
Signature · 1 line
2.3.3
PATCH v3 2/3 NFSD: Add basic READ_PLUS support
In reply to Anna SchumakerThis patch adds READ_PLUS support for both NFS4_CONTENT_DATA and NFS4_CONTENT_HOLE segments. I keep things simple for now by only returning one segment at a time to clients issuing the READ_PLUS call.
--- fs/nfsd/nfs4proc.c | 16 ++++++++ fs/nfsd/nfs4xdr.c | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +16 -0
@@ -1858,6 +1858,16 @@ static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)return (op_encode_hdr_size +2 +XDR_QUADLEN (rlen)) *sizeof (__be32); } +static inline u32 nfsd4_read_plus_rsize (struct svc_rqst *rqstp,struct nfsd4_op *op) +{ +u32 maxcount =svc_max_payload (rqstp); +u32 rlen =min (op->u.read.rd_length, maxcount); +/* enough extra xdr space for encoding either a hole or data segment. */ +u32 xdr =5 ; + +return (op_encode_hdr_size +2 + xdr +XDR_QUADLEN (rlen)) *sizeof (__be32); +} +static inline u32 nfsd4_readdir_rsize (struct svc_rqst *rqstp,struct nfsd4_op *op) {u32 maxcount =0 , rlen =0 ;
@@ -2290,6 +2300,12 @@ static struct nfsd4_operation nfsd4_ops[] = { .op_name = "OP_DEALLOCATE", .op_rsize_bop = (nfsd4op_rsize)nfsd4_write_rsize, }, + [OP_READ_PLUS] = { + .op_func = (nfsd4op_func)nfsd4_read, + .op_name = "OP_READ_PLUS", + .op_rsize_bop = (nfsd4op_rsize)nfsd4_read_plus_rsize, + .op_get_currentstateid = (stateid_getter)nfsd4_get_readstateid, + }, [OP_SEEK] = { .op_func = (nfsd4op_func)nfsd4_seek, .op_name = "OP_SEEK",
fs/nfsd/nfs4xdr.c +108 -2
128 lines not on this page · raw message
Signature · 1 line
2.3.3
PATCH v3 3/3 NFSD: Add support for encoding multiple segments
In reply to Anna SchumakerThis patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
--- fs/nfsd/nfs4proc.c | 4 ++-- fs/nfsd/nfs4xdr.c | 35 ++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 13 deletions(-)
fs/nfsd/nfs4proc.c +2 -2
@@ -1862,8 +1862,8 @@ static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op {u32 maxcount =svc_max_payload (rqstp);u32 rlen =min (op->u.read.rd_length, maxcount); -/* enough extra xdr space for encoding either a hole or data segment. */ -u32 xdr =5 ; +/* Extra xdr padding for encoding multiple segments. */ +u32 xdr =20 ;return (op_encode_hdr_size +2 + xdr +XDR_QUADLEN (rlen)) *sizeof (__be32); }
fs/nfsd/nfs4xdr.c +24 -11
@@ -4117,7 +4117,7 @@ nfsd4_encode_layoutreturn(struct nfsd4_compoundres *resp, __be32 nfserr,static __be32 nfsd4_encode_read_plus_data (struct nfsd4_compoundres *resp,struct nfsd4_read *read, -struct file *file) +struct file *file,loff_t hole_pos) {__be32 *p, err;unsigned long maxcount;
@@ -4128,20 +4128,26 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, struct nfsd4_read *rreturn nfserr_resource;xdr_commit_encode (xdr); +if (hole_pos <= read->rd_offset) + hole_pos =i_size_read (file_inode (file)); + maxcount =svc_max_payload (resp->rqstp); maxcount = min_t(unsigned long, maxcount, (xdr->buf->buflen - xdr->buf->len)); maxcount = min_t(unsigned long, maxcount, read->rd_length); + maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);if (file->f_op->splice_read &&test_bit (RQ_SPLICE_OK , &resp->rqstp->rq_flags)) err =nfsd4_encode_splice_read (resp, read, file, &maxcount);else err =nfsd4_encode_readv (resp, read, file, &maxcount); +clear_bit (RQ_SPLICE_OK , &resp->rqstp->rq_flags); *p++ =cpu_to_be32 (NFS4_CONTENT_DATA ); p =xdr_encode_hyper (p, read->rd_offset); *p++ =cpu_to_be32 (maxcount); read->rd_offset += maxcount; + read->rd_length -= maxcount;return err; }
@@ -4156,7 +4162,7 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp, struct nfsd4_read *rif (data_pos == -ENXIO ) data_pos =i_size_read (file_inode (file));if (data_pos <= read->rd_offset) -return nfsd4_encode_read_plus_data (resp, read, file); +return nfsd4_encode_read_plus_data (resp, read, file,0 ); maxcount = data_pos - read->rd_offset; p =xdr_reserve_space (&resp->xdr,4 +8 +8 );
@@ -4165,6 +4171,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp, struct nfsd4_read *r p =xdr_encode_hyper (p, maxcount); read->rd_offset += maxcount; +if (maxcount > read->rd_length) + read->rd_length =0 ; +else + read->rd_length -= maxcount;return nfs_ok; }
@@ -4197,17 +4207,20 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr, goto err_truncate; } - hole_pos =vfs_llseek (file, read->rd_offset,SEEK_HOLE ); -if (hole_pos == -ENXIO ) - goto out_encode; +do { + hole_pos =vfs_llseek (file, read->rd_offset,SEEK_HOLE ); +if (hole_pos == -ENXIO ) +break ; -if (hole_pos == read->rd_offset) - err =nfsd4_encode_read_plus_hole (resp, read, file); -else - err =nfsd4_encode_read_plus_data (resp, read, file); - segments++; +if (hole_pos == read->rd_offset) + err =nfsd4_encode_read_plus_hole (resp, read, file); +else + err =nfsd4_encode_read_plus_data (resp, read, file, hole_pos); +if (err) +break ; + segments++; + }while (read->rd_length >0 ); -out_encode: eof = (read->rd_offset >=i_size_read (file_inode (file))); *p++ =cpu_to_be32 (eof); *p++ =cpu_to_be32 (segments);
Signature · 1 line
2.3.3
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
--b.
Quoted 119 lines
Signed-off-by Anna Schumaker <[email protected]>--- fs/nfsd/nfs4proc.c | 4 ++-- fs/nfsd/nfs4xdr.c | 35 ++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 13 deletions(-)fs/nfsd/nfs4proc.c +2
-2@@ -1862,8 +1862,8 @@ static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op {u32 maxcount =svc_max_payload (rqstp);u32 rlen =min (op->u.read.rd_length, maxcount); -/* enough extra xdr space for encoding either a hole or data segment. */ -u32 xdr =5 ; +/* Extra xdr padding for encoding multiple segments. */ +u32 xdr =20 ;return (op_encode_hdr_size +2 + xdr +XDR_QUADLEN (rlen)) *sizeof (__be32); }fs/nfsd/nfs4xdr.c +24
-11@@ -4117,7 +4117,7 @@ nfsd4_encode_layoutreturn(struct nfsd4_compoundres *resp, __be32 nfserr,static __be32 nfsd4_encode_read_plus_data (struct nfsd4_compoundres *resp,struct nfsd4_read *read, -struct file *file) +struct file *file,loff_t hole_pos) {__be32 *p, err;unsigned long maxcount;@@ -4128,20 +4128,26 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, struct nfsd4_read *rreturn nfserr_resource;xdr_commit_encode (xdr); +if (hole_pos <= read->rd_offset) + hole_pos =i_size_read (file_inode (file)); + maxcount =svc_max_payload (resp->rqstp); maxcount = min_t(unsigned long, maxcount, (xdr->buf->buflen - xdr->buf->len)); maxcount = min_t(unsigned long, maxcount, read->rd_length); + maxcount = min_t(unsigned long, maxcount, hole_pos - read->rd_offset);if (file->f_op->splice_read &&test_bit (RQ_SPLICE_OK , &resp->rqstp->rq_flags)) err =nfsd4_encode_splice_read (resp, read, file, &maxcount);else err =nfsd4_encode_readv (resp, read, file, &maxcount); +clear_bit (RQ_SPLICE_OK , &resp->rqstp->rq_flags); *p++ =cpu_to_be32 (NFS4_CONTENT_DATA ); p =xdr_encode_hyper (p, read->rd_offset); *p++ =cpu_to_be32 (maxcount); read->rd_offset += maxcount; + read->rd_length -= maxcount;return err; }@@ -4156,7 +4162,7 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp, struct nfsd4_read *rif (data_pos == -ENXIO ) data_pos =i_size_read (file_inode (file));if (data_pos <= read->rd_offset) -return nfsd4_encode_read_plus_data (resp, read, file); +return nfsd4_encode_read_plus_data (resp, read, file,0 ); maxcount = data_pos - read->rd_offset; p =xdr_reserve_space (&resp->xdr,4 +8 +8 );@@ -4165,6 +4171,10 @@ nfsd4_encode_read_plus_hole(struct nfsd4_compoundres *resp, struct nfsd4_read *r p =xdr_encode_hyper (p, maxcount); read->rd_offset += maxcount; +if (maxcount > read->rd_length) + read->rd_length =0 ; +else + read->rd_length -= maxcount;return nfs_ok; }@@ -4197,17 +4207,20 @@ nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr, goto err_truncate; } - hole_pos =vfs_llseek (file, read->rd_offset,SEEK_HOLE ); -if (hole_pos == -ENXIO ) - goto out_encode; +do { + hole_pos =vfs_llseek (file, read->rd_offset,SEEK_HOLE ); +if (hole_pos == -ENXIO ) +break ; -if (hole_pos == read->rd_offset) - err =nfsd4_encode_read_plus_hole (resp, read, file); -else - err =nfsd4_encode_read_plus_data (resp, read, file); - segments++; +if (hole_pos == read->rd_offset) + err =nfsd4_encode_read_plus_hole (resp, read, file); +else + err =nfsd4_encode_read_plus_data (resp, read, file, hole_pos); +if (err) +break ; + segments++; + }while (read->rd_length >0 ); -out_encode: eof = (read->rd_offset >=i_size_read (file_inode (file))); *p++ =cpu_to_be32 (eof); *p++ =cpu_to_be32 (segments);Signature · 6 lines
2.3.3 -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.
Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Also, looks like
xvs_io -c "fiemap -v" <file>
will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)
So, basically, read_plus would save transferring most of the data even when only handling 1MB holes.
But some looked like this:
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)
So the READ_PLUS that caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.
And there were lots of examples in between those two extremes.
(But, check my math, I haven't tested this carefully.)
--b.
#define _GNU_SOURCE
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <err.h>
long round_up(long n, long b)
{
return ((n + b - 1)/b) * b;
}long round_down(long n, long b)
{
return (n/b) * b;
}long hbytes = 0;
long rplusbytes = 0;
do_stats(off_t hole_start, off_t hole_end)
{
off_t hole_start_up, hole_end_down;
hole_start_up = round_up(hole_start, 1024*1024);
hole_end_down = round_down(hole_end, 1024*1024);
hbytes += hole_end - hole_start;
if (hole_start_up < hole_end_down)
rplusbytes += hole_end_down - hole_start_up;
}int main(int argc, char *argv[])
{
off_t hole_start, hole_end;
int fd;
char *name;
/* Map out holes with SEEK_HOLE, SEEK_DATA */
/* Useful statistics:
* - what percentage of file is in holes?
* - what percentage of file would be skipped if we read it
* sequentially in 1MB chunks?
*/
if (argc != 2)
errx(1, "usage: %s <filename>\n", argv[0]);
name = argv[1];
fd = open(name, O_RDONLY);
if (fd == -1)
err(1, "open");
hole_end = 0;
while (1) {
hole_start = lseek(fd, hole_end, SEEK_HOLE);
if (hole_start == -1)
err(1, "lseek");
hole_end = lseek(fd, hole_start, SEEK_DATA);
if (hole_end == -1) {
if (errno == ENXIO)
break;
err(1, "lseek");
}
do_stats(hole_start, hole_end);
}
hole_end = lseek(fd, 0, SEEK_END);
do_stats(hole_start, hole_end);
printf("total hole bytes: %ld (%.0f%)\n", hbytes,
100 * (float)hbytes/hole_end);
printf("in aligned 1MB chunks: %ld (%.0f%)\n", rplusbytes,
100 * (float)rplusbytes/hole_end);
}Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 43 lines · J. Bruce Fields
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of the data even when only handling 1MB holes.But some looked like this:
Quoted 8 lines
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.And there were lots of examples in between those two extremes.
I tested with three different 512 MB files: 100% data, 100% hole, and alternating every megabyte. The results were surprising:
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448
Quoted 1 line
From what I can tell, the 100% hole case takes so long because of the
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took this out to trick the function into thinking that the entire file was already a hole, and runtime dropped to the levels of v4.1 and v4.2. I wonder if this is filesystem dependent? My server is exporting ext4.
Anna
Quoted 80 lines
(But, check my math, I haven't tested this carefully.)
--b.
#define
_GNU_SOURCE
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <err.h>long
round_up(long n, long b){ return ((n + b - 1)/b) * b; }long
round_down(long n, long b){ return (n/b) * b; }long hbytes = 0;
long rplusbytes = 0;
do_stats(off_t hole_start, off_t hole_end){ off_t hole_start_up, hole_end_down; hole_start_up = round_up(hole_start, 1024*1024); hole_end_down = round_down(hole_end, 1024*1024); hbytes += hole_end - hole_start; if (hole_start_up < hole_end_down) rplusbytes += hole_end_down - hole_start_up; }int
main(int argc, char *argv[]){ off_t hole_start, hole_end; int fd; char *name; /* Map out holes with SEEK_HOLE, SEEK_DATA */ /* Useful statistics: * - what percentage of file is in holes? * - what percentage of file would be skipped if we read it * sequentially in 1MB chunks? */ if (argc != 2) errx(1, "usage: %s <filename>\n", argv[0]); name = argv[1]; fd = open(name, O_RDONLY); if (fd == -1) err(1, "open"); hole_end = 0; while (1) { hole_start = lseek(fd, hole_end, SEEK_HOLE); if (hole_start == -1) err(1, "lseek"); hole_end = lseek(fd, hole_start, SEEK_DATA); if (hole_end == -1) { if (errno == ENXIO) break; err(1, "lseek"); } do_stats(hole_start, hole_end); } hole_end = lseek(fd, 0, SEEK_END); do_stats(hole_start, hole_end); printf("total hole bytes: %ld (%.0f%)\n", hbytes, 100 * (float)hbytes/hole_end); printf("in aligned 1MB chunks: %ld (%.0f%)\n", rplusbytes, 100 * (float)rplusbytes/hole_end); }
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 53 lines · Anna Schumaker
Quoted 43 lines · J. Bruce Fields
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of the data even when only handling 1MB holes.But some looked like this:
Quoted 8 lines
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.And there were lots of examples in between those two extremes.
Quoted 12 lines
I tested with three different 512 MB files: 100% data, 100% hole, and alternating every megabyte. The results were surprising:
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 4 lines
From what I can tell, the 100% hole case takes so long because of the
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took this out to trick the function into thinking that the entire file was already a hole, and runtime dropped to the levels of v4.1 and v4.2.
Wait, that 15s is due to just one SEEK_DATA?
Quoted 2 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting ext4.
Sounds like just a bug. I've been doing lots of lseek(.,.,SEEK_DATA) on both ext4 and xfs without seeing anything that weird.
I believe it does return -ENXIO in the case SEEK_DATA is called at an offset beyond which there's no more data. At least that's what I saw in userspace. So maybe your code just isn't handling that case correctly?
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 62 lines · J. Bruce Fields
Quoted 61 lines · Anna Schumaker
Quoted 43 lines · J. Bruce Fields
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of the data even when only handling 1MB holes.But some looked like this:
Quoted 8 lines
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.And there were lots of examples in between those two extremes.
I tested with three different 512 MB files: 100% data, 100% hole, and alternating every megabyte. The results were surprising:
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448
Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
From what I can tell, the 100% hole case takes so long because of the
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took this out to trick the function into thinking that the entire file was already a hole, and runtime dropped to the levels of v4.1 and v4.2.Wait, that 15s is due to just one
SEEK_DATA?
The server is returning a larger hole than the client can read at once, so there are several SEEK_DATA calls made to verify that there are no data segments before the end of the file.
Quoted 6 lines
Quoted 3 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting ext4.
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)on both ext4 and xfs without seeing anything that weird.
It looks like something weird on ext4. I switched my exported filesystem to xfs:
| v4.1 | v4.2 ------+--------+------- data | 0.764s | 1.343s hole | 0.572s | 0.205s mixed | 0.634s | 0.472s
I bumped up the test to 1G files:
| v4.1 | v4.2 ------+--------+------- data | 1.578s | 1.743s hole | 1.241s | 0.443s mixed | 1.884s | 0.913s
Let me know if I should test anything larger!
Anna
Quoted 7 lines
I believe it does return
-ENXIOin the caseSEEK_DATAis called at an offset beyond which there's no more data. At least that's what I saw in userspace. So maybe your code just isn't handling that case correctly?--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 73 lines · Anna Schumaker
Quoted 62 lines · J. Bruce Fields
Quoted 61 lines · Anna Schumaker
Quoted 43 lines · J. Bruce Fields
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of the data even when only handling 1MB holes.But some looked like this:
Quoted 8 lines
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.And there were lots of examples in between those two extremes.
I tested with three different 512 MB files: 100% data, 100% hole, and alternating every megabyte. The results were surprising:
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
From what I can tell, the 100% hole case takes so long because of the
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took this out to trick the function into thinking that the entire file was already a hole, and runtime dropped to the levels of v4.1 and v4.2.Wait, that 15s is due to just one
SEEK_DATA?The server is returning a larger hole than the client can read at once, so there are several
SEEK_DATAcalls made to verify that there are no data segments before the end of the file.Quoted 1 line
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting ext4.
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)on both ext4 and xfs without seeing anything that weird.It looks like something weird on ext4. I switched my exported filesystem to xfs:
Huh. Maybe we should report a bug....
Quoted 4 lines
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343s
That's too bad. Non-sparse files are surely still a common case and we'd like to not see a slowdown there.... I wonder if we can figure out where it's coming from?
Quoted 6 lines
hole | 0.572s | 0.205s mixed | 0.634s | 0.472sI bumped up the test to 1G files:
Quoted 7 lines
| v4.1 | v4.2 ------+--------+------- data | 1.578s | 1.743s hole | 1.241s | 0.443s mixed | 1.884s | 0.913sLet me know if I should test anything larger!
The other thing I'd be interested in would be a "mixed" case that alternates every 4k. That will test the worst case where we we do a 1MB read and get back only a 4k hole. Aligned 1MB holes are somewhat of a best case.
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 85 lines · J. Bruce Fields
Quoted 73 lines · Anna Schumaker
Quoted 62 lines · J. Bruce Fields
Quoted 61 lines · Anna Schumaker
Quoted 43 lines · J. Bruce Fields
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of the data even when only handling 1MB holes.But some looked like this:
Quoted 8 lines
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.And there were lots of examples in between those two extremes.
I tested with three different 512 MB files: 100% data, 100% hole, and alternating every megabyte. The results were surprising:
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
From what I can tell, the 100% hole case takes so long because of the
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took this out to trick the function into thinking that the entire file was already a hole, and runtime dropped to the levels of v4.1 and v4.2.Wait, that 15s is due to just one
SEEK_DATA?The server is returning a larger hole than the client can read at once, so there are several
SEEK_DATAcalls made to verify that there are no data segments before the end of the file.Quoted 1 line
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting ext4.
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)on both ext4 and xfs without seeing anything that weird.It looks like something weird on ext4. I switched my exported filesystem to xfs:
Huh. Maybe we should report a bug....
Quoted 1 line
Quoted 7 lines
Quoted 3 lines
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343sThat's too bad. Non-sparse files are surely still a common case and we'd like to not see a slowdown there.... I wonder if we can figure out where it's coming from?
That's a good question, especially since the 1G file didn't double this time. Maybe a VM quirk?
Quoted 13 lines
Quoted 13 lines
hole | 0.572s | 0.205s mixed | 0.634s | 0.472sI bumped up the test to 1G files:
| v4.1 | v4.2 ------+--------+------- data | 1.578s | 1.743s hole | 1.241s | 0.443s mixed | 1.884s | 0.913s
Quoted 6 lines
Quoted 1 line
Let me know if I should test anything larger!
The other thing I'd be interested in would be a "mixed" case that alternates every 4k. That will test the worst case where we we do a 1MB read and get back only a 4k hole. Aligned 1MB holes are somewhat of a best case.
I probably won't get a chance to test this until I'm back from my vacation, but I'll keep the suggestion in mind!
Anna
Quoted 3 lines
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 90 lines · Anna Schumaker
Quoted 89 lines · J. Bruce Fields
Quoted 73 lines · Anna Schumaker
Quoted 62 lines · J. Bruce Fields
Quoted 61 lines · Anna Schumaker
Quoted 43 lines · J. Bruce Fields
Quoted 20 lines · J. Bruce Fields
Quoted 13 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
This patch implements sending an array of segments back to the client. Clients should be prepared to handle multiple segment reads to make this useful. We try to splice the first data segment into the XDR result, and remaining segments are encoded directly.
I'm still interested in what would happen if we started with an implementation like:
- if the entire requested range falls within a hole, return that single hole.
- otherwise, just treat the thing as one big data segment.Quoted 5 lines
That would provide a benefit in the case there are large-ish holes with minimal impact otherwise.
(Though patches for full support are still useful even if only for client-testing purposes.)
Quoted 8 lines
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks, esandeen.) Running that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only >=rwsize holes might still be useful.
Just for fun.... I wrote the following test program and ran it on my collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13 f21-1.qcow2 total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of the data even when only handling 1MB holes.But some looked like this:
Quoted 8 lines
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014 rhel6-1-1.img total hole bytes: 8077516800 (94%) in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, the one that only caught 1MB holes wouldn't help at all.And there were lots of examples in between those two extremes.
I tested with three different 512 MB files: 100% data, 100% hole, and alternating every megabyte. The results were surprising:
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
From what I can tell, the 100% hole case takes so long because of the
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took this out to trick the function into thinking that the entire file was already a hole, and runtime dropped to the levels of v4.1 and v4.2.Wait, that 15s is due to just one
SEEK_DATA?The server is returning a larger hole than the client can read at once, so there are several
SEEK_DATAcalls made to verify that there are no data segments before the end of the file.Quoted 1 line
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting ext4.
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)on both ext4 and xfs without seeing anything that weird.It looks like something weird on ext4. I switched my exported filesystem to xfs:
Huh. Maybe we should report a bug....
Quoted 4 lines
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343s
Quoted 5 lines
Quoted 3 lines
That's too bad. Non-sparse files are surely still a common case and we'd like to not see a slowdown there.... I wonder if we can figure out where it's coming from?
That's a good question, especially since the 1G file didn't double this time. Maybe a VM quirk?
We definitely need to figure it out, I think. If we can't make READ_PLUS perform as well as READ (or very close to it) in the non-sparse case then I don't think we'll want it, and as Trond suggested we may want to consider something more fiemap-like instead.
I don't know, maybe the client could try to be clever and only use READ_PLUS if the space_used/size ratio is lower than some threshhold, but it could get a little complicated to tune.
It's annoying that asking "does this range contain zeroes" is actually taking longer than just reading the whole range....
--b.
A message that is not in the archive (sent off-list or not yet received): OFB111A6D8.016B8BD5-ON88257E0D.001D174D-88257E0D.005268D6@us.ibm.com
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 133 lines · Marc Eshel
[email protected] wrote on 03/18/2015 02:11:44 PM:
Quoted 5 lines
From: "J. Bruce Fields" <[email protected]>
To: Anna Schumaker <[email protected]>
Cc: [email protected]
Date: 03/18/2015 02:14 PM
Subject: Re: [PATCH v3 3/3] NFSD: Add support for encoding multiplesegments
Quoted 10 lines
Sent by: [email protected]
Quoted 7 lines · Anna Schumaker
Quoted 6 lines · J. Bruce Fields
Quoted 5 lines · Anna Schumaker
Quoted 4 lines · J. Bruce Fields
Quoted 3 lines · Anna Schumaker
Quoted 2 lines · J. Bruce Fields
Quoted 1 line · J. Bruce Fields
On Tue, Mar 17, 2015 at 03:56:33PM -0400, J. Bruce Fields
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Mar 16, 2015 at 05:18:08PM -0400, Anna Schumaker
Quoted 9 lines · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
This patch implements sending an array of segments back
to the client.
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Clients should be prepared to handle multiple segment
reads to make this
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
useful. We try to splice the first data segment into the
XDR result,
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
and remaining segments are encoded directly.
I'm still interested in what would happen if we started with
an
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
implementation like:
- if the entire requested range falls within a hole, return
that
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
single hole.
- otherwise, just treat the thing as one big data segment.That would provide a benefit in the case there are large-ish
holes
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
with minimal impact otherwise.
(Though patches for full support are still useful even if only
for
Quoted 14 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
client-testing purposes.)
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks,
esandeen.) Running
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
that on a few of my test vm images shows a fair number of large (hundreds of megs) files, which suggests identifying only
=rwsize holes
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
might still be useful.
Just for fun.... I wrote the following test program and ran it
on my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13f21-1.qcow2
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of thedataeven
Quoted 10 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
when only handling 1MB holes.
But some looked like this:
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014rhel6-1-1.img
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total hole bytes: 8077516800 (94%)
in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot, theone that
Quoted 14 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
only caught 1MB holes wouldn't help at all.
And there were lots of examples in between those two extremes.
I tested with three different 512 MB files: 100% data, 100%
hole, and alternating every megabyte. The results were surprising:
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 1 line
From what I can tell, the 100% hole case takes so long because
of the
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took thisout to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
trick the function into thinking that the entire file was
already a
Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
hole, and runtime dropped to the levels of v4.1 and v4.2.
Wait, that 15s is due to just one
SEEK_DATA?The server is returning a larger hole than the client can read
at once, so there are several
SEEK_DATAcalls made to verify that there are no data segments before the end of the file.Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting ext4.
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)onQuoted 14 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
both ext4 and xfs without seeing anything that weird.
It looks like something weird on ext4. I switched my exported
filesystem to xfs:
Quoted 10 lines
Quoted 10 lines
Huh. Maybe we should report a bug....
Quoted 4 lines
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343sThat's too bad. Non-sparse files are surely still a common case and we'd like to not see a slowdown there.... I wonder if we can figure
out
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
where it's coming from?
That's a good question, especially since the 1G file didn't double
this time. Maybe a VM quirk?
Quoted 8 lines
Quoted 4 lines
We definitely need to figure it out, I think. If we can't make
READ_PLUSperform as well as READ (or very close to it) in the non-sparse case then I don't think we'll want it, and as Trond suggested we may want to consider something more fiemap-like instead.Testing Anna's NFS client with the Ganesha NFS server and GPFS file system shows the same numbers for READ with v4.1 and
READ_PUSEwith v4.2 of a data file. Using sparse filesREAD_PLUSis 5 times faster than READ.
Thanks! Is it possible to report the exact numbers?
Is Ganesha also implementing READ_PLUS with SEEK_HOLE/SEEK_DATA? If so then maybe the difference is the filesystem. Might be interesting to run the same sort of test with ganesha exporting xfs and/or knfsd exporting GPFS.
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
[email protected] wrote on 03/19/2015 08:36:27 AM:
Quoted 6 lines
From: "J. Bruce Fields" <[email protected]>
To: Marc Eshel/Almaden/IBM@IBMUS
Cc: Anna Schumaker <[email protected]>, linux- [email protected], [email protected]
Date: 03/19/2015 08:36 AM
Subject: Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple
segments
Quoted 18 lines
Sent by: [email protected]
Quoted 15 lines · Marc Eshel
[email protected] wrote on 03/18/2015 02:11:44 PM:
Quoted 5 lines
From: "J. Bruce Fields" <[email protected]>
To: Anna Schumaker <[email protected]>
Cc: [email protected]
Date: 03/18/2015 02:14 PM
Subject: Re: [PATCH v3 3/3] NFSD: Add support for encoding multiplesegments
Quoted 7 lines
Sent by: [email protected]
Quoted 4 lines · Anna Schumaker
Quoted 3 lines · J. Bruce Fields
Quoted 2 lines · Anna Schumaker
Quoted 1 line · J. Bruce Fields
On Wed, Mar 18, 2015 at 02:16:29PM -0400, Anna Schumaker
Quoted 2 lines · wrote:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines · J. Bruce Fields
Quoted 1 line
On Tue, Mar 17, 2015 at 04:07:38PM -0400, J. Bruce Fields
Quoted 5 lines · wrote:
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Tue, Mar 17, 2015 at 03:56:33PM -0400, J. Bruce Fields
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Mar 16, 2015 at 05:18:08PM -0400, Anna Schumaker
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
This patch implements sending an array of segments back
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
to the client.
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Clients should be prepared to handle multiple segment
reads to make this
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
useful. We try to splice the first data segment into the
XDR result,
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
and remaining segments are encoded directly.
I'm still interested in what would happen if we started
with
Quoted 4 lines
Quoted 4 lines
an
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
implementation like:
- if the entire requested range falls within a hole,
return
Quoted 3 lines
Quoted 3 lines
that
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
single hole.
- otherwise, just treat the thing as one big data
segment.
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
That would provide a benefit in the case there are
large-ish
Quoted 4 lines
Quoted 4 lines
holes
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
with minimal impact otherwise.
(Though patches for full support are still useful even if
only
Quoted 10 lines
Quoted 10 lines
for
Quoted 9 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
client-testing purposes.)
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks,
esandeen.) Running
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
that on a few of my test vm images shows a fair number of
large
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
(hundreds of megs) files, which suggests identifying only
=rwsize holes
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
might still be useful.
Just for fun.... I wrote the following test program and ran
it
Quoted 10 lines
Quoted 10 lines
on my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13f21-1.qcow2
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most of
thedata
Quoted 11 lines
Quoted 11 lines
even
Quoted 10 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
when only handling 1MB holes.
But some looked like this:
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014rhel6-1-1.img
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total hole bytes: 8077516800 (94%)
in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot,
the
Quoted 4 lines
Quoted 4 lines
one that
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
only caught 1MB holes wouldn't help at all.
And there were lots of examples in between those two
extremes.
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
I tested with three different 512 MB files: 100% data, 100%
hole, and alternating every megabyte. The results were surprising:
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 1 line
From what I can tell, the 100% hole case takes so long
because
Quoted 2 lines
Quoted 2 lines
of the
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I took
this
Quoted 7 lines
Quoted 7 lines
out to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
trick the function into thinking that the entire file was
already a
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
hole, and runtime dropped to the levels of v4.1 and v4.2.
Wait, that 15s is due to just one
SEEK_DATA?
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
The server is returning a larger hole than the client can read
at once, so there are several
SEEK_DATAcalls made to verify that there are no data segments before the end of the file.Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting
ext4.
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)onQuoted 5 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
both ext4 and xfs without seeing anything that weird.
It looks like something weird on ext4. I switched my exported
filesystem to xfs:
Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Huh. Maybe we should report a bug....
Quoted 4 lines
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343sThat's too bad. Non-sparse files are surely still a common case
and
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
we'd like to not see a slowdown there.... I wonder if we can
figure
Quoted 9 lines
Quoted 9 lines
out
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
where it's coming from?
That's a good question, especially since the 1G file didn't double
this time. Maybe a VM quirk?
We definitely need to figure it out, I think. If we can't make
READ_PLUSperform as well as READ (or very close to it) in the non-sparse case then I don't think we'll want it, and as Trond
suggested
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
we may want to consider something more fiemap-like instead.
Testing Anna's NFS client with the Ganesha NFS server and GPFS file
system
Quoted 5 lines
Quoted 3 lines
shows the same numbers for READ with v4.1 and
READ_PLUSwith v4.2 of adata file. Using sparse files
READ_PLUSis 5 times faster than READ.Thanks! Is it possible to report the exact numbers?
This is a copy of a 100M file.
[root@fin16 ~]# umount /mnt
[root@fin16 ~]# mount -t nfs4 -o minorversion=1 9.1.74.120:/gpfsA /mnt [root@fin16 ~]# time cp /mnt/100M /dev/null
real 0m1.597s user 0m0.000s sys 0m0.062s
[root@fin16 ~]# umount /mnt
[root@fin16 ~]# mount -t nfs4 -o minorversion=2 9.1.74.120:/gpfsA /mnt [root@fin16 ~]# time cp /mnt/100M /dev/null
real 0m1.595s user 0m0.002s sys 0m0.057s
Quoted 5 lines
Is Ganesha also implementing
READ_PLUSwith SEEK_HOLE/SEEK_DATA? If so then maybe the difference is the filesystem. Might be interesting to run the same sort of test with ganesha exporting xfs and/or knfsd exporting GPFS.
GPFS did not implement it using SEEK it just calls the fs read and if there is no data the fs returns ENODATA return code. It is not yet implemented on other FSLAs
Quoted 7 lines
--b.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Maybe this is a question for xfs developers.
So, we have a new READ_PLUS call that's basically just a version of READ optimized for sparse files:
http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion2-33#section-15.10
It allows an NFS server to return either file data (like a normal READ call) or, at the server's discretion, records saying "this range of the data is all zeroes".
Anna tried implementing READ_PLUS for knfsd using vfs_llseek(.,.,SEEK_HOLE) followed by an ordinary read if that determines we're not at a hole.
(Very) preliminary results suggest that's slower than a plain READ for an xfs file with no holes. (And *much* slower in the ext4 case for some reason.)
Is that expected, and should we be doing this some other way instead?
--b.
Quoted 79 lines · Marc Eshel
[email protected] wrote on 03/19/2015 08:36:27 AM:
Quoted 6 lines
From: "J. Bruce Fields" <[email protected]>
To: Marc Eshel/Almaden/IBM@IBMUS
Cc: Anna Schumaker <[email protected]>, linux- [email protected], [email protected]
Date: 03/19/2015 08:36 AM
Subject: Re: [PATCH v3 3/3] NFSD: Add support for encoding multiplesegments
Quoted 13 lines
Sent by: [email protected]
Quoted 10 lines · Marc Eshel
[email protected] wrote on 03/18/2015 02:11:44 PM:
Quoted 9 lines
From: "J. Bruce Fields" <[email protected]>
Quoted 7 lines · Anna Schumaker
Quoted 6 lines · J. Bruce Fields
Quoted 3 lines · Anna Schumaker
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343sThat's too bad. Non-sparse files are surely still a common case
and
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
we'd like to not see a slowdown there.... I wonder if we can
figure
Quoted 9 lines
Quoted 9 lines
out
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
where it's coming from?
That's a good question, especially since the 1G file didn't double
this time. Maybe a VM quirk?
We definitely need to figure it out, I think. If we can't make
READ_PLUSperform as well as READ (or very close to it) in the non-sparse case then I don't think we'll want it, and as Trondsuggested
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
we may want to consider something more fiemap-like instead.
Testing Anna's NFS client with the Ganesha NFS server and GPFS file
system
Quoted 5 lines
Quoted 3 lines
shows the same numbers for READ with v4.1 and
READ_PLUSwith v4.2 of adata file. Using sparse files
READ_PLUSis 5 times faster than READ.Thanks! Is it possible to report the exact numbers?
This is a copy of a 100M file.
[root@fin16 ~]# umount /mnt
[root@fin16 ~]# mount -t nfs4 -o minorversion=1 9.1.74.120:/gpfsA /mnt [root@fin16 ~]# time cp/mnt/100M/dev/nullreal 0m1.597s user 0m0.000s sys 0m0.062s[root@fin16 ~]# umount /mnt
[root@fin16 ~]# mount -t nfs4 -o minorversion=2 9.1.74.120:/gpfsA /mnt [root@fin16 ~]# time cp/mnt/100M/dev/nullreal 0m1.595s user 0m0.002s sys 0m0.057sQuoted 5 lines
Is Ganesha also implementing
READ_PLUSwith SEEK_HOLE/SEEK_DATA? If so then maybe the difference is the filesystem. Might be interesting to run the same sort of test with ganesha exporting xfs and/or knfsd exporting GPFS.GPFS did not implement it using SEEK it just calls the fs read and if there is no data the fs returns ENODATA return code. It is not yet implemented on other FSLAs
Quoted 7 lines
--b.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 210 lines · Marc Eshel
[email protected] wrote on 03/19/2015 08:36:27 AM:
Quoted 6 lines
From: "J. Bruce Fields" <[email protected]>
To: Marc Eshel/Almaden/IBM@IBMUS
Cc: Anna Schumaker <[email protected]>, linux- [email protected], [email protected]
Date: 03/19/2015 08:36 AM
Subject: Re: [PATCH v3 3/3] NFSD: Add support for encoding multiplesegments
Quoted 18 lines
Sent by: [email protected]
Quoted 15 lines · Marc Eshel
[email protected] wrote on 03/18/2015 02:11:44 PM:
Quoted 5 lines
From: "J. Bruce Fields" <[email protected]>
To: Anna Schumaker <[email protected]>
Cc: [email protected]
Date: 03/18/2015 02:14 PM
Subject: Re: [PATCH v3 3/3] NFSD: Add support for encoding multiplesegments
Quoted 7 lines
Sent by: [email protected]
Quoted 4 lines · Anna Schumaker
Quoted 3 lines · J. Bruce Fields
Quoted 2 lines · Anna Schumaker
Quoted 1 line · J. Bruce Fields
On Wed, Mar 18, 2015 at 02:16:29PM -0400, Anna Schumaker
Quoted 2 lines · wrote:
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines · J. Bruce Fields
Quoted 1 line
On Tue, Mar 17, 2015 at 04:07:38PM -0400, J. Bruce Fields
Quoted 5 lines · wrote:
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Tue, Mar 17, 2015 at 03:56:33PM -0400, J. Bruce Fields
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
On Mon, Mar 16, 2015 at 05:18:08PM -0400, Anna Schumaker
Quoted 1 line · wrote:
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
This patch implements sending an array of segments back
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
to the client.
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Clients should be prepared to handle multiple segment
reads to make this
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
useful. We try to splice the first data segment into the
XDR result,
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
and remaining segments are encoded directly.
I'm still interested in what would happen if we started
with
Quoted 4 lines
Quoted 4 lines
an
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
implementation like:
- if the entire requested range falls within a hole,
return
Quoted 3 lines
Quoted 3 lines
that
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
single hole.
- otherwise, just treat the thing as one big datasegment.
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
That would provide a benefit in the case there are
large-ish
Quoted 4 lines
Quoted 4 lines
holes
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
with minimal impact otherwise.
(Though patches for full support are still useful even if
only
Quoted 10 lines
Quoted 10 lines
for
Quoted 9 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
client-testing purposes.)
Also, looks like
xvs_io-c "fiemap -v" <file>will give hole sizes for a given <file>. (Thanks,
esandeen.) Running
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
that on a few of my test vm images shows a fair number of
large
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
(hundreds of megs) files, which suggests identifying only
=rwsize holes
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
might still be useful.
Just for fun.... I wrote the following test program and ran
it
Quoted 10 lines
Quoted 10 lines
on my
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
collection of testing vm's. Some looked like this:
f21-1.qcow2 144784 -rw-------. 1 qemu qemu 8591507456 Mar 16 10:13f21-1.qcow2
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total hole bytes: 8443252736 (98%) in aligned 1MB chunks: 8428453888 (98%)So, basically,
read_pluswould save transferring most ofthedata
Quoted 11 lines
Quoted 11 lines
even
Quoted 10 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines
when only handling 1MB holes.
But some looked like this:
501524 -rw-------. 1 qemu qemu 8589934592 May 20 2014rhel6-1-1.img
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
total hole bytes: 8077516800 (94%)
in aligned 1MB chunks: 0 (0%)So the
READ_PLUSthat caught every hole might save a lot,the
Quoted 4 lines
Quoted 4 lines
one that
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
only caught 1MB holes wouldn't help at all.
And there were lots of examples in between those two
extremes.
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
I tested with three different 512 MB files: 100% data, 100%
hole, and alternating every megabyte. The results were surprising:
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
| v4.1 | v4.2 ----------------------- data | 0.685s | 0.714s hole | 0.485s | 15.547s mixed | 1.283s | 0.448Quoted 1 line
From what I can tell, the 100% hole case takes so long
because
Quoted 2 lines
Quoted 2 lines
of the
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
SEEK_DATAcall innfsd4_encode_read_plus_hole(). I tookthis
Quoted 7 lines
Quoted 7 lines
out to
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
trick the function into thinking that the entire file was
already a
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines
Quoted 1 line
Quoted 1 line
hole, and runtime dropped to the levels of v4.1 and v4.2.
Wait, that 15s is due to just one
SEEK_DATA?Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
The server is returning a larger hole than the client can read
at once, so there are several
SEEK_DATAcalls made to verify that there are no data segments before the end of the file.Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
I wonder
Quoted 1 line
if this is filesystem dependent? My server is exporting
ext4.
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Sounds like just a bug. I've been doing lots of
lseek(.,.,SEEK_DATA)onQuoted 5 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
both ext4 and xfs without seeing anything that weird.
It looks like something weird on ext4. I switched my exported
filesystem to xfs:
Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Huh. Maybe we should report a bug....
Quoted 4 lines
| v4.1 | v4.2------+--------+-------
data | 0.764s | 1.343sThat's too bad. Non-sparse files are surely still a common case
and
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
we'd like to not see a slowdown there.... I wonder if we can
figure
Quoted 9 lines
Quoted 9 lines
out
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
where it's coming from?
That's a good question, especially since the 1G file didn't double
this time. Maybe a VM quirk?
We definitely need to figure it out, I think. If we can't make
READ_PLUSperform as well as READ (or very close to it) in the non-sparse case then I don't think we'll want it, and as Trondsuggested
Quoted 3 lines
Quoted 3 lines
Quoted 1 line
we may want to consider something more fiemap-like instead.
Testing Anna's NFS client with the Ganesha NFS server and GPFS file
system
Quoted 5 lines
Quoted 3 lines
shows the same numbers for READ with v4.1 and
READ_PLUSwith v4.2 of adata file. Using sparse files
READ_PLUSis 5 times faster than READ.Thanks! Is it possible to report the exact numbers?
This is a copy of a 100M file.
[root@fin16 ~]# umount /mnt
[root@fin16 ~]# mount -t nfs4 -o minorversion=1 9.1.74.120:/gpfsA /mnt [root@fin16 ~]# time cp/mnt/100M/dev/nullreal 0m1.597s user 0m0.000s sys 0m0.062s[root@fin16 ~]# umount /mnt
[root@fin16 ~]# mount -t nfs4 -o minorversion=2 9.1.74.120:/gpfsA /mnt [root@fin16 ~]# time cp/mnt/100M/dev/nullreal 0m1.595s user 0m0.002s sys 0m0.057sQuoted 5 lines
Is Ganesha also implementing
READ_PLUSwith SEEK_HOLE/SEEK_DATA? If so then maybe the difference is the filesystem. Might be interesting to run the same sort of test with ganesha exporting xfs and/or knfsd exporting GPFS.GPFS did not implement it using SEEK it just calls the fs read and if there is no data the fs returns ENODATA return code. It is not yet implemented on other FSLAs
Quoted 7 lines
--b.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 11 lines · J. Bruce Fields
Maybe this is a question for xfs developers.
So, we have a new
READ_PLUScall that's basically just a version of READ optimized for sparse files:http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion2-33#section-15.10It allows an NFS server to return either file data (like a normal READ call) or, at the server's discretion, records saying "this range of the data is all zeroes".
Quoted 7 lines
Anna tried implementing
READ_PLUSfor knfsd usingvfs_llseek(.,.,SEEK_HOLE)followed by an ordinary read if that determines we're not at a hole.(Very) preliminary results suggest that's slower than a plain READ for an xfs file with no holes. (And *much* slower in the ext4 case for some reason.)
It should be a fairly cheap operastion, and does extent tree operations that are pretty similar to an (uncached) read. Do you have profiles?
Quoted 1 line
Is that expected, and should we be doing this some other way instead?
Are the read cached or uncached? If they are from pagecache just copying the zeroes is pretty much unbeatable compared to extent tree lookups, so we'd need a new page flag (difficult..) to see that a page is a hole (and then it would only work for the whole page), but for uncached reads an optimization would be to tell a read that it's an NFS READ_PLUS so that it could just read until it reach a hole, and then we'd need some way to communicate the hole size (or just fall back to SEEK_HOLE for that case).
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 20 lines · Christoph Hellwig
Quoted 11 lines · J. Bruce Fields
Maybe this is a question for xfs developers.
So, we have a new
READ_PLUScall that's basically just a version of READ optimized for sparse files:http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion2-33#section-15.10It allows an NFS server to return either file data (like a normal READ call) or, at the server's discretion, records saying "this range of the data is all zeroes".
Quoted 7 lines
Anna tried implementing
READ_PLUSfor knfsd usingvfs_llseek(.,.,SEEK_HOLE)followed by an ordinary read if that determines we're not at a hole.(Very) preliminary results suggest that's slower than a plain READ for an xfs file with no holes. (And *much* slower in the ext4 case for some reason.)
Quoted 6 lines
It should be a fairly cheap operastion, and does extent tree operations that are pretty similar to an (uncached) read. Do you have profiles?
Quoted 1 line
Is that expected, and should we be doing this some other way instead?
Are the read cached or uncached?
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
Quoted 8 lines
If they are from pagecache just copying the zeroes is pretty much unbeatable compared to extent tree lookups, so we'd need a new page flag (difficult..) to see that a page is a hole (and then it would only work for the whole page), but for uncached reads an optimization would be to tell a read that it's an NFS
READ_PLUSso that it could just read until it reach a hole, and then we'd need some way to communicate the hole size (or just fall back toSEEK_HOLEfor that case).
Ugh, OK. We'll do some more tests before coming back to ask about that....
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 24 lines · J. Bruce Fields
Quoted 23 lines · Christoph Hellwig
Quoted 11 lines · J. Bruce Fields
Maybe this is a question for xfs developers.
So, we have a new
READ_PLUScall that's basically just a version of READ optimized for sparse files:http://tools.ietf.org/html/draft-ietf-nfsv4-minorversion2-33#section-15.10It allows an NFS server to return either file data (like a normal READ call) or, at the server's discretion, records saying "this range of the data is all zeroes".
Quoted 7 lines
Anna tried implementing
READ_PLUSfor knfsd usingvfs_llseek(.,.,SEEK_HOLE)followed by an ordinary read if that determines we're not at a hole.(Very) preliminary results suggest that's slower than a plain READ for an xfs file with no holes. (And *much* slower in the ext4 case for some reason.)
It should be a fairly cheap operastion, and does extent tree operations that are pretty similar to an (uncached) read. Do you have profiles?
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Is that expected, and should we be doing this some other way instead?
Are the read cached or uncached?
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
I'm using whatever functions NFSD already uses for reading files, which I expect go through the VFS. Is there a flag that controls cache behavior?
Quoted 10 lines
Quoted 9 lines
If they are from pagecache just copying the zeroes is pretty much unbeatable compared to extent tree lookups, so we'd need a new page flag (difficult..) to see that a page is a hole (and then it would only work for the whole page), but for uncached reads an optimization would be to tell a read that it's an NFS
READ_PLUSso that it could just read until it reach a hole, and then we'd need some way to communicate the hole size (or just fall back toSEEK_HOLEfor that case).
Quoted 2 lines
Ugh, OK. We'll do some more tests before coming back to ask about that....
I only had time for the one run, so I'll do more trials and see if that one read is always so long. I'm still hoping it was something in the way my VM was scheduling its tasks!
Anna
Quoted 6 lines
--b.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 6 lines · Anna Schumaker
Quoted 2 lines
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
I'm using whatever functions NFSD already uses for reading files, which I expect go through the VFS. Is there a flag that controls cache behavior?
There's the O_DIRECT flag, but that's not what I mean. If you just wrote to it it's a cached read, if you did unmount the filesystem after writing, or did an echo to /proc/sys/vm/drop_caches you get uncached read behavior.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 4 lines · Christoph Hellwig
Quoted 3 lines · Anna Schumaker
Quoted 2 lines
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
Quoted 8 lines
Quoted 3 lines
I'm using whatever functions NFSD already uses for reading files, which I expect go through the VFS. Is there a flag that controls cache behavior?
There's the
O_DIRECTflag, but that's not what I mean. If you just wrote to it it's a cached read, if you did unmount the filesystem after writing, or did an echo to/proc/sys/vm/drop_cachesyou get uncached read behavior.
Oh, I'm doing uncached reads for my tests. I'm collecting updated numbers now!
Anna
Quoted 1 line
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
In reply to Christoph HellwigHere are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the READ_PLUS patches:
##########################
# # # Without READ_PLUS # # #
##########################
NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|
#######################
# # # With READ_PLUS # # #
#######################
NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 13 lines · Christoph Hellwig
Quoted 6 lines · Anna Schumaker
Quoted 2 lines
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
I'm using whatever functions NFSD already uses for reading files, which I expect go through the VFS. Is there a flag that controls cache behavior?
There's the
O_DIRECTflag, but that's not what I mean. If you just wrote to it it's a cached read, if you did unmount the filesystem after writing, or did an echo to/proc/sys/vm/drop_cachesyou get uncached read behavior.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial
Quoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Quoted 21 lines · Christoph Hellwig
Quoted 13 lines · Anna Schumaker
Quoted 6 lines
Quoted 2 lines
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
I'm using whatever functions NFSD already uses for reading files, which I expect go through the VFS. Is there a flag that controls cache behavior?
There's the
O_DIRECTflag, but that's not what I mean. If you just wrote to it it's a cached read, if you did unmount the filesystem after writing, or did an echo to/proc/sys/vm/drop_cachesyou get uncached read behavior.--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, PrimaryData [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 69 lines · Trond Myklebust
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
TrialQuoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Sure! Do you want any information besides runtime?
Anna
Quoted 25 lines
Quoted 22 lines · Christoph Hellwig
Quoted 13 lines · Anna Schumaker
Quoted 6 lines
Quoted 2 lines
I don't know, and don't have profiles. I'll either try to reproduce or wait till Anna's back from vacation.
I'm using whatever functions NFSD already uses for reading files, which I expect go through the VFS. Is there a flag that controls cache behavior?
There's the
O_DIRECTflag, but that's not what I mean. If you just wrote to it it's a cached read, if you did unmount the filesystem after writing, or did an echo to/proc/sys/vm/drop_cachesyou get uncached read behavior.--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
In reply to Trond MyklebustQuoted 69 lines · Trond Myklebust
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
TrialQuoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 > /proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.
Quoted 3 lines
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 71 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.
I agree that you have to use the 'drop_caches' interface on the server, but why not just use O_DIRECT on the clients?
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, PrimaryData [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 77 lines · Trond Myklebust
Quoted 75 lines · Anna Schumaker
Quoted 74 lines · J. Bruce Fields
Quoted 72 lines · Trond Myklebust
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
TrialQuoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?
I've been reading by using cat from my test shell script: time cat /nfs/file > /dev/null. I can write something to read files with O_DIRECT if that would be more useful!
Anna
Quoted 1 line
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 81 lines · Anna Schumaker
Quoted 80 lines · Trond Myklebust
Quoted 78 lines · Anna Schumaker
Quoted 76 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
TrialQuoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!
'dd' can do that for you if the appropriate incantations are performed.
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, PrimaryData [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
Got it. I'll sacrifice a goat to 'dd' and rerun the tests with O_DIRECT!
Quoted 1 line
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
In reply to Trond MyklebustI did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128K
Mixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128K
bs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # #
##########################
NFS v4.1, iflag=direct:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2, iflag=direct:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2, iflag=nocache oflag=nocache:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|
#######################
# # # With READ_PLUS # # #
#######################
NFS v4.1, iflag=direct:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2, iflag=direct:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2, iflag=nocache oflag=nocache:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 93 lines · Trond Myklebust
Quoted 81 lines · Anna Schumaker
Quoted 80 lines · Trond Myklebust
Quoted 78 lines · Anna Schumaker
Quoted 76 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
TrialQuoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 84 lines · Anna Schumaker
I did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128KMixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128Kbs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 8 lines
NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s |
That's a bit nasty. Any idea what is going on with the Mixed case here?
Quoted 126 lines
|---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Signature · 3 lines
Trond Myklebust Linux NFS client maintainer, PrimaryData [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 95 lines · Trond Myklebust
Quoted 84 lines · Anna Schumaker
I did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128KMixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128Kbs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|Quoted 8 lines
NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s |
Quoted 1 line
That's a bit nasty. Any idea what is going on with the Mixed case here?
Not offhand, but my first guess would be something to do with extra seeks to find how long each hole and data segment is.
Anna
Quoted 130 lines
Quoted 127 lines
|---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 96 lines · Anna Schumaker
Quoted 95 lines · Trond Myklebust
Quoted 92 lines · Anna Schumaker
I did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128KMixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128Kbs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s |
Quoted 3 lines
Quoted 1 line
That's a bit nasty. Any idea what is going on with the Mixed case here?
Not offhand, but my first guess would be something to do with extra seeks to find how long each hole and data segment is.
Remind us what "mixed" means? (I think you were alternating, but how large is each segment?)
--b.
Quoted 133 lines
Anna
Quoted 130 lines
Quoted 127 lines
|---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 97 lines · J. Bruce Fields
Quoted 96 lines · Anna Schumaker
Quoted 95 lines · Trond Myklebust
Quoted 92 lines · Anna Schumaker
I did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128KMixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128Kbs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s |
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
That's a bit nasty. Any idea what is going on with the Mixed case here?
Not offhand, but my first guess would be something to do with extra seeks to find how long each hole and data segment is.
Remind us what "mixed" means? (I think you were alternating, but how large is each segment?)
"Mixed" is alternating 4K segments.
Quoted 136 lines
--b.
Quoted 133 lines
Anna
Quoted 130 lines
Quoted 127 lines
|---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 98 lines · Anna Schumaker
Quoted 97 lines · J. Bruce Fields
Quoted 96 lines · Anna Schumaker
Quoted 95 lines · Trond Myklebust
Quoted 92 lines · Anna Schumaker
I did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128KMixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128Kbs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s |
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
That's a bit nasty. Any idea what is going on with the Mixed case here?
Not offhand, but my first guess would be something to do with extra seeks to find how long each hole and data segment is.
Remind us what "mixed" means? (I think you were alternating, but how large is each segment?)
"Mixed" is alternating 4K segments.
So it's probably doing 128/4 = 32 reads where previously one was necessary. You could confirm that by looking at the READ counts in /proc/self/mountstats. With odirect turned off maybe that's hidden by readahead?
--b.
Quoted 137 lines
Quoted 137 lines
--b.
Quoted 133 lines
Anna
Quoted 130 lines
Quoted 127 lines
|---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
I just ran some more tests comparing the directio case across different filesystem types. These tests used three 1G files: 100% data, 100% hole, and mixed file with alternating 4k data and hole segments. The mixed case seems to be consistently slower compared to NFS v4.1, and I'm at a loss for anything I could do to make it faster. Here are my numbers:
###########
# # # XFS # # #
###########
NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.883s | 1.808s | 1.781s | 1.685s | 1.591s | 1.746s | | Hole | 1.815s | 1.635s | 1.682s | 1.698s | 1.653s | 1.697s | | Mixed | 2.089s | 2.024s | 1.970s | 1.925s | 2.049s | 2.011s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.849s | 1.879s | 1.852s | 1.799s | 1.781s | 1.832s | | Hole | 0.668s | 0.600s | 0.611s | 0.619s | 0.617s | 0.623s | | Mixed | 5.913s | 5.811s | 5.952s | 5.962s | 5.806s | 5.889s | |---------|---------|---------|---------|---------|---------|---------|
############
# # # EXT4 # # #
############
NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 2.637s | 1.823s | 1.792s | 1.816s | 2.000s | 2.014s | | Hole | 1.734s | 1.743s | 1.709s | 1.761s | 1.871s | 1.764s | | Mixed | 5.465s | 2.158s | 2.254s | 2.676s | 2.422s | 2.995s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.934s | 1.783s | 1.800s | 2.010s | 1.982s | 1.902s | | Hole | 63.568s | 63.423s | 64.671s | 66.190s | 65.985s | 64.767s | | Mixed | 6.010s | 5.798s | 6.146s | 6.460s | 6.720s | 6.225s | |---------|---------|---------|---------|---------|---------|---------|
#############
# # # BTRFS # # #
#############
NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 2.386s | 1.952s | 1.832s | 1.818s | 1.826s | 1.963s | | Hole | 1.759s | 1.717s | 1.754s | 1.621s | 1.708s | 1.712s | | Mixed | 2.889s | 2.272s | 2.778s | 2.277s | 2.255s | 2.494s | |---------|---------|---------|---------|---------|---------|---------|
NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 2.586s | 1.816s | 2.022s | 1.862s | 1.975s | 2.052s | | Hole | 0.646s | 0.659s | 0.669s | 0.628s | 0.605s | 0.641s | | Mixed | 8.555s | 8.553s | 7.904s | 8.567s | 8.286s | 8.373s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 252 lines · J. Bruce Fields
Quoted 98 lines · Anna Schumaker
Quoted 97 lines · J. Bruce Fields
Quoted 96 lines · Anna Schumaker
Quoted 95 lines · Trond Myklebust
Quoted 92 lines · Anna Schumaker
I did two separate dd tests with the same 5G file from yesterday, and still using the same virtual machines. First, I ran dd using direct IO for reads:
dd if=/nfs/file iflag=direct of=/dev/null bs=128KMixed file performance was awful, so I reran without direct IO enabled for comparison:
dd if=/nfs/file iflag=nocache of=/dev/null oflag=nocache bs=128Kbs=128K sets the block size used by dd to the NFS rsize, without this dd will only read 512 bytes at a time and take forever to complete.
##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.704s | 11.055s | 11.329s | 11.453s | 10.741s | 11.256s | | Hole | 9.839s | 9.326s | 9.381s | 9.430s | 8.875s | 9.370s | | Mixed | 19.150s | 19.468s | 18.650s | 18.537s | 19.312s | 19.023s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 10.927s | 10.885s | 11.114s | 11.283s | 10.371s | 10.916s | | Hole | 9.515s | 9.039s | 9.116s | 8.867s | 8.905s | 9.088s | | Mixed | 19.149s | 18.656s | 19.400s | 18.834s | 20.041s | 19.216s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.808s | 6.698s | 7.482s | 6.761s | 7.235s | 6.995s | | Hole | 5.350s | 5.148s | 5.161s | 5.070s | 5.089s | 5.164s | | Mixed | 9.316s | 8.731s | 9.072s | 9.145s | 8.627s | 8.978s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.686s | 6.848s | 6.876s | 6.799s | 7.815s | 7.004s | | Hole | 5.092s | 5.330s | 5.050s | 5.280s | 5.030s | 5.156s | | Mixed | 8.142s | 7.897s | 8.040s | 7.960s | 8.050s | 8.018s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.464s | 10.181s | 10.048s | 9.452s | 10.795s | 9.988s | | Hole | 7.954s | 8.486s | 7.762s | 7.969s | 8.299s | 8.094s | | Mixed | 19.037s | 18.323s | 18.965s | 18.156s | 19.185s | 18.733s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=direct:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 11.923s | 10.026s | 10.222s | 12.387s | 11.431s | 11.198s | | Hole | 3.247s | 3.155s | 3.191s | 3.243s | 3.202s | 3.208s | | Mixed | 54.677s | 54.697s | 52.978s | 53.704s | 54.054s | 54.022s |Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
That's a bit nasty. Any idea what is going on with the Mixed case here?
Not offhand, but my first guess would be something to do with extra seeks to find how long each hole and data segment is.
Remind us what "mixed" means? (I think you were alternating, but how large is each segment?)
"Mixed" is alternating 4K segments.
So it's probably doing 128/4 = 32 reads where previously one was necessary. You could confirm that by looking at the READ counts in
/proc/self/mountstats. With odirect turned off maybe that's hidden by readahead?--b.
Quoted 137 lines
Quoted 137 lines
--b.
Quoted 133 lines
Anna
Quoted 130 lines
Quoted 127 lines
|---------|---------|---------|---------|---------|---------|---------|
NFS v4.1, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.788s | 6.802s | 6.750s | 6.756s | 6.852s | 6.790s | | Hole | 5.143s | 5.165s | 5.104s | 5.154s | 5.116s | 5.136s | | Mixed | 7.902s | 7.693s | 9.169s | 8.186s | 9.157s | 8.421s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2, iflag=nocache oflag=nocache:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.897s | 6.862s | 7.054s | 6.961s | 7.081s | 6.971s | | Hole | 1.690s | 1.673s | 1.553s | 1.554s | 1.490s | 1.592s | | Mixed | 9.009s | 7.840s | 7.661s | 8.945s | 7.649s | 8.221s | |---------|---------|---------|---------|---------|---------|---------|Quoted 85 lines · Trond Myklebust
Quoted 83 lines · Anna Schumaker
Quoted 82 lines · Trond Myklebust
Quoted 79 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the READ_PLUS patches:
##########################
# #
# Without READ_PLUS #
# #
##########################NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------|
| | 1 | 2 | 3 | 4 | 5 | Average |
|---------|---------|---------|---------|---------|---------|---------|
| Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s |
| Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s |
| Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s |
|---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------|
| | 1 | 2 | 3 | 4 | 5 | Average |
|---------|---------|---------|---------|---------|---------|---------|
| Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s |
| Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s |
| Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s |
|---------|---------|---------|---------|---------|---------|---------|#######################
# #
# With READ_PLUS #
# #
#######################NFS v4.1:
Trial
|---------|---------|---------|---------|---------|---------|---------|
| | 1 | 2 | 3 | 4 | 5 | Average |
|---------|---------|---------|---------|---------|---------|---------|
| Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s |
| Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s |
| Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s |
|---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial
|---------|---------|---------|---------|---------|---------|---------|
| | 1 | 2 | 3 | 4 | 5 | Average |
|---------|---------|---------|---------|---------|---------|---------|
| Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s |
| Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s |
| Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s |
|---------|---------|---------|---------|---------|---------|---------|Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I agree that you have to use the '
drop_caches' interface on the server, but why not just useO_DIRECTon the clients?I've been reading by using cat from my test shell script:
time cat /nfs/file > /dev/null. I can write something to read files withO_DIRECTif that would be more useful!'dd' can do that for you if the appropriate incantations are performed.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 6 lines · Anna Schumaker
I just ran some more tests comparing the directio case across different filesystem types. These tests used three 1G files: 100% data, 100% hole, and mixed file with alternating 4k data and hole segments. The mixed case seems to be consistently slower compared to NFS v4.1, and I'm at a loss for anything I could do to make it faster. Here are my numbers:
Have you tried the implementation we discussed that always returns a single segment covering the whole requested range, by treating holes as data if necessary when they don't cover the whole range?
(Also: I assume it's the same as before, but: when you post test results, could you repost if necessary:
- what the actual test is
- what the hardware/software setup is on client and server
so that we have reproduceable results for posterity's sake.)
Interesting that "Mixed" is a little slower even before READ_PLUS.
And I guess we should really report this to ext4 people, looks like they may have a bug.
--b.
Quoted 92 lines
###########
# # # XFS # # ############
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.883s | 1.808s | 1.781s | 1.685s | 1.591s | 1.746s | | Hole | 1.815s | 1.635s | 1.682s | 1.698s | 1.653s | 1.697s | | Mixed | 2.089s | 2.024s | 1.970s | 1.925s | 2.049s | 2.011s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.849s | 1.879s | 1.852s | 1.799s | 1.781s | 1.832s | | Hole | 0.668s | 0.600s | 0.611s | 0.619s | 0.617s | 0.623s | | Mixed | 5.913s | 5.811s | 5.952s | 5.962s | 5.806s | 5.889s | |---------|---------|---------|---------|---------|---------|---------|############
# # # EXT4 # # #############
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 2.637s | 1.823s | 1.792s | 1.816s | 2.000s | 2.014s | | Hole | 1.734s | 1.743s | 1.709s | 1.761s | 1.871s | 1.764s | | Mixed | 5.465s | 2.158s | 2.254s | 2.676s | 2.422s | 2.995s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.934s | 1.783s | 1.800s | 2.010s | 1.982s | 1.902s | | Hole | 63.568s | 63.423s | 64.671s | 66.190s | 65.985s | 64.767s | | Mixed | 6.010s | 5.798s | 6.146s | 6.460s | 6.720s | 6.225s | |---------|---------|---------|---------|---------|---------|---------|#############
# # # BTRFS # # ##############
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 2.386s | 1.952s | 1.832s | 1.818s | 1.826s | 1.963s | | Hole | 1.759s | 1.717s | 1.754s | 1.621s | 1.708s | 1.712s | | Mixed | 2.889s | 2.272s | 2.778s | 2.277s | 2.255s | 2.494s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 2.586s | 1.816s | 2.022s | 1.862s | 1.975s | 2.052s | | Hole | 0.646s | 0.659s | 0.669s | 0.628s | 0.605s | 0.641s | | Mixed | 8.555s | 8.553s | 7.904s | 8.567s | 8.286s | 8.373s | |---------|---------|---------|---------|---------|---------|---------|
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 18 lines · J. Bruce Fields
Quoted 6 lines · Anna Schumaker
I just ran some more tests comparing the directio case across different filesystem types. These tests used three 1G files: 100% data, 100% hole, and mixed file with alternating 4k data and hole segments. The mixed case seems to be consistently slower compared to NFS v4.1, and I'm at a loss for anything I could do to make it faster. Here are my numbers:
Have you tried the implementation we discussed that always returns a single segment covering the whole requested range, by treating holes as data if necessary when they don't cover the whole range?
(Also: I assume it's the same as before, but: when you post test results, could you repost if necessary:
- what the actual test is
- what the hardware/software setup is on client and server
Quoted 6 lines
so that we have reproduceable results for posterity's sake.)
Interesting that "Mixed" is a little slower even before
READ_PLUS.And I guess we should really report this to ext4 people, looks like they may have a bug.
FWIW, this is what I was using to test SEEK_HOLE/SEEK_DATA and map out holes on files on my local disk. Might be worth checking whether the ext4 slowdowns are reproduceable just with something like this, to rule out protocol problems.
--b.
#define _GNU_SOURCE
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <err.h>
long round_up(long n, long b)
{
return ((n + b - 1)/b) * b;
}long round_down(long n, long b)
{
return (n/b) * b;
}long hbytes = 0;
long rplusbytes = 0;
long num_holes = 0;
do_stats(off_t hole_start, off_t hole_end)
{
off_t hole_start_up, hole_end_down;
hole_start_up = round_up(hole_start, 1024*1024);
hole_end_down = round_down(hole_end, 1024*1024);
hbytes += hole_end - hole_start;
if (hole_start < hole_end)
num_holes++;
if (hole_start_up < hole_end_down)
rplusbytes += hole_end_down - hole_start_up;
}int main(int argc, char *argv[])
{
off_t hole_start, hole_end;
int fd;
char *name;
/* Map out holes with SEEK_HOLE, SEEK_DATA */
/* Useful statistics:
* - what percentage of file is in holes?
* - what percentage of file would be skipped if we read it
* sequentially in 1MB chunks?
*/
if (argc != 2)
errx(1, "usage: %s <filename>\n", argv[0]);
name = argv[1];
fd = open(name, O_RDONLY);
if (fd == -1)
err(1, "open");
hole_end = 0;
while (1) {
hole_start = lseek(fd, hole_end, SEEK_HOLE);
if (hole_start == -1)
err(1, "lseek");
hole_end = lseek(fd, hole_start, SEEK_DATA);
if (hole_end == -1) {
if (errno == ENXIO)
break;
err(1, "lseek");
}
do_stats(hole_start, hole_end);
}
hole_end = lseek(fd, 0, SEEK_END);
do_stats(hole_start, hole_end);
printf("%ld holes\n", num_holes);
printf("total hole bytes: %ld (%.0f%)\n", hbytes,
100 * (float)hbytes/hole_end);
printf("in aligned 1MB chunks: %ld (%.0f%)\n", rplusbytes,
100 * (float)rplusbytes/hole_end);
}Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 23 lines · J. Bruce Fields
Quoted 22 lines · J. Bruce Fields
Quoted 6 lines · Anna Schumaker
I just ran some more tests comparing the directio case across different filesystem types. These tests used three 1G files: 100% data, 100% hole, and mixed file with alternating 4k data and hole segments. The mixed case seems to be consistently slower compared to NFS v4.1, and I'm at a loss for anything I could do to make it faster. Here are my numbers:
Have you tried the implementation we discussed that always returns a single segment covering the whole requested range, by treating holes as data if necessary when they don't cover the whole range?
(Also: I assume it's the same as before, but: when you post test results, could you repost if necessary:
- what the actual test is
- what the hardware/software setup is on client and serverso that we have reproduceable results for posterity's sake.)
Interesting that "Mixed" is a little slower even before
READ_PLUS.
Quoted 7 lines
Quoted 2 lines
And I guess we should really report this to ext4 people, looks like they may have a bug.
FWIW, this is what I was using to test SEEK_HOLE/SEEK_DATA and map out holes on files on my local disk. Might be worth checking whether the ext4 slowdowns are reproduceable just with something like this, to rule out protocol problems.
Wheel reinvention. :)
$ rm -f /mnt/scratch/bar $ for i in `seq 20 -2 0`; do
Quoted 2 lines
sudo
xfs_io-f -c "pwrite $((i * 8192)) 4096"/mnt/scratch/bardone
.....
$ sync $ sudo xfs_io -c "seek -ar 0" /mnt/scratch/bar Whence Result DATA 0 HOLE 4096 DATA 16384 HOLE 20480 DATA 32768 HOLE 36864 DATA 49152 HOLE 53248 DATA 65536 HOLE 69632 DATA 81920 HOLE 86016 DATA 98304 HOLE 102400 DATA 114688 HOLE 118784 DATA 131072 HOLE 135168 DATA 147456 HOLE 151552 DATA 163840 HOLE 167936 $
-Dave.
Signature · 2 lines
Dave Chinner [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 10 lines · Dave Chinner
Quoted 9 lines · J. Bruce Fields
Quoted 8 lines · J. Bruce Fields
Quoted 6 lines · Anna Schumaker
I just ran some more tests comparing the directio case across different filesystem types. These tests used three 1G files: 100% data, 100% hole, and mixed file with alternating 4k data and hole segments. The mixed case seems to be consistently slower compared to NFS v4.1, and I'm at a loss for anything I could do to make it faster. Here are my numbers:
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Have you tried the implementation we discussed that always returns a single segment covering the whole requested range, by treating holes as data if necessary when they don't cover the whole range?
Uh, sorry, I forgot, I think you're running with the patches that support full multi-segment READ_PLUS on both sides so there's not that issue with multiplying RPC's in this case.
Still, might be interesting to compare. And wouldn't hurt to remind us of these details when you repost this stuff to help keep my forgetful self going in circles.
Quoted 13 lines
Quoted 13 lines
Quoted 12 lines
(Also: I assume it's the same as before, but: when you post test results, could you repost if necessary:
- what the actual test is
- what the hardware/software setup is on client and serverso that we have reproduceable results for posterity's sake.)
Interesting that "Mixed" is a little slower even before
READ_PLUS.And I guess we should really report this to ext4 people, looks like they may have a bug.
Quoted 6 lines
Quoted 4 lines
FWIW, this is what I was using to test SEEK_HOLE/SEEK_DATA and map out holes on files on my local disk. Might be worth checking whether the ext4 slowdowns are reproduceable just with something like this, to rule out protocol problems.
Wheel reinvention. :)
xfs_io appears to have a lot of wheels. OK, I'll go read that man page one of these days.
--b.
Quoted 37 lines
$ rm -f /mnt/scratch/bar $ for i in `seq 20 -2 0`; doQuoted 2 lines
sudo
xfs_io-f -c "pwrite $((i * 8192)) 4096"/mnt/scratch/bardone.....
$ sync $ sudo xfs_io -c "seek -ar 0" /mnt/scratch/bar Whence Result DATA 0 HOLE 4096 DATA 16384 HOLE 20480 DATA 32768 HOLE 36864 DATA 49152 HOLE 53248 DATA 65536 HOLE 69632 DATA 81920 HOLE 86016 DATA 98304 HOLE 102400 DATA 114688 HOLE 118784 DATA 131072 HOLE 135168 DATA 147456 HOLE 151552 DATA 163840 HOLE 167936 $-Dave.
Signature · 2 lines
Dave Chinner [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
In reply to Anna SchumakerQuoted 27 lines · Anna Schumaker
I just ran some more tests comparing the directio case across different filesystem types. These tests used three 1G files:
100% data, 100% hole, and mixed file with alternating 4k data and hole segments. The mixed case seems to be consistently slower compared to NFS v4.1, and I'm at a loss for anything I could do to make it faster. Here are my numbers:###########
# # # XFS # # ############
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.883s | 1.808s | 1.781s | 1.685s | 1.591s | 1.746s | | Hole | 1.815s | 1.635s | 1.682s | 1.698s | 1.653s | 1.697s | | Mixed | 2.089s | 2.024s | 1.970s | 1.925s | 2.049s | 2.011s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial
Quoted 7 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 1.849s | 1.879s | 1.852s | 1.799s | 1.781s | 1.832s | | Hole | 0.668s | 0.600s | 0.611s | 0.619s | 0.617s | 0.623s | | Mixed | 5.913s | 5.811s | 5.952s | 5.962s | 5.806s | 5.889s | |---------|---------|---------|---------|---------|---------|---------|
What that says to me is that the READ_PLUS when there are (worst case) mixed holes is either burning a lot more CPU than we expected or it is serialising somewhere (not sure where, everything in XFS should be shared locks on read/seek).
Can you run a perf profile (even just a snapshot from perf top) on the server so we can see a bit about what is happening on the CPU for the different workloads?
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
In reply to Anna SchumakerQuoted 71 lines · Anna Schumaker
Quoted 70 lines · J. Bruce Fields
Quoted 69 lines · Trond Myklebust
Quoted 66 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.
What sort of device is the exported xfs filesystem on? (Can't there be a second level of caching on the guest, depending on how it's set up?)
Can we get results on bare metal? (The kvm test might be a good worst-case for read_plus, as I'd expect bandwidth to be relatively high compared to the cost of the extra memcpy's or seek calls. But it also seems more complicated.)
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
Quoted 76 lines · J. Bruce Fields
Quoted 75 lines · Anna Schumaker
Quoted 74 lines · J. Bruce Fields
Quoted 72 lines · Trond Myklebust
Quoted 58 lines · Anna Schumaker
Here are my updated numbers! I tested with files 5G in size: one 100% data, one 100% hole, and one alternating between hole and data every 4K. I collected data for both v4.1 and v4.2 with and without the
READ_PLUSpatches:##########################
# # # Without READ_PLUS # # ###########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 8.723s | 7.243s | 8.252s | 6.997s | 6.980s | 7.639s | | Hole | 5.271s | 5.224s | 5.060s | 4.897s | 5.321s | 5.155s | | Mixed | 8.050s | 10.057s | 7.919s | 8.060s | 9.557s | 8.729s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 6.707s | 7.070s | 6.722s | 6.761s | 6.810s | 6.814s | | Hole | 5.152s | 5.149s | 5.213s | 5.206s | 5.312s | 5.206s | | Mixed | 7.979s | 7.985s | 8.177s | 7.772s | 8.280s | 8.039s | |---------|---------|---------|---------|---------|---------|---------|#######################
# # # With READ_PLUS # # ########################
NFS v4.1:
Trial|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 9.082s | 7.008s | 7.116s | 6.771s | 7.902s | 7.576s | | Hole | 5.333s | 5.358s | 5.380s | 5.161s | 5.282s | 5.303s | | Mixed | 8.189s | 8.308s | 9.540s | 7.937s | 8.420s | 8.479s | |---------|---------|---------|---------|---------|---------|---------|NFS v4.2:
TrialQuoted 8 lines
|---------|---------|---------|---------|---------|---------|---------| | | 1 | 2 | 3 | 4 | 5 | Average | |---------|---------|---------|---------|---------|---------|---------| | Data | 7.033s | 6.829s | 7.025s | 6.873s | 7.134s | 6.979s | | Hole | 1.794s | 1.800s | 1.905s | 1.811s | 1.725s | 1.807s | | Mixed | 7.590s | 8.777s | 9.423s | 10.366s | 8.024s | 8.836s | |---------|---------|---------|---------|---------|---------|---------|So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Also, could you describe the setup (are these still kvm's), and how you're clearing the cache between runs?
These are still KVMs and my server is exporting an xfs filesystem. I clear caches by running "echo 3 >
/proc/sys/vm/drop_caches" on the server before every read, and I remount my client after reading each set of three files once.What sort of device is the exported xfs filesystem on? (Can't there be a second level of caching on the guest, depending on how it's set up?)
My host is a macbook pro running Archlinux, and I have all my virtio disks set to "cache mode = none". Let me know if you were asking something different!
Quoted 5 lines
Can we get results on bare metal? (The kvm test might be a good worst-case for
read_plus, as I'd expect bandwidth to be relatively high compared to the cost of the extra memcpy's or seek calls. But it also seems more complicated.)
I do all of my testing on kvm these days! I'll see how difficult it is to setup refind with a custom kernel to test between my laptop and my desktop (or I could run the test between my raspberry pis!)
Anna
Quoted 3 lines
--b.
Re: [PATCH v3 3/3] NFSD: Add support for encoding multiple segments
In reply to Trond MyklebustQuoted 3 lines · Trond Myklebust
So there is a clear win in the 100% hole case here, but otherwise the statistical fluctuations are dominating the numbers. Can you get us a little more stats and then perhaps run the results through nfsometer?
And that's just the uncached reads if I understand the thread correctly. The cached case isn't uncommon in real life, so regressing it isn't really an option either.