Skip to contentSkip to conversation

NFSv4.2: Add support for the COPY operation

PATCH In review Anna Schumaker 3 participants 14 messages Dec 3, 2015 – Dec 7, 2015

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover NFSv4.2: Add support for the COPY operation

These patches add client and server support for the NFS v4.2 COPY operation, and depend on the new copy_file_range() system call currently scheduled for Linux 4.5. I gathered performance information by comparing the runtime and RPC count of my test program, included as an RFC patch, against /usr/bin/cp for various file sizes.

/usr/bin/cp:

                      size:    513MB   1024MB   1536MB   2048MB
------------- ------------- -------- -------- -------- --------
nfs v4 client        total:     8203    16396    24588    32780
------------- ------------- -------- -------- -------- --------
nfs v4 client         read:     4096     8192    12288    16384
nfs v4 client        write:     4096     8192    12288    16384
nfs v4 client       commit:        1        1        1        1
nfs v4 client         open:        1        1        1        1
nfs v4 client    open_noat:        2        2        2        2
nfs v4 client        close:        1        1        1        1
nfs v4 client      setattr:        2        2        2        2
nfs v4 client       access:        2        3        3        3
nfs v4 client      getattr:        2        2        2        2
/usr/bin/cp /nfs/test-512  /nfs/test-copy  0.00s user 0.32s system 14% cpu 2.209 total
/usr/bin/cp /nfs/test-1024 /nfs/test-copy  0.00s user 0.66s system 18% cpu 3.651 total
/usr/bin/cp /nfs/test-1536 /nfs/test-copy  0.02s user 0.97s system 18% cpu 5.477 total
/usr/bin/cp /nfs/test-2048 /nfs/test-copy  0.00s user 1.38s system 15% cpu 9.085 total

Copy system call:

                      size:    512MB   1024MB   1536MB   2048MB
------------- ------------- -------- -------- -------- --------
nfs v4 client        total:        6        6        6        6
------------- ------------- -------- -------- -------- --------
nfs v4 client         open:        2        2        2        2
nfs v4 client        close:        2        2        2        2
nfs v4 client       access:        1        1        1        1
nfs v4 client         copy:        1        1        1        1
./nfscopy /nfs/test-512  /nfs/test-copy  0.00s user 0.00s system 0% cpu 1.148 total
./nfscopy /nfs/test-1024 /nfs/test-copy  0.00s user 0.00s system 0% cpu 2.293 total
./nfscopy /nfs/test-1536 /nfs/test-copy  0.00s user 0.00s system 0% cpu 3.037 total
./nfscopy /nfs/test-2048 /nfs/test-copy  0.00s user 0.00s system 0% cpu 4.045 total

Questions, comments, and other testing ideas would be greatly appreciated!

Thanks,
Anna

Anna Schumaker (4):
NFSD: Pass filehandle to nfs4_preprocess_stateid_op()
NFSD: Implement the COPY call
NFS: Add COPY nfs operation
vfs_copy_range() test program

 fs/nfs/nfs42.h            |   1 +
 fs/nfs/nfs42proc.c        |  58 ++++++++++++++++++++
 fs/nfs/nfs42xdr.c         | 136 ++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfs/nfs4file.c         |   8 +++
 fs/nfs/nfs4proc.c         |   1 +
 fs/nfs/nfs4xdr.c          |   1 +
 fs/nfs/objlayout/Makefile |   0
 fs/nfsd/nfs4proc.c        |  91 ++++++++++++++++++++++++++++---
 fs/nfsd/nfs4state.c       |   5 +-
 fs/nfsd/nfs4xdr.c         |  62 ++++++++++++++++++++-
 fs/nfsd/state.h           |   4 +-
 fs/nfsd/vfs.c             |  17 ++++++
 fs/nfsd/vfs.h             |   1 +
 fs/nfsd/xdr4.h            |  23 ++++++++
 include/linux/nfs4.h      |   1 +
 include/linux/nfs_fs_sb.h |   1 +
 include/linux/nfs_xdr.h   |  27 +++++++++
 nfscopy.c                 |  59 ++++++++++++++++++++
 18 files changed, 482 insertions(+), 14 deletions(-)
 create mode 100644 fs/nfs/objlayout/Makefile
 create mode 100644 nfscopy.c
Signature · 1 line
2.6.3
#1 lore raw

PATCH 1/3 NFSD: Pass filehandle to nfs4_preprocess_stateid_op()

This will be needed so COPY can look up the saved_fh in addition to the current_fh.

---
 fs/nfsd/nfs4proc.c  | 16 +++++++++-------
 fs/nfsd/nfs4state.c |  5 ++---
 fs/nfsd/state.h     |  4 ++--
 3 files changed, 13 insertions(+), 12 deletions(-)
fs/nfsd/nfs4proc.c +9 -7
index a9f096c..3ba10a3 100644
@@ -774,8 +774,9 @@ nfsd4_read(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		clear_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
 
 	/* check stateid */
-	status = nfs4_preprocess_stateid_op(rqstp, cstate, &read->rd_stateid,
-			RD_STATE, &read->rd_filp, &read->rd_tmp_file);
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
+					&read->rd_stateid, RD_STATE,
+					&read->rd_filp, &read->rd_tmp_file);
 	if (status) {
 		dprintk("NFSD: nfsd4_read: couldn't process stateid!\n");
 		goto out;
@@ -921,7 +922,8 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 
 	if (setattr->sa_iattr.ia_valid & ATTR_SIZE) {
 		status = nfs4_preprocess_stateid_op(rqstp, cstate,
-			&setattr->sa_stateid, WR_STATE, NULL, NULL);
+				&cstate->current_fh, &setattr->sa_stateid,
+				WR_STATE, NULL, NULL);
 		if (status) {
 			dprintk("NFSD: nfsd4_setattr: couldn't process stateid!\n");
 			return status;
@@ -985,8 +987,8 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	if (write->wr_offset >= OFFSET_MAX)
 		return nfserr_inval;
 
-	status = nfs4_preprocess_stateid_op(rqstp, cstate, stateid, WR_STATE,
-			&filp, NULL);
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
+						stateid, WR_STATE, &filp, NULL);
 	if (status) {
 		dprintk("NFSD: nfsd4_write: couldn't process stateid!\n");
 		return status;
@@ -1016,7 +1018,7 @@ nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	__be32 status = nfserr_notsupp;
 	struct file *file;
 
-	status = nfs4_preprocess_stateid_op(rqstp, cstate,
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    &fallocate->falloc_stateid,
 					    WR_STATE, &file, NULL);
 	if (status != nfs_ok) {
@@ -1055,7 +1057,7 @@ nfsd4_seek(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	__be32 status;
 	struct file *file;
 
-	status = nfs4_preprocess_stateid_op(rqstp, cstate,
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    &seek->seek_stateid,
 					    RD_STATE, &file, NULL);
 	if (status) {
fs/nfsd/nfs4state.c +2 -3
index 6b800b5..df5dba6 100644
@@ -4797,10 +4797,9 @@ nfs4_check_file(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfs4_stid *s,
  */
 __be32
 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
-		struct nfsd4_compound_state *cstate, stateid_t *stateid,
-		int flags, struct file **filpp, bool *tmp_file)
+		struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
+		stateid_t *stateid, int flags, struct file **filpp, bool *tmp_file)
 {
-	struct svc_fh *fhp = &cstate->current_fh;
 	struct inode *ino = d_inode(fhp->fh_dentry);
 	struct net *net = SVC_NET(rqstp);
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
fs/nfsd/state.h +2 -2
index 77fdf4d..99432b7 100644
@@ -578,8 +578,8 @@ struct nfsd4_compound_state;
 struct nfsd_net;
 
 extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp,
-		struct nfsd4_compound_state *cstate, stateid_t *stateid,
-		int flags, struct file **filp, bool *tmp_file);
+		struct nfsd4_compound_state *cstate, struct svc_fh *fhp,
+		stateid_t *stateid, int flags, struct file **filp, bool *tmp_file);
 __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     stateid_t *stateid, unsigned char typemask,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
Signature · 1 line
2.6.3
#2 lore raw

PATCH 2/3 NFSD: Implement the COPY call

In reply to Anna Schumaker

From: Anna Schumaker <[email protected]>

I only implemented the sync version of this call, since it's the easiest. I can simply call vfs_copy_range() and have the vfs do the right thing for the filesystem being exported.

---
 fs/nfsd/nfs4proc.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfs4xdr.c  | 62 ++++++++++++++++++++++++++++++++++++++++++--
 fs/nfsd/vfs.c      | 17 +++++++++++++
 fs/nfsd/vfs.h      |  1 +
 fs/nfsd/xdr4.h     | 23 +++++++++++++++++
 5 files changed, 176 insertions(+), 2 deletions(-)
fs/nfsd/nfs4proc.c +75 -0
index 3ba10a3..e0b1f43 100644
@@ -1012,6 +1012,63 @@ nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 }
 
 static __be32
+nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+		  struct nfsd4_copy *copy, struct file **src, struct file **dst)
+{
+	__be32 status;
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->save_fh,
+						&copy->cp_src_stateid, RD_STATE,
+						src, NULL);
+	if (status) {
+		dprintk("NFSD: nfsd4_copy: couldn't process src stateid!\n");
+		return status;
+	}
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
+						&copy->cp_dst_stateid, WR_STATE,
+						dst, NULL);
+	if (status) {
+		dprintk("NFSD: nfsd4_copy: couldn't process dst stateid!\n");
+		fput(*src);
+	}
+
+	return status;
+}
+
+static __be32
+nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+		struct nfsd4_copy *copy)
+{
+	ssize_t bytes;
+	__be32 status;
+	struct file *src = NULL, *dst = NULL;
+
+	status = nfsd4_verify_copy(rqstp, cstate, copy, &src, &dst);
+	if (status)
+		return status;
+
+	bytes = nfsd_copy_range(src, copy->cp_src_pos,
+				 dst, copy->cp_dst_pos,
+				 copy->cp_count);
+
+	if (bytes < 0)
+		status = nfserrno(bytes);
+	else {
+		copy->cp_res.wr_bytes_written = bytes;
+		copy->cp_res.wr_stable_how = NFS_FILE_SYNC;
+		copy->cp_consecutive = 1;
+		copy->cp_synchronous = 1;
+		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
+		status = nfs_ok;
+	}
+
+	fput(src);
+	fput(dst);
+	return status;
+}
+
+static __be32
 nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		struct nfsd4_fallocate *fallocate, int flags)
 {
@@ -1925,6 +1982,18 @@ static inline u32 nfsd4_create_session_rsize(struct svc_rqst *rqstp, struct nfsd
 		op_encode_channel_attrs_maxsz) * sizeof(__be32);
 }
 
+static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+{
+	return (op_encode_hdr_size +
+		1 /* wr_callback */ +
+		op_encode_stateid_maxsz /* wr_callback */ +
+		2 /* wr_count */ +
+		1 /* wr_committed */ +
+		op_encode_verifier_maxsz +
+		1 /* cr_consecutive */ +
+		1 /* cr_synchronous */) * sizeof(__be32);
+}
+
 #ifdef CONFIG_NFSD_PNFS
 /*
  * At this stage we don't really know what layout driver will handle the request,
@@ -2281,6 +2350,12 @@ static struct nfsd4_operation nfsd4_ops[] = {
 		.op_name = "OP_DEALLOCATE",
 		.op_rsize_bop = (nfsd4op_rsize)nfsd4_only_status_rsize,
 	},
+	[OP_COPY] = {
+		.op_func = (nfsd4op_func)nfsd4_copy,
+		.op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME,
+		.op_name = "OP_COPY",
+		.op_rsize_bop = (nfsd4op_rsize)nfsd4_copy_rsize,
+	},
 	[OP_SEEK] = {
 		.op_func = (nfsd4op_func)nfsd4_seek,
 		.op_name = "OP_SEEK",
fs/nfsd/nfs4xdr.c +60 -2

86 lines not on this page · raw message

fs/nfsd/vfs.c +17 -0
index 994d66f..225ff12 100644
@@ -36,6 +36,7 @@
 #endif /* CONFIG_NFSD_V3 */
 
 #ifdef CONFIG_NFSD_V4
+#include "../internal.h"
 #include "acl.h"
 #include "idmap.h"
 #endif /* CONFIG_NFSD_V4 */
@@ -498,6 +499,22 @@ __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif
 
+ssize_t nfsd_copy_range(struct file *src, u64 src_pos,
+		       struct file *dst, u64 dst_pos,
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;
+
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);
+	if (bytes > 0)
+		vfs_fsync_range(dst, dst_pos, dst_pos + bytes, 0);
+	return bytes;
+}
+
 __be32 nfsd4_vfs_fallocate(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			   struct file *file, loff_t offset, loff_t len,
 			   int flags)
fs/nfsd/vfs.h +1 -0
index fcfc48c..0243c5a 100644
@@ -91,6 +91,7 @@ __be32		nfsd_symlink(struct svc_rqst *, struct svc_fh *,
 				struct svc_fh *res);
 __be32		nfsd_link(struct svc_rqst *, struct svc_fh *,
 				char *, int, struct svc_fh *);
+ssize_t		nfsd_copy_range(struct file *, u64, struct file *, u64, u64);
 __be32		nfsd_rename(struct svc_rqst *,
 				struct svc_fh *, char *, int,
 				struct svc_fh *, char *, int);
fs/nfsd/xdr4.h +23 -0
index ce7362c..7ef82c9 100644
@@ -491,6 +491,28 @@ struct nfsd4_fallocate {
 	u64		falloc_length;
 };
 
+struct nfsd42_write_res {
+	u64			wr_bytes_written;
+	u32			wr_stable_how;
+	nfs4_verifier		wr_verifier;
+};
+
+struct nfsd4_copy {
+	/* request */
+	stateid_t	cp_src_stateid;
+	stateid_t	cp_dst_stateid;
+	u64		cp_src_pos;
+	u64		cp_dst_pos;
+	u64		cp_count;
+
+	/* both */
+	bool		cp_consecutive;
+	bool		cp_synchronous;
+
+	/* response */
+	struct nfsd42_write_res	cp_res;
+};
+
 struct nfsd4_seek {
 	/* request */
 	stateid_t	seek_stateid;
@@ -555,6 +577,7 @@ struct nfsd4_op {
 		/* NFSv4.2 */
 		struct nfsd4_fallocate		allocate;
 		struct nfsd4_fallocate		deallocate;
+		struct nfsd4_copy		copy;
 		struct nfsd4_seek		seek;
 	} u;
 	struct nfs4_replay *			replay;
Signature · 1 line
2.6.3
#3 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 507

Quoted 10 lines · Anna Schumaker
fs/nfsd/vfs.c +6 -0
@@ -498,6 +499,22 @@ __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif
 
+ssize_t nfsd_copy_range(struct file *src, u64 src_pos,
+		       struct file *dst, u64 dst_pos,
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;

Why that value? Could I get a comment here?

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 512

Quoted 5 lines
fs/nfsd/vfs.c +5 -0
+
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);

Sorry, I lost track of the copy discussions: does this only work on filesystems with special support, or does it fall back on doing the copy by hand? Which filesystems (of the exportable filesystems) support this?

--b.

Quoted 8 lines
fs/nfsd/vfs.c +5 -0
+	if (bytes > 0)
+		vfs_fsync_range(dst, dst_pos, dst_pos + bytes, 0);
+	return bytes;
+}
+
 __be32 nfsd4_vfs_fallocate(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			   struct file *file, loff_t offset, loff_t len,
 			   int flags)
#4 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

Quoted 5 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
(quoted diff) +0 -0
@@ -498,6 +499,22 @@ __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif
 
Quoted 8 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 507

Quoted 6 lines
fs/nfsd/vfs.c +6 -0
+ssize_t nfsd_copy_range(struct file *src, u64 src_pos,
+		       struct file *dst, u64 dst_pos,
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;

Why that value? Could I get a comment here?

Whoops! I had a comment there at one point, but I must have deleted it :(. That value is to cap copies to 256MB.

Quoted 11 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 512

Quoted 6 lines
fs/nfsd/vfs.c +5 -0
+
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);

Sorry, I lost track of the copy discussions: does this only work on filesystems with special support, or does it fall back on doing the copy by hand? Which filesystems (of the exportable filesystems) support this?

The system call falls back on doing the copy by hand if there is no filesystem acceleration.

Anna

Quoted 16 lines

--b.

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 520

Quoted 8 lines
fs/nfsd/vfs.c +5 -0
+	if (bytes > 0)
+		vfs_fsync_range(dst, dst_pos, dst_pos + bytes, 0);
+	return bytes;
+}
+
 __be32 nfsd4_vfs_fallocate(struct svc_rqst *rqstp, struct svc_fh *fhp,
 			   struct file *file, loff_t offset, loff_t len,
 			   int flags)

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#5 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

Quoted 15 lines · Anna Schumaker
Quoted 5 lines · J. Bruce Fields
Quoted 4 lines · Anna Schumaker
(quoted diff) +0 -0
@@ -498,6 +499,22 @@ __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif
 
Quoted 8 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 507

Quoted 6 lines
fs/nfsd/vfs.c +6 -0
+ssize_t nfsd_copy_range(struct file *src, u64 src_pos,
+		       struct file *dst, u64 dst_pos,
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;

Why that value? Could I get a comment here?

Quoted 1 line

Whoops! I had a comment there at one point, but I must have deleted it :(. That value is to cap copies to 256MB.

Could you include some justification for the choice of that particular value?

Quoted 11 lines
Quoted 9 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 512

Quoted 4 lines
fs/nfsd/vfs.c +4 -0
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);

Sorry, I lost track of the copy discussions: does this only work on filesystems with special support, or does it fall back on doing the copy by hand? Which filesystems (of the exportable filesystems) support this?

The system call falls back on doing the copy by hand if there is no filesystem acceleration.

Is this practical? It means a huge range in possible latency of the single COPY call depending on filesystem.

I guess I can live with it and we can see if people run into problems in practice. But let's make sure this is documented.

--b.

#6 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

Quoted 16 lines · J. Bruce Fields
Quoted 15 lines · Anna Schumaker
Quoted 13 lines · J. Bruce Fields

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 507

Quoted 10 lines · Anna Schumaker
fs/nfsd/vfs.c +6 -0
@@ -498,6 +499,22 @@ __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif
 
+ssize_t nfsd_copy_range(struct file *src, u64 src_pos,
+		       struct file *dst, u64 dst_pos,
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;

Why that value? Could I get a comment here?

Quoted 4 lines
Quoted 1 line

Whoops! I had a comment there at one point, but I must have deleted it :(. That value is to cap copies to 256MB.

Could you include some justification for the choice of that particular value?

Yeah, I can run tests with different values and include the results in v2.

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 512

Quoted 5 lines
fs/nfsd/vfs.c +4 -0
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);

Sorry, I lost track of the copy discussions: does this only work on filesystems with special support, or does it fall back on doing the copy by hand? Which filesystems (of the exportable filesystems) support this?

Quoted 4 lines
Quoted 1 line

The system call falls back on doing the copy by hand if there is no filesystem acceleration.

Is this practical? It means a huge range in possible latency of the single COPY call depending on filesystem.

That's why I'm breaking copies into smaller chunks, rather than doing everything at once.

Quoted 3 lines

I guess I can live with it and we can see if people run into problems in practice. But let's make sure this is documented.

Okay. I'll add documentation about this!

Anna

Quoted 7 lines

--b.
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#7 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

Quoted 40 lines · Anna Schumaker
Quoted 16 lines · J. Bruce Fields
Quoted 15 lines · Anna Schumaker
Quoted 13 lines · J. Bruce Fields

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 507

Quoted 10 lines · Anna Schumaker
fs/nfsd/vfs.c +6 -0
@@ -498,6 +499,22 @@ __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp,
 }
 #endif
 
+ssize_t nfsd_copy_range(struct file *src, u64 src_pos,
+		       struct file *dst, u64 dst_pos,
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;

Why that value? Could I get a comment here?

Quoted 4 lines
Quoted 1 line

Whoops! I had a comment there at one point, but I must have deleted it :(. That value is to cap copies to 256MB.

Could you include some justification for the choice of that particular value?

Yeah, I can run tests with different values and include the results in v2.

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 512

Quoted 5 lines
fs/nfsd/vfs.c +4 -0
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);

Sorry, I lost track of the copy discussions: does this only work on filesystems with special support, or does it fall back on doing the copy by hand? Which filesystems (of the exportable filesystems) support this?

Quoted 4 lines
Quoted 1 line

The system call falls back on doing the copy by hand if there is no filesystem acceleration.

Is this practical? It means a huge range in possible latency of the single COPY call depending on filesystem.

Quoted 7 lines

That's why I'm breaking copies into smaller chunks, rather than doing everything at once.

Quoted 3 lines

I guess I can live with it and we can see if people run into problems in practice. But let's make sure this is documented.

Okay. I'll add documentation about this!

OK, thanks. I think those are my only concerns, good to see that this is all the new code we need now.--b.

#8 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

In reply to Anna Schumaker
Quoted 11 lines

static __be32

fs/nfsd/nfs4proc.c +10 -0
+nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+		  struct nfsd4_copy *copy, struct file **src, struct file **dst)
+{
+	__be32 status;
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->save_fh,
+						&copy->cp_src_stateid, RD_STATE,
+						src, NULL);
+	if (status) {
+		dprintk("NFSD: nfsd4_copy: couldn't process src stateid!\n");

fs/nfsd/nfs4proc.c nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, · line 1034

Quoted 10 lines
fs/nfsd/nfs4proc.c +10 -0
+		return status;
+	}
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
+						&copy->cp_dst_stateid, WR_STATE,
+						dst, NULL);
+	if (status) {
+		dprintk("NFSD: nfsd4_copy: couldn't process dst stateid!\n");
+		fput(*src);
+	}

This is missing a return status. On the clone side that caused really hard to debug crashes when xfstests hit this case. While you're at it I'd suggest to also kill the nfsd4_verify_copy heper. You might also need a check for invalid file types that maps to the correct NFS error code, similar to clone.

fs/nfsd/nfs4proc.c nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, · line 1056

Quoted 2 lines
fs/nfsd/nfs4proc.c +2 -0
+	if (bytes < 0)
+		status = nfserrno(bytes);

maybe use a goto here?

fs/nfsd/nfs4proc.c nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, · line 1060

Quoted 4 lines
fs/nfsd/nfs4proc.c +4 -0
+	else {
+		copy->cp_res.wr_bytes_written = bytes;
+		copy->cp_res.wr_stable_how = NFS_FILE_SYNC;
+		copy->cp_consecutive = 1;

is there anything in the linux semantics that guarantees consecutive operation?

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 515

Quoted 12 lines
fs/nfsd/vfs.c +12 -0
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;
+
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);
+	if (bytes > 0)
+		vfs_fsync_range(dst, dst_pos, dst_pos + bytes, 0);
+	return bytes;

How about returning NFS_UNSTABLE above and avoiding the fsync here?

#9 lore raw

Re: [PATCH v1 2/3] NFSD: Implement the COPY call

Quoted 19 lines · Christoph Hellwig
Quoted 19 lines

static __be32

fs/nfsd/nfs4proc.c +18 -0
+nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
+		  struct nfsd4_copy *copy, struct file **src, struct file **dst)
+{
+	__be32 status;
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->save_fh,
+						&copy->cp_src_stateid, RD_STATE,
+						src, NULL);
+	if (status) {
+		dprintk("NFSD: nfsd4_copy: couldn't process src stateid!\n");
+		return status;
+	}
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
+						&copy->cp_dst_stateid, WR_STATE,
+						dst, NULL);
+	if (status) {
+		dprintk("NFSD: nfsd4_copy: couldn't process dst stateid!\n");
Quoted 8 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+		fput(*src);
+	}

This is missing a return status. On the clone side that caused really hard to debug crashes when xfstests hit this case. While you're at it I'd suggest to also kill the nfsd4_verify_copy heper. You might also need a check for invalid file types that maps to the correct NFS error code, similar to clone.

I just updated against the clone code, and I made some of these changes earlier this afternoon. I kept the verify_copy() helper around and changed clone to call it, since all of the stateid verification code would be almost identical.

Quoted 5 lines

fs/nfsd/nfs4proc.c nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, · line 1056

Quoted 3 lines
fs/nfsd/nfs4proc.c +2 -0
+	if (bytes < 0)
+		status = nfserrno(bytes);

maybe use a goto here?

Maybe. I'll see how the code looks!

Quoted 8 lines

fs/nfsd/nfs4proc.c nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, · line 1060

Quoted 5 lines
fs/nfsd/nfs4proc.c +4 -0
+	else {
+		copy->cp_res.wr_bytes_written = bytes;
+		copy->cp_res.wr_stable_how = NFS_FILE_SYNC;
+		copy->cp_consecutive = 1;

is there anything in the linux semantics that guarantees consecutive operation?

I think so. The splice fallback iterates starting at the beginning of the file, so if something goes wrong later on then the earlier pages should at least be copied.

Quoted 15 lines

fs/nfsd/vfs.c __be32 nfsd4_set_nfs4_label(struct svc_rqst *rqstp, struct svc_fh *fhp, · line 515

Quoted 13 lines
fs/nfsd/vfs.c +12 -0
+		       u64 count)
+{
+	ssize_t bytes;
+	u64 limit = 0x10000000;
+
+	if (count > limit)
+		count = limit;
+
+	bytes = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0);
+	if (bytes > 0)
+		vfs_fsync_range(dst, dst_pos, dst_pos + bytes, 0);
+	return bytes;

How about returning NFS_UNSTABLE above and avoiding the fsync here?

I was just looking into this, too. I'm trying to figure out the right way to handle this on the client side, since right now we ignore this value. I have gotten as far as "if the file is open with O_SYNC, then we should commit after copying."

Anna

Quoted 5 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

#10 lore raw

PATCH 3/3 NFS: Add COPY nfs operation

In reply to Anna Schumaker

From: Anna Schumaker <[email protected]>

This adds the copy_range file_ops function pointer used by the sys_copy_range() function call. This patch only implements sync copies, so if an async copy happens we decode the stateid and ignore it.

---
 fs/nfs/nfs42.h            |   1 +
 fs/nfs/nfs42proc.c        |  58 ++++++++++++++++++++
 fs/nfs/nfs42xdr.c         | 136 ++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfs/nfs4file.c         |   8 +++
 fs/nfs/nfs4proc.c         |   1 +
 fs/nfs/nfs4xdr.c          |   1 +
 fs/nfs/objlayout/Makefile |   0
 include/linux/nfs4.h      |   1 +
 include/linux/nfs_fs_sb.h |   1 +
 include/linux/nfs_xdr.h   |  27 +++++++++
 10 files changed, 234 insertions(+)
 create mode 100644 fs/nfs/objlayout/Makefile
fs/nfs/nfs42.h +1 -0
index b587ccd..b6cd153 100644
@@ -13,6 +13,7 @@
 
 /* nfs4.2proc.c */
 int nfs42_proc_allocate(struct file *, loff_t, loff_t);
+ssize_t nfs42_proc_copy(struct file *, loff_t, struct file *, loff_t, size_t);
 int nfs42_proc_deallocate(struct file *, loff_t, loff_t);
 loff_t nfs42_proc_llseek(struct file *, loff_t, int);
 int nfs42_proc_layoutstats_generic(struct nfs_server *,
fs/nfs/nfs42proc.c +58 -0

64 lines not on this page · raw message

fs/nfs/nfs42xdr.c +136 -0

175 lines not on this page · raw message

fs/nfs/nfs4file.c +8 -0
index db9b5fe..8eb7802 100644
@@ -160,6 +160,13 @@ nfs4_file_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 }
 
 #ifdef CONFIG_NFS_V4_2
+static ssize_t nfs4_copy_file_range(struct file *file_in, loff_t pos_in,
+				    struct file *file_out, loff_t pos_out,
+				    size_t count, unsigned int flags)
+{
+	return nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count);
+}
+
 static loff_t nfs4_file_llseek(struct file *filep, loff_t offset, int whence)
 {
 	loff_t ret;
@@ -340,6 +347,7 @@ const struct file_operations nfs4_file_operations = {
 	.check_flags	= nfs_check_flags,
 	.setlease	= simple_nosetlease,
 #ifdef CONFIG_NFS_V4_2
+	.copy_file_range = nfs4_copy_file_range,
 	.llseek		= nfs4_file_llseek,
 	.fallocate	= nfs42_fallocate,
 	.unlocked_ioctl = nfs4_ioctl,
fs/nfs/nfs4proc.c +1 -0
index 8981803..f9d14f1 100644
@@ -8721,6 +8721,7 @@ static const struct nfs4_minor_version_ops nfs_v4_2_minor_ops = {
 		| NFS_CAP_STATEID_NFSV41
 		| NFS_CAP_ATOMIC_OPEN_V1
 		| NFS_CAP_ALLOCATE
+		| NFS_CAP_COPY
 		| NFS_CAP_DEALLOCATE
 		| NFS_CAP_SEEK
 		| NFS_CAP_LAYOUTSTATS
fs/nfs/nfs4xdr.c +1 -0
index 4e44412..bc3a3f5 100644
@@ -7515,6 +7515,7 @@ struct rpc_procinfo	nfs4_procedures[] = {
 	PROC(DEALLOCATE,	enc_deallocate,		dec_deallocate),
 	PROC(LAYOUTSTATS,	enc_layoutstats,	dec_layoutstats),
 	PROC(CLONE,		enc_clone,		dec_clone),
+	PROC(COPY,		enc_copy,		dec_copy),
 #endif /* CONFIG_NFS_V4_2 */
 };
 
fs/nfs/objlayout/Makefile new file +0 -0
new file mode 100644
index 0000000..e69de29
include/linux/nfs4.h +1 -0
index e7e7853..bebafaf 100644
@@ -504,6 +504,7 @@ enum {
 	NFSPROC4_CLNT_DEALLOCATE,
 	NFSPROC4_CLNT_LAYOUTSTATS,
 	NFSPROC4_CLNT_CLONE,
+	NFSPROC4_CLNT_COPY,
 };
 
 /* nfs41 types */
include/linux/nfs_fs_sb.h +1 -0
index 2469ab0..8b772ca 100644
@@ -245,5 +245,6 @@ struct nfs_server {
 #define NFS_CAP_DEALLOCATE	(1U << 21)
 #define NFS_CAP_LAYOUTSTATS	(1U << 22)
 #define NFS_CAP_CLONE		(1U << 23)
+#define NFS_CAP_COPY		(1U << 24)
 
 #endif
include/linux/nfs_xdr.h +27 -0
index 11bbae4..304b8e4 100644
@@ -1342,6 +1342,33 @@ struct nfs42_falloc_res {
 	const struct nfs_server		*falloc_server;
 };
 
+struct nfs42_copy_args {
+	struct nfs4_sequence_args	seq_args;
+
+	struct nfs_fh			*src_fh;
+	nfs4_stateid			src_stateid;
+	u64				src_pos;
+
+	struct nfs_fh			*dst_fh;
+	nfs4_stateid			dst_stateid;
+	u64				dst_pos;
+
+	u64				count;
+};
+
+struct nfs42_write_res {
+	u64		count;
+	u32		committed;
+	nfs4_verifier	verifier;
+};
+
+struct nfs42_copy_res {
+	struct nfs4_sequence_res	seq_res;
+	struct nfs42_write_res		write_res;
+	bool				consecutive;
+	bool				synchronous;
+};
+
 struct nfs42_seek_args {
 	struct nfs4_sequence_args	seq_args;
 
Signature · 1 line
2.6.3
#11 lore raw

Re: [PATCH v1 3/3] NFS: Add COPY nfs operation

Quoted 5 lines · Anna Schumaker

From: Anna Schumaker <[email protected]>

This adds the copy_range file_ops function pointer used by the sys_copy_range() function call. This patch only implements sync copies, so if an async copy happens we decode the stateid and ignore it.

What is the user visible behavior if a server doesn't set cr_synchronous? I can't see any explicit handling of it, which makes me a little sceptic that everything is just going to work fine.

#12 lore raw

[RFC v1 4/3] vfs_copy_range() test program

In reply to Anna Schumaker

This is a simple C program that I used for calling the copy system call.
Usage: ./nfscopy /nfs/original.txt /nfs/copy.txt

---
 nfscopy.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 nfscopy.c
nfscopy.c new file +59 -0
new file mode 100644
index 0000000..3417a14
@@ -0,0 +1,59 @@
+#define _GNU_SOURCE
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+loff_t copy_file_range(int fd_in, loff_t *off_in, int fd_out,
+		       loff_t *off_out, size_t len, unsigned int flags)
+{
+	return syscall(__NR_copy_file_range, fd_in, off_in, fd_out,
+		       off_out, len, flags);
+}
+
+int main(int argc, char **argv)
+{
+	int fd_in, fd_out;
+	struct stat stat;
+	loff_t len, ret;
+	char buf[2];
+
+	if (argc != 3) {
+		fprintf(stderr, "Usage: %s <source> <destination>\n", argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	fd_in = open(argv[1], O_RDONLY);
+	if (fd_in == -1) {
+		perror("open (argv[1])");
+		exit(EXIT_FAILURE);
+	}
+
+	if (fstat(fd_in, &stat) == -1) {
+		perror("fstat");
+		exit(EXIT_FAILURE);
+	}
+	len = stat.st_size;
+
+	fd_out = open(argv[2], O_CREAT|O_WRONLY|O_TRUNC, 0644);
+	if (fd_out == -1) {
+		perror("open (argv[2])");
+		exit(EXIT_FAILURE);
+	}
+
+	do {
+		ret = copy_file_range(fd_in, NULL, fd_out, NULL, len, 0);
+		if (ret == -1) {
+			perror("copy_file_range");
+			exit(EXIT_FAILURE);
+		}
+
+		len -= ret;
+	} while (len > 0);
+
+	close(fd_in);
+	close(fd_out);
+	exit(EXIT_SUCCESS);
+}
Signature · 1 line
2.6.3