NFSD support for asynchronous COPY

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c | 20 ++++++++++++++++++++
 fs/nfsd/nfs4xdr.c  | 30 ++++++++++++++++++++++++++++--
 fs/nfsd/xdr4.h     | 10 ++++++++++
 3 files changed, 58 insertions(+), 2 deletions(-)

fs/nfsd/nfs4proc.c +20 -0

index 3c69db7..8601fc4 100644
@@ -1142,6 +1142,13 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	fput(file);
 	return status;
 }
+static __be32
+nfsd4_offload_status(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	return nfserr_notsupp;
+}
 
 static __be32
 nfsd4_allocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,

@@ -2039,6 +2046,14 @@ static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		1 /* cr_synchronous */) * sizeof(__be32);
 }
 
+static inline u32 nfsd4_offload_status_rsize(struct svc_rqst *rqstp,
+					     struct nfsd4_op *op)
+{
+	return (op_encode_hdr_size +
+		2 /* osr_count */ +
+		1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32);
+}
+
 #ifdef CONFIG_NFSD_PNFS
 static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 {

@@ -2452,6 +2467,11 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		.op_name = "OP_SEEK",
 		.op_rsize_bop = nfsd4_seek_rsize,
 	},
+	[OP_OFFLOAD_STATUS] = {
+		.op_func = nfsd4_offload_status,
+		.op_name = "OP_OFFLOAD_STATUS",
+		.op_rsize_bop = nfsd4_offload_status_rsize,
+	},
 };
 
 /**

fs/nfsd/nfs4xdr.c +8 -1

index 2c61c6b..ed8b61f 100644
@@ -1768,6 +1768,13 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 }
 
 static __be32
+nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp,
+			    struct nfsd4_offload_status *os)
+{
+	return nfsd4_decode_stateid(argp, &os->stateid);
+}
+
+static __be32
 nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek)
 {
 	DECODE_HEAD;

@@ -1874,7 +1881,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 	[OP_LAYOUTERROR]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_notsupp,
-	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_dec)nfsd4_decode_notsupp,

@@ -4216,6 +4223,25 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 }

fs/nfsd/nfs4xdr.c +12 -1

+
+	p = xdr_reserve_space(xdr, 8 + 4);
+	if (!p)
+		return nfserr_resource;
+	p = xdr_encode_hyper(p, os->count);
+	*p++ = cpu_to_be32(0);
+
+	return nfserr;
+}
+
+static __be32
 nfsd4_encode_seek(struct nfsd4_compoundres *resp, __be32 nfserr,
 		  struct nfsd4_seek *seek)
 {

@@ -4318,7 +4344,7 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp,
 	[OP_LAYOUTERROR]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_LAYOUTSTATS]	= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_OFFLOAD_CANCEL]	= (nfsd4_enc)nfsd4_encode_noop,
-	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_noop,
+	[OP_OFFLOAD_STATUS]	= (nfsd4_enc)nfsd4_encode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_enc)nfsd4_encode_noop,
 	[OP_SEEK]		= (nfsd4_enc)nfsd4_encode_seek,
 	[OP_WRITE_SAME]		= (nfsd4_enc)nfsd4_encode_noop,

fs/nfsd/xdr4.h +10 -0

index 4ac2676..9b0c099 100644
@@ -542,6 +542,15 @@ struct nfsd4_seek {
 	loff_t		seek_pos;
 };
 
+struct nfsd4_offload_status {
+	/* request */
+	stateid_t	stateid;
+
+	/* response */
+	u64		count;
+	u32		status;
+};
+
 struct nfsd4_op {
 	int					opnum;
 	const struct nfsd4_operation *		opdesc;

@@ -600,6 +609,7 @@ struct nfsd4_op {
 		struct nfsd4_fallocate		deallocate;
 		struct nfsd4_clone		clone;
 		struct nfsd4_copy		copy;
+		struct nfsd4_offload_status	offload_status;
 		struct nfsd4_seek		seek;
 	} u;
 	struct nfs4_replay *			replay;

Signature · 1 line

1.8.3.1

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c | 13 +++++++++++++
 fs/nfsd/nfs4xdr.c  |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

fs/nfsd/nfs4proc.c +8 -0

index 8601fc4..11ade90 100644
@@ -1121,6 +1121,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 }
 
 static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+		     struct nfsd4_compound_state *cstate,
+		     union nfsd4_op_u *u)
+{
+	return 0;
+}
+
+static __be32
 nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		struct nfsd4_fallocate *fallocate, int flags)
 {

fs/nfsd/nfs4proc.c +4 -0

+		.op_func = nfsd4_offload_cancel,
+		.op_name = "OP_OFFLOAD_CANCEL",
+		.op_rsize_bop = nfsd4_only_status_rsize,
+	},
 };
 
 /**

fs/nfsd/nfs4xdr.c +1 -1

index ed8b61f..31ee5e1 100644
@@ -1880,7 +1880,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
 	[OP_IO_ADVISE]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTERROR]	= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_LAYOUTSTATS]	= (nfsd4_dec)nfsd4_decode_notsupp,
-	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_notsupp,
+	[OP_OFFLOAD_CANCEL]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_OFFLOAD_STATUS]	= (nfsd4_dec)nfsd4_decode_offload_status,
 	[OP_READ_PLUS]		= (nfsd4_dec)nfsd4_decode_notsupp,
 	[OP_SEEK]		= (nfsd4_dec)nfsd4_decode_seek,

Signature · 1 line

1.8.3.1

Quoted 24 lines · Olga Kornievskaia

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c | 13 +++++++++++++
 fs/nfsd/nfs4xdr.c  |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

fs/nfsd/nfs4proc.c +8 -0

index 8601fc4..11ade90 100644
@@ -1121,6 +1121,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 }
 
 static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+                  struct nfsd4_compound_state *cstate,
+                  union nfsd4_op_u *u)
+{
+     return 0;
+}
+
+static __be32
 nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
              struct nfsd4_fallocate *fallocate, int flags)

fs/nfsd/nfs4proc.c static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) · line 2483

Quoted 6 lines

fs/nfsd/nfs4proc.c +1 -0

 {

@@ -2472,6 +2480,11 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
              .op_name = "OP_OFFLOAD_STATUS",
              .op_rsize_bop = nfsd4_offload_status_rsize,
      },
+     [OP_OFFLOAD_CANCEL] = {

I think you want .op_flags = OP_MODIFIES_SOMETHING as well.

Quoted 21 lines

fs/nfsd/nfs4proc.c +4 -0

+             .op_func = nfsd4_offload_cancel,
+             .op_name = "OP_OFFLOAD_CANCEL",
+             .op_rsize_bop = nfsd4_only_status_rsize,
+     },
 };
 
 /**

fs/nfsd/nfs4xdr.c +1 -2

index ed8b61f..31ee5e1 100644
@@ -1880,7 +1880,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
      [OP_IO_ADVISE]          = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_LAYOUTERROR]        = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_LAYOUTSTATS]        = (nfsd4_dec)nfsd4_decode_notsupp,
-     [OP_OFFLOAD_CANCEL]     = (nfsd4_dec)nfsd4_decode_notsupp,
+     [OP_OFFLOAD_CANCEL]     = (nfsd4_dec)nfsd4_decode_offload_status,
      [OP_OFFLOAD_STATUS]     = (nfsd4_dec)nfsd4_decode_offload_status,
      [OP_READ_PLUS]          = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_SEEK]               = (nfsd4_dec)nfsd4_decode_seek,
--

Quoted 2 lines

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 25 lines · J. Bruce Fields

Quoted 24 lines · Olga Kornievskaia

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c | 13 +++++++++++++
 fs/nfsd/nfs4xdr.c  |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

fs/nfsd/nfs4proc.c +8 -0

index 8601fc4..11ade90 100644
@@ -1121,6 +1121,14 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 }
 
 static __be32
+nfsd4_offload_cancel(struct svc_rqst *rqstp,
+                  struct nfsd4_compound_state *cstate,
+                  union nfsd4_op_u *u)
+{
+     return 0;
+}
+
+static __be32
 nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
              struct nfsd4_fallocate *fallocate, int flags)

Quoted 8 lines

fs/nfsd/nfs4proc.c static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) · line 2483

Quoted 6 lines

fs/nfsd/nfs4proc.c +1 -0

 {

@@ -2472,6 +2480,11 @@ static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
              .op_name = "OP_OFFLOAD_STATUS",
              .op_rsize_bop = nfsd4_offload_status_rsize,
      },
+     [OP_OFFLOAD_CANCEL] = {

I think you want .op_flags = OP_MODIFIES_SOMETHING as well.

Got it. Is it because it modifies state of the server? I guess I'm not clear what OP_MODIFIES_SOMETHING represents.

Quoted 29 lines

Quoted 23 lines

fs/nfsd/nfs4proc.c +4 -0

+             .op_func = nfsd4_offload_cancel,
+             .op_name = "OP_OFFLOAD_CANCEL",
+             .op_rsize_bop = nfsd4_only_status_rsize,
+     },
 };
 
 /**

fs/nfsd/nfs4xdr.c +1 -2

index ed8b61f..31ee5e1 100644
@@ -1880,7 +1880,7 @@ static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, str
      [OP_IO_ADVISE]          = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_LAYOUTERROR]        = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_LAYOUTSTATS]        = (nfsd4_dec)nfsd4_decode_notsupp,
-     [OP_OFFLOAD_CANCEL]     = (nfsd4_dec)nfsd4_decode_notsupp,
+     [OP_OFFLOAD_CANCEL]     = (nfsd4_dec)nfsd4_decode_offload_status,
      [OP_OFFLOAD_STATUS]     = (nfsd4_dec)nfsd4_decode_offload_status,
      [OP_READ_PLUS]          = (nfsd4_dec)nfsd4_decode_notsupp,
      [OP_SEEK]               = (nfsd4_dec)nfsd4_decode_seek,
--

Quoted 2 lines

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

In case a failure
happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 164 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   9 ++-
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   7 +++
 4 files changed, 162 insertions(+), 20 deletions(-)

fs/nfsd/nfs4proc.c +145 -19

index 11ade90..de21b1a 100644
@@ -76,6 +76,21 @@
 { }
 #endif
 
+static struct workqueue_struct *copy_wq;
+
+int nfsd4_create_copy_queue(void)
+{
+	copy_wq = create_singlethread_workqueue("nfsd4_copy");
+	if (!copy_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+void nfsd4_destroy_copy_queue(void)
+{
+	destroy_workqueue(copy_wq);
+}
+
 #define NFSDDBG_FACILITY		NFSDDBG_PROC
 
 static u32 nfsd_attrmask[] = {

@@ -1085,37 +1100,148 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+	.release = nfsd4_cb_offload_release,
+	.done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+	copy->cp_consecutive = 1;
+	copy->cp_synchronous = sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied = 0;
+	size_t bytes_total = copy->cp_count;
+	size_t bytes_to_copy;
+	u64 src_pos = copy->cp_src_pos;
+	u64 dst_pos = copy->cp_dst_pos;
+
+	do {
+		bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
+		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+					copy->fh_dst, dst_pos, bytes_to_copy);
+		if (bytes_copied <= 0)
+			break;
+		bytes_total -= bytes_copied;
+		copy->cp_res.wr_bytes_written += bytes_copied;
+		src_pos += bytes_copied;
+		dst_pos += bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes = _nfsd_copy_file_range(copy);
+	if (bytes < 0)
+		status = nfserrno(bytes);
+	else
+		status = nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;
+	dst->cp_consecutive = src->cp_consecutive;
+	dst->cp_synchronous = src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->fh = src->fh;
+	dst->cp_clp = src->cp_clp;
+	dst->fh_src = src->fh_src;
+	dst->fh_dst = src->fh_dst;
+	dst->net = src->net;
+}
+
+static void nfsd4_do_async_copy(struct work_struct *work)
+{
+	struct nfsd4_copy *copy =
+		container_of(work, struct nfsd4_copy, cp_work);
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr = nfsd4_do_copy(copy, 0);
+	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp = copy->cp_clp;
+	cb_copy->nfserr = copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	kfree(copy);
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
 	struct nfsd4_copy *copy = &u->copy;
-	struct file *src, *dst;
 	__be32 status;
-	ssize_t bytes;
 
-	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-				   &copy->cp_dst_stateid, &dst);
+	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);
 	if (status)
 		goto out;
 
-	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-
-	if (bytes < 0)
-		status = nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written = bytes;
-		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
-		copy->cp_synchronous = 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-		status = nfs_ok;
+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	if (!copy->cp_synchronous) {
+		struct nfsd4_copy *async_copy;
+
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {
+			status = nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
+		queue_work(copy_wq, &async_copy->cp_work);
+	} else {
+		status = nfsd4_do_copy(copy, 1);
 	}
-
-	fput(src);
-	fput(dst);
 out:
 	return status;
 }

fs/nfsd/nfs4state.c +8 -1

index 0c04f81..fe6f275 100644
@@ -7033,8 +7033,14 @@ static int nfs4_state_create_net(struct net *net)
 		goto out_free_laundry;
 
 	set_max_delegations();
-	return 0;
 
+	ret = nfsd4_create_copy_queue();
+	if (ret)
+		goto out_free_callback;
+
+	return 0;
+out_free_callback:
+	nfsd4_destroy_callback_queue();
 out_free_laundry:
 	destroy_workqueue(laundry_wq);
 out_cleanup_cred:

@@ -7097,6 +7103,7 @@ static int nfs4_state_create_net(struct net *net)
 	destroy_workqueue(laundry_wq);
 	nfsd4_destroy_callback_queue();
 	cleanup_callback_cred();
+	nfsd4_destroy_copy_queue();
 }
 
 static void

fs/nfsd/state.h +2 -0

index f8b0210..b5358cf 100644
@@ -630,6 +630,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
 							struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
+extern int nfsd4_create_copy_queue(void);
+extern void nfsd4_destroy_copy_queue(void);
 
 struct nfs4_file *find_file(struct knfsd_fh *fh);
 void put_nfs4_file(struct nfs4_file *fi);

fs/nfsd/xdr4.h +7 -0

index 9b0c099..f7a94d3 100644
@@ -529,6 +529,13 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct work_struct      cp_work;
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
 };
 
 struct nfsd4_seek {

Signature · 1 line

1.8.3.1

On Sep 28, 2017, at 2:07 PM, J. Bruce Fields <[email protected]> =

=20

Quoted 8 lines · Olga Kornievskaia

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.
=20
nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

=20
I don't think you need to do this--just call vfs_copy_file_range() directly, as nfsd_copy_file_range does.
=20
The 4MB chunking was only there to prevent a synchronous copy from taking too long, which isn't an issue in the async case. =20

--b.
=20

Quoted 8 lines

In case a failure
happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results. =20

Signed-off-by Olga Kornievskaia <[email protected]>

---
fs/nfsd/nfs4proc.c | 164 =

Quoted 24 lines

fs/nfsd/nfs4state.c |   9 ++-
fs/nfsd/state.h     |   2 +
fs/nfsd/xdr4.h      |   7 +++
4 files changed, 162 insertions(+), 20 deletions(-)

=20

fs/nfsd/nfs4proc.c +0 -0

index 11ade90..de21b1a 100644
@@ -76,6 +76,21 @@
 { }
 #endif

=20

fs/nfsd/nfs4proc.c +11 -0

+static struct workqueue_struct *copy_wq;
+
+int nfsd4_create_copy_queue(void)
+{
+	copy_wq =3D create_singlethread_workqueue("nfsd4_copy");
+	if (!copy_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+void nfsd4_destroy_copy_queue(void)

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec =

Quoted 8 lines

fs/nfsd/nfs4proc.c +4 -0

+{
+	destroy_workqueue(copy_wq);
+}
+

#define NFSDDBG_FACILITY NFSDDBG_PROC
=20
static u32 nfsd_attrmask[] =3D {

fs/nfsd/nfs4proc.c +0 -0

@@ -1085,37 +1100,148 @@ static int fill_in_write_vector(struct kvec =

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)

Quoted 6 lines

out:

	return status;
}

fs/nfsd/nfs4proc.c +3 -0

+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy =3D container_of(cb, struct nfsd4_copy, =

Quoted 30 lines

fs/nfsd/nfs4proc.c +30 -0

+
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops =3D {
+	.release =3D nfsd4_cb_offload_release,
+	.done =3D nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how =3D NFS_UNSTABLE;
+	copy->cp_consecutive =3D 1;
+	copy->cp_synchronous =3D sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied =3D 0;

fs/nfsd/nfs4proc.c

Quoted 8 lines

fs/nfsd/nfs4proc.c +8 -0

+	size_t bytes_total =3D copy->cp_count;
+	size_t bytes_to_copy;
+	u64 src_pos =3D copy->cp_src_pos;
+	u64 dst_pos =3D copy->cp_dst_pos;
+
+	do {
+		bytes_to_copy =3D min_t(u64, bytes_total, MAX_RW_COUNT);
+		bytes_copied =3D nfsd_copy_file_range(copy->fh_src, =

fs/nfsd/nfs4proc.c

Quoted 1 line

fs/nfsd/nfs4proc.c +1 -0

+					copy->fh_dst, dst_pos, =

Quoted 19 lines

fs/nfsd/nfs4proc.c +19 -0

+		if (bytes_copied <=3D 0)
+			break;
+		bytes_total -=3D bytes_copied;
+		copy->cp_res.wr_bytes_written +=3D bytes_copied;
+		src_pos +=3D bytes_copied;
+		dst_pos +=3D bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes =3D _nfsd_copy_file_range(copy);
+	if (bytes < 0)
+		status =3D nfserrno(bytes);
+	else

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)

Quoted 8 lines

fs/nfsd/nfs4proc.c +8 -0

+		status =3D nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct =

fs/nfsd/nfs4proc.c

Quoted 2 lines

fs/nfsd/nfs4proc.c +2 -0

+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, =

fs/nfsd/nfs4proc.c

Quoted 1 line

fs/nfsd/nfs4proc.c +1 -0

+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, =

Quoted 23 lines

fs/nfsd/nfs4proc.c +23 -0

+	dst->cp_src_pos =3D src->cp_src_pos;
+	dst->cp_dst_pos =3D src->cp_dst_pos;
+	dst->cp_count =3D src->cp_count;
+	dst->cp_consecutive =3D src->cp_consecutive;
+	dst->cp_synchronous =3D src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->fh =3D src->fh;
+	dst->cp_clp =3D src->cp_clp;
+	dst->fh_src =3D src->fh_src;
+	dst->fh_dst =3D src->fh_dst;
+	dst->net =3D src->net;
+}
+
+static void nfsd4_do_async_copy(struct work_struct *work)
+{
+	struct nfsd4_copy *copy =3D
+		container_of(work, struct nfsd4_copy, cp_work);
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr =3D nfsd4_do_copy(copy, 0);
+	cb_copy =3D kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)

Quoted 8 lines

fs/nfsd/nfs4proc.c +8 -0

+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp =3D copy->cp_clp;
+	cb_copy->nfserr =3D copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, =

Quoted 7 lines

fs/nfsd/nfs4proc.c +4 -0

+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	kfree(copy);
+}

=20
static __be32
nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state =

fs/nfsd/nfs4proc.c

Quoted 8 lines

union nfsd4_op_u *u)

{
	struct nfsd4_copy *copy =3D &u->copy;

fs/nfsd/nfs4proc.c +0 -1

-	struct file *src, *dst;

	__be32 status;

fs/nfsd/nfs4proc.c +0 -1

-	ssize_t bytes;

=20

fs/nfsd/nfs4proc.c +0 -1

-	status =3D nfsd4_verify_copy(rqstp, cstate, =

fs/nfsd/nfs4proc.c

Quoted 2 lines

fs/nfsd/nfs4proc.c +1 -1

-				   &copy->cp_dst_stateid, &dst);
+	status =3D nfsd4_verify_copy(rqstp, cstate, =

Quoted 8 lines

fs/nfsd/nfs4proc.c +2 -0

+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);

	if (status)
		goto out;

=20

fs/nfsd/nfs4proc.c +0 -3

-	bytes =3D nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-

fs/nfsd/nfs4proc.c

Quoted 8 lines

fs/nfsd/nfs4proc.c +0 -8

-	if (bytes < 0)
-		status =3D nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written =3D bytes;
-		copy->cp_res.wr_stable_how =3D NFS_UNSTABLE;
-		copy->cp_consecutive =3D 1;
-		copy->cp_synchronous =3D 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, =

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)

Quoted 10 lines

fs/nfsd/nfs4proc.c +9 -1

-		status =3D nfs_ok;
+	copy->cp_clp =3D cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net =3D SVC_NET(rqstp);
+	if (!copy->cp_synchronous) {
+		struct nfsd4_copy *async_copy;
+
+		status =3D nfsd4_init_copy_res(copy, 0);
+		async_copy =3D kzalloc(sizeof(struct nfsd4_copy), =

Quoted 15 lines

fs/nfsd/nfs4proc.c +11 -0

+		if (!async_copy) {
+			status =3D nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
+		queue_work(copy_wq, &async_copy->cp_work);
+	} else {
+		status =3D nfsd4_do_copy(copy, 1);

	}

fs/nfsd/nfs4proc.c +0 -3

-
-	fput(src);
-	fput(dst);

fs/nfsd/nfs4state.c static int nfs4_state_create_net(struct net =

Quoted 8 lines

out:

	return status;
}

fs/nfsd/nfs4state.c +0 -0

index 0c04f81..fe6f275 100644
@@ -7033,8 +7033,14 @@ static int nfs4_state_create_net(struct net =

Quoted 8 lines

fs/nfsd/nfs4state.c +0 -0

 		goto out_free_laundry;

=20

	set_max_delegations();

fs/nfsd/nfs4state.c +0 -1

-	return 0;

=20

fs/nfsd/nfs4state.c +3 -0

+	ret =3D nfsd4_create_copy_queue();
+	if (ret)
+		goto out_free_callback;

fs/nfsd/nfs4state.c static int nfs4_state_create_net(struct net =

Quoted 8 lines

fs/nfsd/nfs4state.c +4 -0

+
+	return 0;
+out_free_callback:
+	nfsd4_destroy_callback_queue();

out_free_laundry:

	destroy_workqueue(laundry_wq);

out_cleanup_cred:

fs/nfsd/nfs4state.c +0 -0

@@ -7097,6 +7103,7 @@ static int nfs4_state_create_net(struct net =

Quoted 4 lines

fs/nfsd/nfs4state.c +1 -0

 	destroy_workqueue(laundry_wq);
 	nfsd4_destroy_callback_queue();
 	cleanup_callback_cred();
+	nfsd4_destroy_copy_queue();

fs/nfsd/state.h extern void nfsd4_init_cb(struct nfsd4_callback =

Quoted 8 lines

fs/nfsd/nfs4state.c +0 -0

 }

=20
static void

fs/nfsd/state.h +0 -0

index f8b0210..b5358cf 100644
@@ -630,6 +630,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback =

Quoted 1 line

extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char =

Quoted 1 line

struct nfsd_net =

Quoted 1 line

extern bool nfs4_has_reclaimed_state(const char *name, struct =

Quoted 26 lines

fs/nfsd/state.h +2 -0

+extern int nfsd4_create_copy_queue(void);
+extern void nfsd4_destroy_copy_queue(void);

=20
struct nfs4_file *find_file(struct knfsd_fh *fh);
void put_nfs4_file(struct nfs4_file *fi);

fs/nfsd/xdr4.h +7 -0

index 9b0c099..f7a94d3 100644
@@ -529,6 +529,13 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct work_struct      cp_work;
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
 };

=20
struct nfsd4_seek {
--=20
1.8.3.1
=20

Quoted 16 lines · J. Bruce Fields

Quoted 8 lines · Olga Kornievskaia

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

I don't think you need to do this--just call vfs_copy_file_range() directly, as nfsd_copy_file_range does.

Quoted 3 lines

The 4MB chunking was only there to prevent a synchronous copy from taking too long, which isn't an issue in the async case.

One reason is to allow for the copy to be cancelled in the “middle”. Otherwise, we have no control over once we call into the vfs_copy_file_range to stop the on-going copy. At least this way we check every 4MB chunks.

Quoted 272 lines

--b.

Quoted 269 lines

In case a failure
happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results.

Signed-off-by Olga Kornievskaia <[email protected]>

---

fs/nfsd/nfs4proc.c  | 164 ++++++++++++++++++++++++++++++++++++++++++++++------
fs/nfsd/nfs4state.c |   9 ++-
fs/nfsd/state.h     |   2 +
fs/nfsd/xdr4.h      |   7 +++
4 files changed, 162 insertions(+), 20 deletions(-)

fs/nfsd/nfs4proc.c +15 -0

index 11ade90..de21b1a 100644
@@ -76,6 +76,21 @@
 { }
 #endif
 
+static struct workqueue_struct *copy_wq;
+
+int nfsd4_create_copy_queue(void)
+{
+	copy_wq = create_singlethread_workqueue("nfsd4_copy");
+	if (!copy_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+void nfsd4_destroy_copy_queue(void)
+{
+	destroy_workqueue(copy_wq);
+}
+
 #define NFSDDBG_FACILITY		NFSDDBG_PROC
 
 static u32 nfsd_attrmask[] = {

@@ -1085,37 +1100,148 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)

out:

	return status;
}

fs/nfsd/nfs4proc.c +107 -0

+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+	.release = nfsd4_cb_offload_release,
+	.done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+	copy->cp_consecutive = 1;
+	copy->cp_synchronous = sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied = 0;
+	size_t bytes_total = copy->cp_count;
+	size_t bytes_to_copy;
+	u64 src_pos = copy->cp_src_pos;
+	u64 dst_pos = copy->cp_dst_pos;
+
+	do {
+		bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
+		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+					copy->fh_dst, dst_pos, bytes_to_copy);
+		if (bytes_copied <= 0)
+			break;
+		bytes_total -= bytes_copied;
+		copy->cp_res.wr_bytes_written += bytes_copied;
+		src_pos += bytes_copied;
+		dst_pos += bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes = _nfsd_copy_file_range(copy);
+	if (bytes < 0)
+		status = nfserrno(bytes);
+	else
+		status = nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;
+	dst->cp_consecutive = src->cp_consecutive;
+	dst->cp_synchronous = src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->fh = src->fh;
+	dst->cp_clp = src->cp_clp;
+	dst->fh_src = src->fh_src;
+	dst->fh_dst = src->fh_dst;
+	dst->net = src->net;
+}
+
+static void nfsd4_do_async_copy(struct work_struct *work)
+{
+	struct nfsd4_copy *copy =
+		container_of(work, struct nfsd4_copy, cp_work);
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr = nfsd4_do_copy(copy, 0);
+	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp = copy->cp_clp;
+	cb_copy->nfserr = copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	kfree(copy);
+}

static __be32
nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
union nfsd4_op_u *u)

{
	struct nfsd4_copy *copy = &u->copy;

fs/nfsd/nfs4proc.c +0 -1

-	struct file *src, *dst;

	__be32 status;

fs/nfsd/nfs4proc.c +3 -3

-	ssize_t bytes;
 
-	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-				   &copy->cp_dst_stateid, &dst);
+	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);

	if (status)
		goto out;

fs/nfsd/nfs4proc.c +20 -12

-	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-
-	if (bytes < 0)
-		status = nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written = bytes;
-		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
-		copy->cp_synchronous = 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-		status = nfs_ok;
+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	if (!copy->cp_synchronous) {
+		struct nfsd4_copy *async_copy;
+
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {
+			status = nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
+		queue_work(copy_wq, &async_copy->cp_work);
+	} else {
+		status = nfsd4_do_copy(copy, 1);

	}

fs/nfsd/nfs4proc.c +0 -3

-
-	fput(src);
-	fput(dst);

out:

	return status;
}

fs/nfsd/nfs4state.c +7 -1

index 0c04f81..fe6f275 100644
@@ -7033,8 +7033,14 @@ static int nfs4_state_create_net(struct net *net)
 		goto out_free_laundry;
 
 	set_max_delegations();
-	return 0;
 
+	ret = nfsd4_create_copy_queue();
+	if (ret)
+		goto out_free_callback;
+
+	return 0;
+out_free_callback:
+	nfsd4_destroy_callback_queue();

out_free_laundry:

	destroy_workqueue(laundry_wq);

out_cleanup_cred:

fs/nfsd/nfs4state.c +1 -0

@@ -7097,6 +7103,7 @@ static int nfs4_state_create_net(struct net *net)
 	destroy_workqueue(laundry_wq);
 	nfsd4_destroy_callback_queue();
 	cleanup_callback_cred();
+	nfsd4_destroy_copy_queue();
 }
 

static void

fs/nfsd/state.h +2 -0

index f8b0210..b5358cf 100644
@@ -630,6 +630,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
 							struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
+extern int nfsd4_create_copy_queue(void);
+extern void nfsd4_destroy_copy_queue(void);
 
 struct nfs4_file *find_file(struct knfsd_fh *fh);
 void put_nfs4_file(struct nfs4_file *fi);

fs/nfsd/xdr4.h +7 -0

index 9b0c099..f7a94d3 100644
@@ -529,6 +529,13 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct work_struct      cp_work;
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
 };
 
 struct nfsd4_seek {

Signature · 1 line

1.8.3.1

Quoted 28 lines · J. Bruce Fields

Quoted 16 lines · Olga Kornievskaia

Quoted 16 lines · J. Bruce Fields

Quoted 8 lines · Olga Kornievskaia

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

I don't think you need to do this--just call vfs_copy_file_range() directly, as nfsd_copy_file_range does.

Quoted 7 lines

Quoted 3 lines

The 4MB chunking was only there to prevent a synchronous copy from taking too long, which isn't an issue in the async case.

One reason is to allow for the copy to be cancelled in the “middle”. Otherwise, we have no control over once we call into the vfs_copy_file_range to stop the on-going copy. At least this way we check every 4MB chunks.

Quoted 7 lines

Yes, see my other email, I think we should signal the thread to stop it.

Even with copying in chunks--we'd like to be able to cancel the copy mid-chunk.

Talking this over here with Jeff and Trond.... I don't *think* there's an easy way to cancel a long-running work item.

Yes I believe I was looking into signaling but couldn’t figure out how to do it with the current code...

Quoted 3 lines

So probably you want to create your own thread for the copy instead. It looks like kthread_create/kthread_stop/kthread_should_stop are what you want? You can see some examples if you "git grep kthread net/sunrpc”.

Ok so should I instead create a dedicated thread to do all copies and have a way to give this thread work? Or are you ok with creating a thread for each individual copy which would be much simpler?

Quoted 2 lines

Quoted 2 lines

On Sep 28, 2017, at 2:55 PM, J. Bruce Fields <[email protected]> wrot=

Quoted 4 lines

Quoted 4 lines · Olga Kornievskaia

Quoted 2 lines

Quoted 2 lines

On Sep 28, 2017, at 2:07 PM, J. Bruce Fields <[email protected]> wr=

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines · Olga Kornievskaia

Quoted 8 lines

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 6 lines

I don't think you need to do this--just call vfs_copy_file_range() directly, as nfsd_copy_file_range does.

The 4MB chunking was only there to prevent a synchronous copy from taking too long, which isn't an issue in the async case.

One reason is to allow for the copy to be cancelled in the =E2=80=9Cm=

Quoted 1 line

Quoted 1 line

Quoted 1 line

Otherwise, we have no control over once we call into the vfs_copy_fil=

Quoted 1 line

Quoted 1 line

Quoted 1 line

to stop the on-going copy. At least this way we check every 4MB chunk=

Quoted 2 lines

Quoted 2 lines

Yes, see my other email, I think we should signal the thread to stop i=

Quoted 8 lines

Quoted 6 lines

Even with copying in chunks--we'd like to be able to cancel the copy mid-chunk.

Talking this over here with Jeff and Trond.... I don't *think* there's an easy way to cancel a long-running work item.

Yes I believe I was looking into signaling but couldn=E2=80=99t figure o=

Quoted 3 lines

with the current code...

Quoted 1 line

So probably you want to create your own thread for the copy instead. =

Quoted 1 line

Quoted 1 line

looks like kthread_create/kthread_stop/kthread_should_stop are what yo=

Quoted 1 line

Quoted 1 line

want? You can see some examples if you "git grep kthread net/sunrpc=

Quoted 2 lines

Ok so should I instead create a dedicated thread to do all copies and ha=

Quoted 2 lines

a way to give this thread work? Or are you ok with creating a thread for each individual copy which would be much simpler?

I'd be in favor of a thread per copy. That should make the cancelling either as well.

Quoted 6 lines · Olga Kornievskaia

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.

Quoted 2 lines

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

I don't think you need to do this--just call vfs_copy_file_range() directly, as nfsd_copy_file_range does.

The 4MB chunking was only there to prevent a synchronous copy from taking too long, which isn't an issue in the async case.

--b.

Quoted 267 lines

In case a failure
happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 164 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   9 ++-
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   7 +++
 4 files changed, 162 insertions(+), 20 deletions(-)

fs/nfsd/nfs4proc.c +145 -19

index 11ade90..de21b1a 100644
@@ -76,6 +76,21 @@
 { }
 #endif
 
+static struct workqueue_struct *copy_wq;
+
+int nfsd4_create_copy_queue(void)
+{
+     copy_wq = create_singlethread_workqueue("nfsd4_copy");
+     if (!copy_wq)
+             return -ENOMEM;
+     return 0;
+}
+
+void nfsd4_destroy_copy_queue(void)
+{
+     destroy_workqueue(copy_wq);
+}
+
 #define NFSDDBG_FACILITY             NFSDDBG_PROC
 
 static u32 nfsd_attrmask[] = {

@@ -1085,37 +1100,148 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
      return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+     struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+     kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+                              struct rpc_task *task)
+{
+     return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+     .release = nfsd4_cb_offload_release,
+     .done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+             sizeof(copy->cp_dst_stateid));
+     copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+     copy->cp_consecutive = 1;
+     copy->cp_synchronous = sync;
+     gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+     return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+     ssize_t bytes_copied = 0;
+     size_t bytes_total = copy->cp_count;
+     size_t bytes_to_copy;
+     u64 src_pos = copy->cp_src_pos;
+     u64 dst_pos = copy->cp_dst_pos;
+
+     do {
+             bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
+             bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+                                     copy->fh_dst, dst_pos, bytes_to_copy);
+             if (bytes_copied <= 0)
+                     break;
+             bytes_total -= bytes_copied;
+             copy->cp_res.wr_bytes_written += bytes_copied;
+             src_pos += bytes_copied;
+             dst_pos += bytes_copied;
+     } while (bytes_total > 0 && !copy->cp_synchronous);
+     return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+     __be32 status;
+     ssize_t bytes;
+
+     bytes = _nfsd_copy_file_range(copy);
+     if (bytes < 0)
+             status = nfserrno(bytes);
+     else
+             status = nfsd4_init_copy_res(copy, sync);
+
+     fput(copy->fh_src);
+     fput(copy->fh_dst);
+     return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+     dst->cp_src_pos = src->cp_src_pos;
+     dst->cp_dst_pos = src->cp_dst_pos;
+     dst->cp_count = src->cp_count;
+     dst->cp_consecutive = src->cp_consecutive;
+     dst->cp_synchronous = src->cp_synchronous;
+     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+     /* skipping nfsd4_callback */
+     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+     dst->fh = src->fh;
+     dst->cp_clp = src->cp_clp;
+     dst->fh_src = src->fh_src;
+     dst->fh_dst = src->fh_dst;
+     dst->net = src->net;
+}
+
+static void nfsd4_do_async_copy(struct work_struct *work)
+{
+     struct nfsd4_copy *copy =
+             container_of(work, struct nfsd4_copy, cp_work);
+     struct nfsd4_copy *cb_copy;
+
+     copy->nfserr = nfsd4_do_copy(copy, 0);
+     cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+     if (!cb_copy)
+             goto out;
+     memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+     cb_copy->cp_clp = copy->cp_clp;
+     cb_copy->nfserr = copy->nfserr;
+     memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+     nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+                     &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+     nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+     kfree(copy);
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
              union nfsd4_op_u *u)
 {
      struct nfsd4_copy *copy = &u->copy;
-     struct file *src, *dst;
      __be32 status;
-     ssize_t bytes;
 
-     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-                                &copy->cp_dst_stateid, &dst);
+     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+                                &copy->fh_src, &copy->cp_dst_stateid,
+                                &copy->fh_dst);
      if (status)
              goto out;
 
-     bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-                     dst, copy->cp_dst_pos, copy->cp_count);
-
-     if (bytes < 0)
-             status = nfserrno(bytes);
-     else {
-             copy->cp_res.wr_bytes_written = bytes;
-             copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-             copy->cp_consecutive = 1;
-             copy->cp_synchronous = 1;
-             gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-             status = nfs_ok;
+     copy->cp_clp = cstate->clp;
+     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+             sizeof(struct knfsd_fh));
+     copy->net = SVC_NET(rqstp);
+     if (!copy->cp_synchronous) {
+             struct nfsd4_copy *async_copy;
+
+             status = nfsd4_init_copy_res(copy, 0);
+             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+             if (!async_copy) {
+                     status = nfserrno(-ENOMEM);
+                     goto out;
+             }
+             dup_copy_fields(copy, async_copy);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+                     sizeof(copy->cp_dst_stateid));
+             INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
+             queue_work(copy_wq, &async_copy->cp_work);
+     } else {
+             status = nfsd4_do_copy(copy, 1);
      }
-
-     fput(src);
-     fput(dst);
 out:
      return status;
 }

fs/nfsd/nfs4state.c +8 -1

index 0c04f81..fe6f275 100644
@@ -7033,8 +7033,14 @@ static int nfs4_state_create_net(struct net *net)
              goto out_free_laundry;
 
      set_max_delegations();
-     return 0;
 
+     ret = nfsd4_create_copy_queue();
+     if (ret)
+             goto out_free_callback;
+
+     return 0;
+out_free_callback:
+     nfsd4_destroy_callback_queue();
 out_free_laundry:
      destroy_workqueue(laundry_wq);
 out_cleanup_cred:

@@ -7097,6 +7103,7 @@ static int nfs4_state_create_net(struct net *net)
      destroy_workqueue(laundry_wq);
      nfsd4_destroy_callback_queue();
      cleanup_callback_cred();
+     nfsd4_destroy_copy_queue();
 }
 
 static void

fs/nfsd/state.h +2 -0

index f8b0210..b5358cf 100644
@@ -630,6 +630,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
                                                      struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
+extern int nfsd4_create_copy_queue(void);
+extern void nfsd4_destroy_copy_queue(void);
 
 struct nfs4_file *find_file(struct knfsd_fh *fh);
 void put_nfs4_file(struct nfs4_file *fi);

fs/nfsd/xdr4.h +7 -1

index 9b0c099..f7a94d3 100644
@@ -529,6 +529,13 @@ struct nfsd4_copy {
      struct nfsd4_callback   cp_cb;
      __be32                  nfserr;
      struct knfsd_fh         fh;
+
+     struct work_struct      cp_work;
+     struct nfs4_client      *cp_clp;
+
+     struct file             *fh_src;
+     struct file             *fh_dst;
+     struct net              *net;
 };
 
 struct nfsd4_seek {
--

Quoted 2 lines

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 7 lines · J. Bruce Fields

Quoted 6 lines · Olga Kornievskaia

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then queue work and reply back to the client indicating copy is asynchronous.

Quoted 8 lines

Quoted 2 lines

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy.

I don't think you need to do this--just call vfs_copy_file_range() directly, as nfsd_copy_file_range does.

The 4MB chunking was only there to prevent a synchronous copy from taking too long, which isn't an issue in the async case.

Before I remove the loop, the other reason I kept the loop was for a meaningful OFFLOAD_STATUS. Without the loop, any query for the OFFLOAD_STATUS would either return 0 or if it's really really really really lucky then it might return full-bytes copy value. There will never be an intermediate reply. I kept the loop and kept track of bytes copied so far for the status query.

Quoted 277 lines

--b.

Quoted 267 lines

In case a failure
happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 164 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   9 ++-
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   7 +++
 4 files changed, 162 insertions(+), 20 deletions(-)

fs/nfsd/nfs4proc.c +145 -19

index 11ade90..de21b1a 100644
@@ -76,6 +76,21 @@
 { }
 #endif
 
+static struct workqueue_struct *copy_wq;
+
+int nfsd4_create_copy_queue(void)
+{
+     copy_wq = create_singlethread_workqueue("nfsd4_copy");
+     if (!copy_wq)
+             return -ENOMEM;
+     return 0;
+}
+
+void nfsd4_destroy_copy_queue(void)
+{
+     destroy_workqueue(copy_wq);
+}
+
 #define NFSDDBG_FACILITY             NFSDDBG_PROC
 
 static u32 nfsd_attrmask[] = {

@@ -1085,37 +1100,148 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
      return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+     struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+     kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+                              struct rpc_task *task)
+{
+     return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+     .release = nfsd4_cb_offload_release,
+     .done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+             sizeof(copy->cp_dst_stateid));
+     copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+     copy->cp_consecutive = 1;
+     copy->cp_synchronous = sync;
+     gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+     return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+     ssize_t bytes_copied = 0;
+     size_t bytes_total = copy->cp_count;
+     size_t bytes_to_copy;
+     u64 src_pos = copy->cp_src_pos;
+     u64 dst_pos = copy->cp_dst_pos;
+
+     do {
+             bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
+             bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+                                     copy->fh_dst, dst_pos, bytes_to_copy);
+             if (bytes_copied <= 0)
+                     break;
+             bytes_total -= bytes_copied;
+             copy->cp_res.wr_bytes_written += bytes_copied;
+             src_pos += bytes_copied;
+             dst_pos += bytes_copied;
+     } while (bytes_total > 0 && !copy->cp_synchronous);
+     return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+     __be32 status;
+     ssize_t bytes;
+
+     bytes = _nfsd_copy_file_range(copy);
+     if (bytes < 0)
+             status = nfserrno(bytes);
+     else
+             status = nfsd4_init_copy_res(copy, sync);
+
+     fput(copy->fh_src);
+     fput(copy->fh_dst);
+     return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+     dst->cp_src_pos = src->cp_src_pos;
+     dst->cp_dst_pos = src->cp_dst_pos;
+     dst->cp_count = src->cp_count;
+     dst->cp_consecutive = src->cp_consecutive;
+     dst->cp_synchronous = src->cp_synchronous;
+     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+     /* skipping nfsd4_callback */
+     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+     dst->fh = src->fh;
+     dst->cp_clp = src->cp_clp;
+     dst->fh_src = src->fh_src;
+     dst->fh_dst = src->fh_dst;
+     dst->net = src->net;
+}
+
+static void nfsd4_do_async_copy(struct work_struct *work)
+{
+     struct nfsd4_copy *copy =
+             container_of(work, struct nfsd4_copy, cp_work);
+     struct nfsd4_copy *cb_copy;
+
+     copy->nfserr = nfsd4_do_copy(copy, 0);
+     cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+     if (!cb_copy)
+             goto out;
+     memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+     cb_copy->cp_clp = copy->cp_clp;
+     cb_copy->nfserr = copy->nfserr;
+     memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+     nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+                     &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+     nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+     kfree(copy);
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
              union nfsd4_op_u *u)
 {
      struct nfsd4_copy *copy = &u->copy;
-     struct file *src, *dst;
      __be32 status;
-     ssize_t bytes;
 
-     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-                                &copy->cp_dst_stateid, &dst);
+     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+                                &copy->fh_src, &copy->cp_dst_stateid,
+                                &copy->fh_dst);
      if (status)
              goto out;
 
-     bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-                     dst, copy->cp_dst_pos, copy->cp_count);
-
-     if (bytes < 0)
-             status = nfserrno(bytes);
-     else {
-             copy->cp_res.wr_bytes_written = bytes;
-             copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-             copy->cp_consecutive = 1;
-             copy->cp_synchronous = 1;
-             gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-             status = nfs_ok;
+     copy->cp_clp = cstate->clp;
+     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+             sizeof(struct knfsd_fh));
+     copy->net = SVC_NET(rqstp);
+     if (!copy->cp_synchronous) {
+             struct nfsd4_copy *async_copy;
+
+             status = nfsd4_init_copy_res(copy, 0);
+             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+             if (!async_copy) {
+                     status = nfserrno(-ENOMEM);
+                     goto out;
+             }
+             dup_copy_fields(copy, async_copy);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+                     sizeof(copy->cp_dst_stateid));
+             INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
+             queue_work(copy_wq, &async_copy->cp_work);
+     } else {
+             status = nfsd4_do_copy(copy, 1);
      }
-
-     fput(src);
-     fput(dst);
 out:
      return status;
 }

fs/nfsd/nfs4state.c +8 -1

index 0c04f81..fe6f275 100644
@@ -7033,8 +7033,14 @@ static int nfs4_state_create_net(struct net *net)
              goto out_free_laundry;
 
      set_max_delegations();
-     return 0;
 
+     ret = nfsd4_create_copy_queue();
+     if (ret)
+             goto out_free_callback;
+
+     return 0;
+out_free_callback:
+     nfsd4_destroy_callback_queue();
 out_free_laundry:
      destroy_workqueue(laundry_wq);
 out_cleanup_cred:

@@ -7097,6 +7103,7 @@ static int nfs4_state_create_net(struct net *net)
      destroy_workqueue(laundry_wq);
      nfsd4_destroy_callback_queue();
      cleanup_callback_cred();
+     nfsd4_destroy_copy_queue();
 }
 
 static void

fs/nfsd/state.h +2 -0

index f8b0210..b5358cf 100644
@@ -630,6 +630,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
                                                      struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
+extern int nfsd4_create_copy_queue(void);
+extern void nfsd4_destroy_copy_queue(void);
 
 struct nfs4_file *find_file(struct knfsd_fh *fh);
 void put_nfs4_file(struct nfs4_file *fi);

fs/nfsd/xdr4.h +7 -1

index 9b0c099..f7a94d3 100644
@@ -529,6 +529,13 @@ struct nfsd4_copy {
      struct nfsd4_callback   cp_cb;
      __be32                  nfserr;
      struct knfsd_fh         fh;
+
+     struct work_struct      cp_work;
+     struct nfs4_client      *cp_clp;
+
+     struct file             *fh_src;
+     struct file             *fh_dst;
+     struct net              *net;
 };
 
 struct nfsd4_seek {
--

Quoted 2 lines

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Asynchronous copies are handled by a single threaded workqueue. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy.

Then
queue work and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in 4MBchunk so do a loop over the total number of bytes need to copy. In case a failure happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 164 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   9 ++-
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   7 +++
 4 files changed, 162 insertions(+), 20 deletions(-)

fs/nfsd/nfs4proc.c +145 -19

index 11ade90..de21b1a 100644
@@ -76,6 +76,21 @@
 { }
 #endif
 
+static struct workqueue_struct *copy_wq;
+
+int nfsd4_create_copy_queue(void)
+{
+	copy_wq = create_singlethread_workqueue("nfsd4_copy");
+	if (!copy_wq)
+		return -ENOMEM;
+	return 0;
+}
+
+void nfsd4_destroy_copy_queue(void)
+{
+	destroy_workqueue(copy_wq);
+}
+
 #define NFSDDBG_FACILITY		NFSDDBG_PROC
 
 static u32 nfsd_attrmask[] = {

@@ -1085,37 +1100,148 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+	.release = nfsd4_cb_offload_release,
+	.done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+	copy->cp_consecutive = 1;
+	copy->cp_synchronous = sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied = 0;
+	size_t bytes_total = copy->cp_count;
+	size_t bytes_to_copy;
+	u64 src_pos = copy->cp_src_pos;
+	u64 dst_pos = copy->cp_dst_pos;
+
+	do {
+		bytes_to_copy = min_t(u64, bytes_total, MAX_RW_COUNT);
+		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+					copy->fh_dst, dst_pos, bytes_to_copy);
+		if (bytes_copied <= 0)
+			break;
+		bytes_total -= bytes_copied;
+		copy->cp_res.wr_bytes_written += bytes_copied;
+		src_pos += bytes_copied;
+		dst_pos += bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes = _nfsd_copy_file_range(copy);
+	if (bytes < 0)
+		status = nfserrno(bytes);
+	else
+		status = nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;
+	dst->cp_consecutive = src->cp_consecutive;
+	dst->cp_synchronous = src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->fh = src->fh;
+	dst->cp_clp = src->cp_clp;
+	dst->fh_src = src->fh_src;
+	dst->fh_dst = src->fh_dst;
+	dst->net = src->net;
+}
+
+static void nfsd4_do_async_copy(struct work_struct *work)
+{
+	struct nfsd4_copy *copy =
+		container_of(work, struct nfsd4_copy, cp_work);
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr = nfsd4_do_copy(copy, 0);
+	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp = copy->cp_clp;
+	cb_copy->nfserr = copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	kfree(copy);
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
 	struct nfsd4_copy *copy = &u->copy;
-	struct file *src, *dst;
 	__be32 status;
-	ssize_t bytes;
 
-	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-				   &copy->cp_dst_stateid, &dst);
+	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);
 	if (status)
 		goto out;
 
-	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-
-	if (bytes < 0)
-		status = nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written = bytes;
-		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
-		copy->cp_synchronous = 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-		status = nfs_ok;
+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	if (!copy->cp_synchronous) {
+		struct nfsd4_copy *async_copy;
+
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {
+			status = nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
+		queue_work(copy_wq, &async_copy->cp_work);
+	} else {
+		status = nfsd4_do_copy(copy, 1);
 	}
-
-	fput(src);
-	fput(dst);
 out:
 	return status;
 }

fs/nfsd/nfs4state.c +8 -1

index 0c04f81..fe6f275 100644
@@ -7033,8 +7033,14 @@ static int nfs4_state_create_net(struct net *net)
 		goto out_free_laundry;
 
 	set_max_delegations();
-	return 0;
 
+	ret = nfsd4_create_copy_queue();
+	if (ret)
+		goto out_free_callback;
+
+	return 0;
+out_free_callback:
+	nfsd4_destroy_callback_queue();
 out_free_laundry:
 	destroy_workqueue(laundry_wq);
 out_cleanup_cred:

@@ -7097,6 +7103,7 @@ static int nfs4_state_create_net(struct net *net)
 	destroy_workqueue(laundry_wq);
 	nfsd4_destroy_callback_queue();
 	cleanup_callback_cred();
+	nfsd4_destroy_copy_queue();
 }
 
 static void

fs/nfsd/state.h +2 -0

index f8b0210..b5358cf 100644
@@ -630,6 +630,8 @@ extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(const char *name,
 							struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(const char *name, struct nfsd_net *nn);
+extern int nfsd4_create_copy_queue(void);
+extern void nfsd4_destroy_copy_queue(void);
 
 struct nfs4_file *find_file(struct knfsd_fh *fh);
 void put_nfs4_file(struct nfs4_file *fi);

fs/nfsd/xdr4.h +7 -0

index 9b0c099..f7a94d3 100644
@@ -529,6 +529,13 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct work_struct      cp_work;
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
 };
 
 struct nfsd4_seek {

Signature · 1 line

1.8.3.1

Generate a new stateid to be used for reply to the asynchronous COPY (this would also be used later by COPY_NOTIFY as well). Associate the stateid with the parent OPEN/LOCK/DELEG stateid that can be freed during the free of the parent stateid. However, right now deciding to bind the lifetime to when the vfs copy is done. This way don't need to keep the nfsd_net structure for the callback. The drawback is that time copy state information is available for query by OFFLOAD_STATUS is slightly less.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 31 ++++++++++++++++++-------
 fs/nfsd/nfs4state.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 ++++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 114 insertions(+), 8 deletions(-)

fs/nfsd/netns.h +8 -0

index 3714231..2c88a95 100644
@@ -119,6 +119,14 @@ struct nfsd_net {
 	u32 clverifier_counter;
 
 	struct svc_serv *nfsd_serv;
+
+	/*
+	 * clientid and stateid data for construction of net unique COPY
+	 * stateids.
+	 */
+	u32		s2s_cp_cl_id;
+	struct idr	s2s_cp_stateids;
+	spinlock_t	s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */

fs/nfsd/nfs4proc.c +23 -8

index 044af6b..3cddebb 100644
@@ -1046,7 +1046,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		  stateid_t *src_stateid, struct file **src,
-		  stateid_t *dst_stateid, struct file **dst)
+		  stateid_t *dst_stateid, struct file **dst,
+		  struct nfs4_stid **stid)
 {
 	__be32 status;
 

@@ -1060,7 +1061,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    dst_stateid, WR_STATE, dst, NULL,
-					    NULL);
+					    stid);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
 		goto out_put_src;

@@ -1091,7 +1092,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	__be32 status;
 
 	status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-				   &clone->cl_dst_stateid, &dst);
+				   &clone->cl_dst_stateid, &dst, NULL);
 	if (status)
 		goto out;
 

@@ -1123,8 +1124,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-		sizeof(copy->cp_dst_stateid));
 	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
 	copy->cp_consecutive = 1;
 	copy->cp_synchronous = sync;

@@ -1188,6 +1187,8 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
 	dst->fh_src = src->fh_src;
 	dst->fh_dst = src->fh_dst;
 	dst->net = src->net;
+	dst->stid = src->stid;
+	dst->cps = src->cps;
 }
 
 static void nfsd4_do_async_copy(struct work_struct *work)

@@ -1208,6 +1209,9 @@ static void nfsd4_do_async_copy(struct work_struct *work)
 			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
 	nfsd4_run_cb(&cb_copy->cp_cb);
 out:
+	list_del(&copy->cps->cp_list);
+	nfs4_free_cp_state(copy->cps);
+	nfs4_put_stid(copy->stid);
 	kfree(copy);
 }
 

@@ -1220,7 +1224,7 @@ static void nfsd4_do_async_copy(struct work_struct *work)
 
 	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
 				   &copy->fh_src, &copy->cp_dst_stateid,
-				   &copy->fh_dst);
+				   &copy->fh_dst, &copy->stid);
 	if (status)
 		goto out;
 

@@ -1229,6 +1233,7 @@ static void nfsd4_do_async_copy(struct work_struct *work)
 		sizeof(struct knfsd_fh));
 	copy->net = SVC_NET(rqstp);
 	if (!copy->cp_synchronous) {
+		struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 		struct nfsd4_copy *async_copy;
 
 		status = nfsd4_init_copy_res(copy, 0);

@@ -1237,9 +1242,19 @@ static void nfsd4_do_async_copy(struct work_struct *work)
 			status = nfserrno(-ENOMEM);
 			goto out;
 		}
+		copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+		if (!copy->cps) {
+			status = nfserrno(-ENOMEM);
+			kfree(async_copy);
+			goto out;
+		}
+		/* take a reference on the parent stateid so it's not
+		 * not freed by the copy compound
+		 */
+		atomic_inc(&copy->stid->sc_count);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+			sizeof(copy->cps->cp_stateid));
 		dup_copy_fields(copy, async_copy);
-		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-			sizeof(copy->cp_dst_stateid));
 		INIT_WORK(&async_copy->cp_work, nfsd4_do_async_copy);
 		queue_work(copy_wq, &async_copy->cp_work);
 	} else {

fs/nfsd/nfs4state.c +66 -0

index 2c89dae..be59baf 100644
@@ -658,6 +658,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	/* Will be incremented before return to client: */
 	atomic_set(&stid->sc_count, 1);
 	spin_lock_init(&stid->sc_lock);
+	INIT_LIST_HEAD(&stid->sc_cp_list);
 
 	/*
 	 * It shouldn't be a problem to reuse an opaque stateid value.

@@ -674,6 +675,66 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+					       struct nfs4_stid *p_stid)
+{
+	struct nfs4_cp_state *cps;
+	int new_id;
+
+	cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+	if (!cps)
+		return NULL;
+	idr_preload(GFP_KERNEL);
+	spin_lock(&nn->s2s_cp_lock);
+	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+	spin_unlock(&nn->s2s_cp_lock);
+	idr_preload_end();
+	if (new_id < 0)
+		goto out_free;
+	cps->cp_stateid.si_opaque.so_id = new_id;
+	cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+	cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+	cps->cp_p_stid = p_stid;
+	INIT_LIST_HEAD(&cps->cp_list);
+	list_add(&cps->cp_list, &p_stid->sc_cp_list);
+
+	return cps;
+out_free:
+	kfree(cps);
+	return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+	struct nfsd_net *nn;
+
+	nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+	spin_lock(&nn->s2s_cp_lock);
+	idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+	spin_unlock(&nn->s2s_cp_lock);
+
+	kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+	struct nfs4_cp_state *cps;
+
+	might_sleep();
+
+	while (!list_empty(&stid->sc_cp_list)) {
+		cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+				       cp_list);
+		list_del(&cps->cp_list);
+		nfs4_free_cp_state(cps);
+	}
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
 	struct nfs4_stid *stid;

@@ -819,6 +880,9 @@ static void block_delegations(struct knfsd_fh *fh)
 	}
 	idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
 	spin_unlock(&clp->cl_lock);
+
+	nfs4_free_cp_statelist(s);
+
 	s->sc_free(s);
 	if (fp)
 		put_nfs4_file(fp);

@@ -6952,6 +7016,8 @@ static int nfs4_state_create_net(struct net *net)
 	INIT_LIST_HEAD(&nn->close_lru);
 	INIT_LIST_HEAD(&nn->del_recall_lru);
 	spin_lock_init(&nn->client_lock);
+	spin_lock_init(&nn->s2s_cp_lock);
+	idr_init(&nn->s2s_cp_stateids);
 
 	spin_lock_init(&nn->blocked_locks_lock);
 	INIT_LIST_HEAD(&nn->blocked_locks_lru);

fs/nfsd/nfsctl.c +1 -0

index 6493df6..232270d 100644
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
 	nn->nfsd4_grace = 90;
 	nn->clverifier_counter = prandom_u32();
 	nn->clientid_counter = prandom_u32();
+	nn->s2s_cp_cl_id = nn->clientid_counter++;
 	return 0;
 
 out_idmap_error:

fs/nfsd/state.h +14 -0

index cfd13f0..8724955 100644
@@ -93,6 +93,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+	struct list_head	sc_cp_list;
 	unsigned char		sc_type;
 	stateid_t		sc_stateid;
 	spinlock_t		sc_lock;

@@ -102,6 +103,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+	stateid_t		cp_stateid;
+	struct list_head	cp_list;	/* per parent nfs4_stid */
+	struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:

@@ -607,6 +619,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
 				  void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);

fs/nfsd/xdr4.h +2 -0

index f7a94d3..4ccd43b 100644
@@ -536,6 +536,8 @@ struct nfsd4_copy {
 	struct file             *fh_src;
 	struct file             *fh_dst;
 	struct net              *net;
+	struct nfs4_stid        *stid;
+	struct nfs4_cp_state	*cps;
 };
 
 struct nfsd4_seek {

Signature · 1 line

1.8.3.1

Quoted 8 lines

Generate a new stateid to be used for reply to the asynchronous COPY (this would also be used later by COPY_NOTIFY as well). Associate the stateid with the parent OPEN/LOCK/DELEG stateid that can be freed during the free of the parent stateid. However, right now deciding to bind the lifetime to when the vfs copy is done. This way don't need to keep the nfsd_net structure for the callback. The drawback is that time copy state information is available for query by OFFLOAD_STATUS is slightly less.

I don't think we're under any obligation to handle OFFLOAD_STATUS after the copy is done, are we?

There's a possibility that OFFLOAD_STATUS might arrive just as the copy finishes, and the client might even received the failed response to OFFLOAD_STATUS before it receives the final CB_OFFLOAD. I think it's up to the client to handle that situation.

--b.

Quoted 6 lines

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 31 ++++++++++++++++++-------

fs/nfsd/nfs4state.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++=

Quoted 23 lines

 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 ++++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 114 insertions(+), 8 deletions(-)

fs/nfsd/netns.h +8 -0

index 3714231..2c88a95 100644
@@ -119,6 +119,14 @@ struct nfsd_net {
      u32 clverifier_counter;
 
      struct svc_serv *nfsd_serv;
+
+     /*
+      * clientid and stateid data for construction of net unique COPY
+      * stateids.
+      */
+     u32             s2s_cp_cl_id;
+     struct idr      s2s_cp_stateids;
+     spinlock_t      s2s_cp_lock;
 };
 

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, =

Quoted 6 lines

fs/nfsd/netns.h +0 -0

 /* Simple check to find out if a given net was properly initialized */

fs/nfsd/nfs4proc.c +0 -0

index 044af6b..3cddebb 100644
@@ -1046,7 +1046,8 @@ static int fill_in_write_vector(struct kvec *vec, =