NFSD support for asynchronous COPY

Upon receiving a request for async copy, create a new kthread. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then start the thread and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in a loop over the total number of bytes is needed to copy. In case a failure happens in the middle, we ignore the error and return how much we copied so far.

Once done creating a workitem for the callback
workqueue and send CB_OFFLOAD with the results.

In the following patches the unique stateid will be generated for the async COPY to return, at that point will enable the async feature.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 176 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   2 +
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   9 +++
 4 files changed, 170 insertions(+), 19 deletions(-)

fs/nfsd/nfs4proc.c +157 -19

index e11494f..91d1544 100644
@@ -35,6 +35,7 @@
 #include <linux/file.h>
 #include <linux/falloc.h>
 #include <linux/slab.h>
+#include <linux/kthread.h>
 
 #include "idmap.h"
 #include "cache.h"

@@ -1083,39 +1084,176 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
 	return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+	struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+				 struct rpc_task *task)
+{
+	return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+	.release = nfsd4_cb_offload_release,
+	.done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+		sizeof(copy->cp_dst_stateid));
+	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+	copy->cp_consecutive = 1;
+	copy->cp_synchronous = sync;
+	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+	return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+	ssize_t bytes_copied = 0;
+	size_t bytes_total = copy->cp_count;
+	u64 src_pos = copy->cp_src_pos;
+	u64 dst_pos = copy->cp_dst_pos;
+
+	do {
+		bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+				copy->fh_dst, dst_pos, bytes_total);
+		if (bytes_copied <= 0)
+			break;
+		bytes_total -= bytes_copied;
+		copy->cp_res.wr_bytes_written += bytes_copied;
+		src_pos += bytes_copied;
+		dst_pos += bytes_copied;
+	} while (bytes_total > 0 && !copy->cp_synchronous);
+	return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+	__be32 status;
+	ssize_t bytes;
+
+	bytes = _nfsd_copy_file_range(copy);
+	if (bytes < 0 && !copy->cp_res.wr_bytes_written)
+		status = nfserrno(bytes);
+	else
+		status = nfsd4_init_copy_res(copy, sync);
+
+	fput(copy->fh_src);
+	fput(copy->fh_dst);
+	return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;
+	dst->cp_consecutive = src->cp_consecutive;
+	dst->cp_synchronous = src->cp_synchronous;
+	memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+	/* skipping nfsd4_callback */
+	memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+	dst->net = src->net;
+	dst->cp_clp = src->cp_clp;
+	atomic_inc(&dst->cp_clp->cl_refcount);
+	dst->fh_dst = get_file(src->fh_dst);
+	dst->fh_src = get_file(src->fh_src);
+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
+{
+	fput(copy->fh_dst);
+	fput(copy->fh_src);
+	if (remove) {
+		spin_lock(&copy->cp_clp->async_lock);
+		list_del(&copy->copies);
+		spin_unlock(&copy->cp_clp->async_lock);
+	}
+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}
+
+static int nfsd4_do_async_copy(void *data)
+{
+	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
+	struct nfsd4_copy *cb_copy;
+
+	copy->nfserr = nfsd4_do_copy(copy, 0);
+	cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+	if (!cb_copy)
+		goto out;
+	memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+	cb_copy->cp_clp = copy->cp_clp;
+	atomic_inc(&cb_copy->cp_clp->cl_refcount);
+	cb_copy->nfserr = copy->nfserr;
+	memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+	nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+			&nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+	nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+	cleanup_async_copy(copy, true);
+	return 0;
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
 	struct nfsd4_copy *copy = &u->copy;
-	struct file *src, *dst;
 	__be32 status;
-	ssize_t bytes;
+	struct nfsd4_copy *async_copy = NULL;
 
-	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-				   &copy->cp_dst_stateid, &dst);
+	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+				   &copy->fh_src, &copy->cp_dst_stateid,
+				   &copy->fh_dst);
 	if (status)
 		goto out;
 
-	bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-			dst, copy->cp_dst_pos, copy->cp_count);
-
-	if (bytes < 0)
-		status = nfserrno(bytes);
-	else {
-		copy->cp_res.wr_bytes_written = bytes;
-		copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-		copy->cp_consecutive = 1;
-		copy->cp_synchronous = 1;
-		gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-		status = nfs_ok;
+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	/* for now disable asynchronous copy feature */
+	copy->cp_synchronous = 1;
+	if (!copy->cp_synchronous) {
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {
+			status = nfserrno(-ENOMEM);
+			goto out;
+		}
+		dup_copy_fields(copy, async_copy);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+			sizeof(copy->cp_dst_stateid));
+		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+				async_copy, "%s", "copy thread");
+		if (IS_ERR(async_copy->copy_task)) {
+			status = PTR_ERR(async_copy->copy_task);
+			goto out_err;
+		}
+		spin_lock(&async_copy->cp_clp->async_lock);
+		list_add(&async_copy->copies,
+				&async_copy->cp_clp->async_copies);
+		spin_unlock(&async_copy->cp_clp->async_lock);
+		wake_up_process(async_copy->copy_task);
+	} else {
+		status = nfsd4_do_copy(copy, 1);
 	}
-
-	fput(src);
-	fput(dst);
 out:
 	return status;
+out_err:
+	cleanup_async_copy(async_copy, false);
+	goto out;
 }
 
 static __be32

fs/nfsd/nfs4state.c +2 -0

index 150521c..fe665b0 100644
@@ -1790,6 +1790,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 #ifdef CONFIG_NFSD_PNFS
 	INIT_LIST_HEAD(&clp->cl_lo_states);
 #endif
+	INIT_LIST_HEAD(&clp->async_copies);
+	spin_lock_init(&clp->async_lock);
 	spin_lock_init(&clp->cl_lock);
 	rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
 	return clp;

fs/nfsd/state.h +2 -0

index 5c16d35..491030e 100644
@@ -355,6 +355,8 @@ struct nfs4_client {
 	struct rpc_wait_queue	cl_cb_waitq;	/* backchannel callers may */
 						/* wait here for slots */
 	struct net		*net;
+	struct list_head	async_copies;	/* list of async copies */
+	spinlock_t		async_lock;	/* lock for async copies */
 };
 
 /* struct nfs4_client_reset

fs/nfsd/xdr4.h +9 -0

index 4fc0e35..3b6b655 100644
@@ -529,6 +529,15 @@ struct nfsd4_copy {
 	struct nfsd4_callback	cp_cb;
 	__be32			nfserr;
 	struct knfsd_fh		fh;
+
+	struct nfs4_client      *cp_clp;
+
+	struct file             *fh_src;
+	struct file             *fh_dst;
+	struct net              *net;
+
+	struct list_head	copies;
+	struct task_struct	*copy_task;
 };
 
 struct nfsd4_seek {

Signature · 1 line

1.8.3.1

Quoted 10 lines · Olga Kornievskaia

Upon receiving a request for async copy, create a new kthread. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then start the thread and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in a loop over the total number of bytes is needed to copy. In case a failure happens in the middle, we ignore the error and return how much we copied so far.

Might be worth including a comment explaining this (that we ignore the error, and that the client can always retry after a short read if it wants an error), maybe as a comment to the

        (bytes < 0 && !copy->cp_res.wr_bytes_written)

condition in nfsd4_do_copy().

--b.

Quoted 270 lines

Once done creating a workitem for the callback
workqueue and send CB_OFFLOAD with the results.

In the following patches the unique stateid will be generated for the async COPY to return, at that point will enable the async feature.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 176 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   2 +
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   9 +++
 4 files changed, 170 insertions(+), 19 deletions(-)

fs/nfsd/nfs4proc.c +157 -19

index e11494f..91d1544 100644
@@ -35,6 +35,7 @@
 #include <linux/file.h>
 #include <linux/falloc.h>
 #include <linux/slab.h>
+#include <linux/kthread.h>
 
 #include "idmap.h"
 #include "cache.h"

@@ -1083,39 +1084,176 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
      return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+     struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+     atomic_dec(&copy->cp_clp->cl_refcount);
+     kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+                              struct rpc_task *task)
+{
+     return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+     .release = nfsd4_cb_offload_release,
+     .done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+             sizeof(copy->cp_dst_stateid));
+     copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+     copy->cp_consecutive = 1;
+     copy->cp_synchronous = sync;
+     gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+     return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)
+{
+     ssize_t bytes_copied = 0;
+     size_t bytes_total = copy->cp_count;
+     u64 src_pos = copy->cp_src_pos;
+     u64 dst_pos = copy->cp_dst_pos;
+
+     do {
+             bytes_copied = nfsd_copy_file_range(copy->fh_src, src_pos,
+                             copy->fh_dst, dst_pos, bytes_total);
+             if (bytes_copied <= 0)
+                     break;
+             bytes_total -= bytes_copied;
+             copy->cp_res.wr_bytes_written += bytes_copied;
+             src_pos += bytes_copied;
+             dst_pos += bytes_copied;
+     } while (bytes_total > 0 && !copy->cp_synchronous);
+     return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+     __be32 status;
+     ssize_t bytes;
+
+     bytes = _nfsd_copy_file_range(copy);
+     if (bytes < 0 && !copy->cp_res.wr_bytes_written)
+             status = nfserrno(bytes);
+     else
+             status = nfsd4_init_copy_res(copy, sync);
+
+     fput(copy->fh_src);
+     fput(copy->fh_dst);
+     return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+     dst->cp_src_pos = src->cp_src_pos;
+     dst->cp_dst_pos = src->cp_dst_pos;
+     dst->cp_count = src->cp_count;
+     dst->cp_consecutive = src->cp_consecutive;
+     dst->cp_synchronous = src->cp_synchronous;
+     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+     /* skipping nfsd4_callback */
+     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+     dst->net = src->net;
+     dst->cp_clp = src->cp_clp;
+     atomic_inc(&dst->cp_clp->cl_refcount);
+     dst->fh_dst = get_file(src->fh_dst);
+     dst->fh_src = get_file(src->fh_src);
+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
+{
+     fput(copy->fh_dst);
+     fput(copy->fh_src);
+     if (remove) {
+             spin_lock(&copy->cp_clp->async_lock);
+             list_del(&copy->copies);
+             spin_unlock(&copy->cp_clp->async_lock);
+     }
+     atomic_dec(&copy->cp_clp->cl_refcount);
+     kfree(copy);
+}
+
+static int nfsd4_do_async_copy(void *data)
+{
+     struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
+     struct nfsd4_copy *cb_copy;
+
+     copy->nfserr = nfsd4_do_copy(copy, 0);
+     cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+     if (!cb_copy)
+             goto out;
+     memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+     cb_copy->cp_clp = copy->cp_clp;
+     atomic_inc(&cb_copy->cp_clp->cl_refcount);
+     cb_copy->nfserr = copy->nfserr;
+     memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+     nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+                     &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+     nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+     cleanup_async_copy(copy, true);
+     return 0;
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
              union nfsd4_op_u *u)
 {
      struct nfsd4_copy *copy = &u->copy;
-     struct file *src, *dst;
      __be32 status;
-     ssize_t bytes;
+     struct nfsd4_copy *async_copy = NULL;
 
-     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-                                &copy->cp_dst_stateid, &dst);
+     status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+                                &copy->fh_src, &copy->cp_dst_stateid,
+                                &copy->fh_dst);
      if (status)
              goto out;
 
-     bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-                     dst, copy->cp_dst_pos, copy->cp_count);
-
-     if (bytes < 0)
-             status = nfserrno(bytes);
-     else {
-             copy->cp_res.wr_bytes_written = bytes;
-             copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-             copy->cp_consecutive = 1;
-             copy->cp_synchronous = 1;
-             gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-             status = nfs_ok;
+     copy->cp_clp = cstate->clp;
+     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+             sizeof(struct knfsd_fh));
+     copy->net = SVC_NET(rqstp);
+     /* for now disable asynchronous copy feature */
+     copy->cp_synchronous = 1;
+     if (!copy->cp_synchronous) {
+             status = nfsd4_init_copy_res(copy, 0);
+             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+             if (!async_copy) {
+                     status = nfserrno(-ENOMEM);
+                     goto out;
+             }
+             dup_copy_fields(copy, async_copy);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+                     sizeof(copy->cp_dst_stateid));
+             async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+                             async_copy, "%s", "copy thread");
+             if (IS_ERR(async_copy->copy_task)) {
+                     status = PTR_ERR(async_copy->copy_task);
+                     goto out_err;
+             }
+             spin_lock(&async_copy->cp_clp->async_lock);
+             list_add(&async_copy->copies,
+                             &async_copy->cp_clp->async_copies);
+             spin_unlock(&async_copy->cp_clp->async_lock);
+             wake_up_process(async_copy->copy_task);
+     } else {
+             status = nfsd4_do_copy(copy, 1);
      }
-
-     fput(src);
-     fput(dst);
 out:
      return status;
+out_err:
+     cleanup_async_copy(async_copy, false);
+     goto out;
 }
 
 static __be32

fs/nfsd/nfs4state.c +2 -0

index 150521c..fe665b0 100644
@@ -1790,6 +1790,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 #ifdef CONFIG_NFSD_PNFS
      INIT_LIST_HEAD(&clp->cl_lo_states);
 #endif
+     INIT_LIST_HEAD(&clp->async_copies);
+     spin_lock_init(&clp->async_lock);
      spin_lock_init(&clp->cl_lock);
      rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
      return clp;

fs/nfsd/state.h +2 -0

index 5c16d35..491030e 100644
@@ -355,6 +355,8 @@ struct nfs4_client {
      struct rpc_wait_queue   cl_cb_waitq;    /* backchannel callers may */
                                              /* wait here for slots */
      struct net              *net;
+     struct list_head        async_copies;   /* list of async copies */
+     spinlock_t              async_lock;     /* lock for async copies */
 };
 
 /* struct nfs4_client_reset

fs/nfsd/xdr4.h +9 -1

index 4fc0e35..3b6b655 100644
@@ -529,6 +529,15 @@ struct nfsd4_copy {
      struct nfsd4_callback   cp_cb;
      __be32                  nfserr;
      struct knfsd_fh         fh;
+
+     struct nfs4_client      *cp_clp;
+
+     struct file             *fh_src;
+     struct file             *fh_dst;
+     struct net              *net;
+
+     struct list_head        copies;
+     struct task_struct      *copy_task;
 };
 
 struct nfsd4_seek {
--

Quoted 1 line

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Review

Quoted 6 lines · Olga Kornievskaia

(quoted diff) +6 -0

+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+             sizeof(copy->cp_dst_stateid));
+     copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+     copy->cp_consecutive = 1;

The cp_consecutive field was there before you came along, but I don't really see the point of it; all we do is ignore it and always set it to one. I'd just hard-code it into the xdr code:

commit f0a45f80333e

Author: J. Bruce Fields <[email protected]>
Date:   Wed Mar 7 15:37:35 2018 -0500

nfsd: remove unsused "cp_consecutive" field

Signed-off-by J. Bruce Fields <[email protected]>

fs/nfsd/nfs4proc.c +0 -1

index a0bed2b2004d..40dfb759d38d 100644
@@ -1106,7 +1106,6 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
        else {
                copy->cp_res.wr_bytes_written = bytes;
                copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-               copy->cp_consecutive = 1;
                copy->cp_synchronous = 1;
                gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
                status = nfs_ok;

fs/nfsd/nfs4xdr.c +2 -2

index e502fd16246b..9f8ef2c6e508 100644
@@ -1759,7 +1759,7 @@ nfsd4_decode_copy(struct nfsd4_compoundargs *argp, struct nfsd4_copy *copy)
        p = xdr_decode_hyper(p, &copy->cp_src_pos);
        p = xdr_decode_hyper(p, &copy->cp_dst_pos);
        p = xdr_decode_hyper(p, &copy->cp_count);
-       copy->cp_consecutive = be32_to_cpup(p++);
+       p++; /* ca_consecutive: we always do consecutive copies */
        copy->cp_synchronous = be32_to_cpup(p++);
        tmp = be32_to_cpup(p); /* Source server list not supported */
 

@@ -4214,7 +4214,7 @@ nfsd4_encode_copy(struct nfsd4_compoundres *resp, __be32 nfserr,
                return nfserr;
 
        p = xdr_reserve_space(&resp->xdr, 4 + 4);
-       *p++ = cpu_to_be32(copy->cp_consecutive);
+       *p++ = xdr_one; /* cr_consecutive */
        *p++ = cpu_to_be32(copy->cp_synchronous);
        return 0;
 }

fs/nfsd/xdr4.h +0 -0

index bc29511b6405..7cbc129092fe 100644

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

fs/nfsd/nfs4proc.c +7 -0

+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+	memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+	memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+	dst->cp_src_pos = src->cp_src_pos;
+	dst->cp_dst_pos = src->cp_dst_pos;
+	dst->cp_count = src->cp_count;

fs/nfsd/nfs4proc.c +3 -0

+	atomic_dec(&copy->cp_clp->cl_refcount);
+	kfree(copy);
+}

fs/nfsd/nfs4proc.c +10 -0

+	copy->cp_clp = cstate->clp;
+	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+		sizeof(struct knfsd_fh));
+	copy->net = SVC_NET(rqstp);
+	/* for now disable asynchronous copy feature */
+	copy->cp_synchronous = 1;
+	if (!copy->cp_synchronous) {
+		status = nfsd4_init_copy_res(copy, 0);
+		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+		if (!async_copy) {

fs/nfsd/nfs4proc.c +12 -3

+			goto out_err;
+		}
+		spin_lock(&async_copy->cp_clp->async_lock);
+		list_add(&async_copy->copies,
+				&async_copy->cp_clp->async_copies);
+		spin_unlock(&async_copy->cp_clp->async_lock);
+		wake_up_process(async_copy->copy_task);
+	} else {
+		status = nfsd4_do_copy(copy, 1);
 	}
-
-	fput(src);
-	fput(dst);
 out:
 	return status;
+out_err:
+	cleanup_async_copy(async_copy, false);
+	goto out;
 }
 
 static __be32

It looks like "struct nfsd_copy" is being used in at least three different ways: to hold the xdr arguments and reply, to hold the state needed while doing a copy, and to hold the callback arguments. I wonder if the code would be clearer if those were three different data structures.

Quoted 12 lines

fs/nfsd/nfs4proc.c +12 -0

+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+     dst->cp_src_pos = src->cp_src_pos;
+     dst->cp_dst_pos = src->cp_dst_pos;
+     dst->cp_count = src->cp_count;
+     dst->cp_consecutive = src->cp_consecutive;
+     dst->cp_synchronous = src->cp_synchronous;
+     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+     /* skipping nfsd4_callback */
+     memcpy(&dst->fh, &src->fh, sizeof(src->fh));

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1170

Quoted 5 lines

fs/nfsd/nfs4proc.c +5 -0

+     dst->net = src->net;
+     dst->cp_clp = src->cp_clp;
+     atomic_inc(&dst->cp_clp->cl_refcount);
+     dst->fh_dst = get_file(src->fh_dst);
+     dst->fh_src = get_file(src->fh_src);

Just another minor gripe, but: could we name those fd_* or file_* or something instead of fh? I tend to assume "fh" means "nfs filehandle".

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1181

Quoted 12 lines

fs/nfsd/nfs4proc.c +11 -0

+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
+{
+     fput(copy->fh_dst);
+     fput(copy->fh_src);
+     if (remove) {
+             spin_lock(&copy->cp_clp->async_lock);
+             list_del(&copy->copies);
+             spin_unlock(&copy->cp_clp->async_lock);
+     }

I don't understand yet why you need these two cases. Anyway, I'd rather you did this in the caller. So the caller can do either:

        spin_lock()
        list_del()
        spin_unlock()
        cleanup_async_copy()

or just

cleanup_async_copy()

Or define another helper for the first case if you're really doing it a lot. Just don't make me have to remember what that second argument means each time I see it used.

Quoted 3 lines

fs/nfsd/nfs4proc.c +3 -0

+     atomic_dec(&copy->cp_clp->cl_refcount);
+     kfree(copy);
+}

...

Quoted 10 lines

fs/nfsd/nfs4proc.c +10 -0

+     copy->cp_clp = cstate->clp;
+     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+             sizeof(struct knfsd_fh));
+     copy->net = SVC_NET(rqstp);
+     /* for now disable asynchronous copy feature */
+     copy->cp_synchronous = 1;
+     if (!copy->cp_synchronous) {
+             status = nfsd4_init_copy_res(copy, 0);
+             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+             if (!async_copy) {

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1241

Quoted 10 lines

fs/nfsd/nfs4proc.c +10 -0

+                     status = nfserrno(-ENOMEM);
+                     goto out;
+             }
+             dup_copy_fields(copy, async_copy);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+                     sizeof(copy->cp_dst_stateid));
+             async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+                             async_copy, "%s", "copy thread");
+             if (IS_ERR(async_copy->copy_task)) {
+                     status = PTR_ERR(async_copy->copy_task);

status should be an nfs error.

--b.

Quoted 11 lines

fs/nfsd/nfs4proc.c +9 -1

+                     goto out_err;
+             }
+             spin_lock(&async_copy->cp_clp->async_lock);
+             list_add(&async_copy->copies,
+                             &async_copy->cp_clp->async_copies);
+             spin_unlock(&async_copy->cp_clp->async_lock);
+             wake_up_process(async_copy->copy_task);
+     } else {
+             status = nfsd4_do_copy(copy, 1);
      }
-

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1259

Quoted 10 lines

fs/nfsd/nfs4proc.c +3 -2

-     fput(src);
-     fput(dst);
 out:
      return status;
+out_err:
+     cleanup_async_copy(async_copy, false);
+     goto out;
 }
 
 static __be32

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 12 lines · J. Bruce Fields

I don't understand yet why you need these two cases. Anyway, I'd rather you did this in the caller. So the caller can do either:

        spin_lock()
        list_del()
        spin_unlock()
        cleanup_async_copy()

or just

cleanup_async_copy()

Quoted 3 lines

Or define another helper for the first case if you're really doing it a lot. Just don't make me have to remember what that second argument means each time I see it used.

Ok I'll fix it. It was added after I changed where copy was added to the list. In one case, now I needed to do all of the cleanup but the copy wasn't added to the list yet.

It looks like "struct nfsd_copy" is being used in at least three different ways: to hold the xdr arguments and reply, to hold the state needed while doing a copy, and to hold the callback arguments. I wonder if the code would be clearer if those were three different data structures.

Quoted 8 lines

fs/nfsd/nfs4proc.c +7 -0

+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+     memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+     memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+     dst->cp_src_pos = src->cp_src_pos;
+     dst->cp_dst_pos = src->cp_dst_pos;
+     dst->cp_count = src->cp_count;

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1170

Quoted 10 lines

fs/nfsd/nfs4proc.c +10 -0

+     dst->cp_consecutive = src->cp_consecutive;
+     dst->cp_synchronous = src->cp_synchronous;
+     memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+     /* skipping nfsd4_callback */
+     memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+     dst->net = src->net;
+     dst->cp_clp = src->cp_clp;
+     atomic_inc(&dst->cp_clp->cl_refcount);
+     dst->fh_dst = get_file(src->fh_dst);
+     dst->fh_src = get_file(src->fh_src);

Just another minor gripe, but: could we name those fd_* or file_* or something instead of fh? I tend to assume "fh" means "nfs filehandle".

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1181

Quoted 11 lines

fs/nfsd/nfs4proc.c +11 -0

+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
+{
+     fput(copy->fh_dst);
+     fput(copy->fh_src);
+     if (remove) {
+             spin_lock(&copy->cp_clp->async_lock);
+             list_del(&copy->copies);
+             spin_unlock(&copy->cp_clp->async_lock);
+     }

I don't understand yet why you need these two cases. Anyway, I'd rather you did this in the caller. So the caller can do either:

        spin_lock()
        list_del()
        spin_unlock()
        cleanup_async_copy()

or just

cleanup_async_copy()

Or define another helper for the first case if you're really doing it a lot. Just don't make me have to remember what that second argument means each time I see it used.

Quoted 3 lines

fs/nfsd/nfs4proc.c +3 -0

+     atomic_dec(&copy->cp_clp->cl_refcount);
+     kfree(copy);
+}

...

Quoted 10 lines

fs/nfsd/nfs4proc.c +10 -0

+     copy->cp_clp = cstate->clp;
+     memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+             sizeof(struct knfsd_fh));
+     copy->net = SVC_NET(rqstp);
+     /* for now disable asynchronous copy feature */
+     copy->cp_synchronous = 1;
+     if (!copy->cp_synchronous) {
+             status = nfsd4_init_copy_res(copy, 0);
+             async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+             if (!async_copy) {

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1241

Quoted 10 lines

fs/nfsd/nfs4proc.c +10 -0

+                     status = nfserrno(-ENOMEM);
+                     goto out;
+             }
+             dup_copy_fields(copy, async_copy);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+                     sizeof(copy->cp_dst_stateid));
+             async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+                             async_copy, "%s", "copy thread");
+             if (IS_ERR(async_copy->copy_task)) {
+                     status = PTR_ERR(async_copy->copy_task);

status should be an nfs error.

--b.

Quoted 11 lines

fs/nfsd/nfs4proc.c +9 -1

+                     goto out_err;
+             }
+             spin_lock(&async_copy->cp_clp->async_lock);
+             list_add(&async_copy->copies,
+                             &async_copy->cp_clp->async_copies);
+             spin_unlock(&async_copy->cp_clp->async_lock);
+             wake_up_process(async_copy->copy_task);
+     } else {
+             status = nfsd4_do_copy(copy, 1);
      }
-

fs/nfsd/nfs4proc.c static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write) · line 1259

Quoted 10 lines

fs/nfsd/nfs4proc.c +3 -2

-     fput(src);
-     fput(dst);
 out:
      return status;
+out_err:
+     cleanup_async_copy(async_copy, false);
+     goto out;
 }
 
 static __be32

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

fs/nfsd/nfs4state.c static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid) · line 5150

Quoted 9 lines · Olga Kornievskaia

fs/nfsd/nfs4state.c +3 -1

@@ -5143,8 +5144,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
      if (!status && filpp)
              status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
 out:
-     if (s)
+     if (s) {
+             if (!status && cstid)
+                     *cstid = s;
              nfs4_put_stid(s);

Are you sure that's right? Normally you wouldn't drop a reference on something you're returning.

Quoted 13 lines · J. Bruce Fields

fs/nfsd/nfs4state.c static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid) · line 5150

Quoted 9 lines · Olga Kornievskaia

fs/nfsd/nfs4state.c +3 -1

@@ -5143,8 +5144,11 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
      if (!status && filpp)
              status = nfs4_check_file(rqstp, fhp, s, filpp, tmp_file, flags);
 out:
-     if (s)
+     if (s) {
+             if (!status && cstid)
+                     *cstid = s;
              nfs4_put_stid(s);

Are you sure that's right? Normally you wouldn't drop a reference on something you're returning.

Hm. Should it be taken here then? I do up the reference on the stateid later in the code in the nfds4_copy().

Generate a new stateid to be used for reply to the asynchronous COPY (this would also be used later by COPY_NOTIFY as well). Associate the stateid with the parent OPEN/LOCK/DELEG stateid that can be freed during the free of the parent stateid. However, right now deciding to bind the lifetime to when the vfs copy is done. This way don't need to keep the nfsd_net structure for the callback. The drawback is that time copy state information is available for query by OFFLOAD_STATUS is slightly less.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
 fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 +++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 118 insertions(+), 10 deletions(-)

fs/nfsd/netns.h +8 -0

index 36358d4..e83cf6e 100644
@@ -122,6 +122,14 @@ struct nfsd_net {
 
 	wait_queue_head_t ntf_wq;
 	atomic_t ntf_refcnt;
+
+	/*
+	 * clientid and stateid data for construction of net unique COPY
+	 * stateids.
+	 */
+	u32		s2s_cp_cl_id;
+	struct idr	s2s_cp_stateids;
+	spinlock_t	s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */

fs/nfsd/nfs4proc.c +24 -10

index aafd5a58..eb9f528 100644
@@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		  stateid_t *src_stateid, struct file **src,
-		  stateid_t *dst_stateid, struct file **dst)
+		  stateid_t *dst_stateid, struct file **dst,
+		  struct nfs4_stid **stid)
 {
 	__be32 status;
 

@@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
 	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
 					    dst_stateid, WR_STATE, dst, NULL,
-					    NULL);
+					    stid);
 	if (status) {
 		dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
 		goto out_put_src;

@@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 	__be32 status;
 
 	status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-				   &clone->cl_dst_stateid, &dst);
+				   &clone->cl_dst_stateid, &dst, NULL);
 	if (status)
 		goto out;
 

@@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-	memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-		sizeof(copy->cp_dst_stateid));
 	copy->cp_res.wr_stable_how = NFS_UNSTABLE;
 	copy->cp_consecutive = 1;
 	copy->cp_synchronous = sync;

@@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
 	atomic_inc(&dst->cp_clp->cl_refcount);
 	dst->fh_dst = get_file(src->fh_dst);
 	dst->fh_src = get_file(src->fh_src);
+	dst->stid = src->stid;
+	dst->cps = src->cps;
 }
 
 static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
 {
+	list_del(&copy->cps->cp_list);
+	nfs4_free_cp_state(copy->cps);
+	nfs4_put_stid(copy->stid);
 	fput(copy->fh_dst);
 	fput(copy->fh_src);
 	if (remove) {

@@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
 
 	status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
 				   &copy->fh_src, &copy->cp_dst_stateid,
-				   &copy->fh_dst);
+				   &copy->fh_dst, &copy->stid);
 	if (status)
 		goto out;
 

@@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,
 		sizeof(struct knfsd_fh));
 	copy->net = SVC_NET(rqstp);
-	/* for now disable asynchronous copy feature */
-	copy->cp_synchronous = 1;
 	if (!copy->cp_synchronous) {
+		struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
 		status = nfsd4_init_copy_res(copy, 0);
 		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
 		if (!async_copy) {
 			status = nfserrno(-ENOMEM);
 			goto out;
 		}
+		copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+		if (!copy->cps) {
+			status = nfserrno(-ENOMEM);
+			kfree(async_copy);
+			goto out;
+		}
+		/* take a reference on the parent stateid so it's not
+		 * not freed by the copy compound
+		 */
+		refcount_inc(&copy->stid->sc_count);
+		memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+			sizeof(copy->cps->cp_stateid));
 		dup_copy_fields(copy, async_copy);
-		memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-			sizeof(copy->cp_dst_stateid));
 		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
 				async_copy, "%s", "copy thread");
 		if (IS_ERR(async_copy->copy_task)) {

fs/nfsd/nfs4state.c +69 -0

index e05832b..bd76bd1 100644
@@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	/* Will be incremented before return to client: */
 	refcount_set(&stid->sc_count, 1);
 	spin_lock_init(&stid->sc_lock);
+	INIT_LIST_HEAD(&stid->sc_cp_list);
 
 	/*
 	 * It shouldn't be a problem to reuse an opaque stateid value.

@@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+					       struct nfs4_stid *p_stid)
+{
+	struct nfs4_cp_state *cps;
+	int new_id;
+
+	cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+	if (!cps)
+		return NULL;
+	idr_preload(GFP_KERNEL);
+	spin_lock(&nn->s2s_cp_lock);
+	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+	spin_unlock(&nn->s2s_cp_lock);
+	idr_preload_end();
+	if (new_id < 0)
+		goto out_free;
+	cps->cp_stateid.si_opaque.so_id = new_id;
+	cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+	cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+	cps->cp_p_stid = p_stid;
+	spin_lock(&p_stid->sc_lock);
+	list_add(&cps->cp_list, &p_stid->sc_cp_list);
+	spin_unlock(&p_stid->sc_lock);
+
+	return cps;
+out_free:
+	kfree(cps);
+	return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+	struct nfsd_net *nn;
+
+	nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+	spin_lock(&nn->s2s_cp_lock);
+	idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+	spin_unlock(&nn->s2s_cp_lock);
+
+	kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+	struct nfs4_cp_state *cps;
+
+	might_sleep();
+
+	spin_lock(&stid->sc_lock);
+	while (!list_empty(&stid->sc_cp_list)) {
+		cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+				       cp_list);
+		list_del(&cps->cp_list);
+		nfs4_free_cp_state(cps);
+	}
+	spin_unlock(&stid->sc_lock);
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
 	struct nfs4_stid *stid;

@@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
 	}
 	idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
 	spin_unlock(&clp->cl_lock);
+
+	nfs4_free_cp_statelist(s);
+
 	s->sc_free(s);
 	if (fp)
 		put_nfs4_file(fp);

@@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
 	INIT_LIST_HEAD(&nn->close_lru);
 	INIT_LIST_HEAD(&nn->del_recall_lru);
 	spin_lock_init(&nn->client_lock);
+	spin_lock_init(&nn->s2s_cp_lock);
+	idr_init(&nn->s2s_cp_stateids);
 
 	spin_lock_init(&nn->blocked_locks_lock);
 	INIT_LIST_HEAD(&nn->blocked_locks_lru);

fs/nfsd/nfsctl.c +1 -0

index d107b44..63edf68 100644
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
 	nn->nfsd4_grace = 90;
 	nn->clverifier_counter = prandom_u32();
 	nn->clientid_counter = prandom_u32();
+	nn->s2s_cp_cl_id = nn->clientid_counter++;
 
 	atomic_set(&nn->ntf_refcnt, 0);
 	init_waitqueue_head(&nn->ntf_wq);

fs/nfsd/state.h +14 -0

index 9b7d7a0..49709d1 100644
@@ -94,6 +94,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+	struct list_head	sc_cp_list;
 	unsigned char		sc_type;
 	stateid_t		sc_stateid;
 	spinlock_t		sc_lock;

@@ -103,6 +104,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+	stateid_t		cp_stateid;
+	struct list_head	cp_list;	/* per parent nfs4_stid */
+	struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:

@@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
 				  void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);

fs/nfsd/xdr4.h +2 -0

index 3b6b655..5d493b9 100644
@@ -535,6 +535,8 @@ struct nfsd4_copy {
 	struct file             *fh_src;
 	struct file             *fh_dst;
 	struct net              *net;
+	struct nfs4_stid        *stid;
+	struct nfs4_cp_state    *cps;
 
 	struct list_head	copies;
 	struct task_struct	*copy_task;

Signature · 1 line

1.8.3.1

Quoted 8 lines · Olga Kornievskaia

Generate a new stateid to be used for reply to the asynchronous COPY (this would also be used later by COPY_NOTIFY as well). Associate the stateid with the parent OPEN/LOCK/DELEG stateid that can be freed during the free of the parent stateid. However, right now deciding to bind the lifetime to when the vfs copy is done. This way don't need to keep the nfsd_net structure for the callback. The drawback is that time copy state information is available for query by OFFLOAD_STATUS is slightly less.

I don't understand yet the purpose of the parent stateid.

I understand that it's part of the COPY arguments and that we need to check it when we first process the COPY. But what's it used for after that? Why do we need to keep it around?

--b.

Quoted 295 lines

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
 fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 +++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 118 insertions(+), 10 deletions(-)

fs/nfsd/netns.h +8 -0

index 36358d4..e83cf6e 100644
@@ -122,6 +122,14 @@ struct nfsd_net {
 
      wait_queue_head_t ntf_wq;
      atomic_t ntf_refcnt;
+
+     /*
+      * clientid and stateid data for construction of net unique COPY
+      * stateids.
+      */
+     u32             s2s_cp_cl_id;
+     struct idr      s2s_cp_stateids;
+     spinlock_t      s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */

fs/nfsd/nfs4proc.c +24 -10

index aafd5a58..eb9f528 100644
@@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                stateid_t *src_stateid, struct file **src,
-               stateid_t *dst_stateid, struct file **dst)
+               stateid_t *dst_stateid, struct file **dst,
+               struct nfs4_stid **stid)
 {
      __be32 status;
 

@@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
      status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
                                          dst_stateid, WR_STATE, dst, NULL,
-                                         NULL);
+                                         stid);
      if (status) {
              dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
              goto out_put_src;

@@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
      __be32 status;
 
      status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-                                &clone->cl_dst_stateid, &dst);
+                                &clone->cl_dst_stateid, &dst, NULL);
      if (status)
              goto out;
 

@@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-             sizeof(copy->cp_dst_stateid));
      copy->cp_res.wr_stable_how = NFS_UNSTABLE;
      copy->cp_consecutive = 1;
      copy->cp_synchronous = sync;

@@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
      atomic_inc(&dst->cp_clp->cl_refcount);
      dst->fh_dst = get_file(src->fh_dst);
      dst->fh_src = get_file(src->fh_src);
+     dst->stid = src->stid;
+     dst->cps = src->cps;
 }
 
 static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
 {
+     list_del(&copy->cps->cp_list);
+     nfs4_free_cp_state(copy->cps);
+     nfs4_put_stid(copy->stid);
      fput(copy->fh_dst);
      fput(copy->fh_src);
      if (remove) {

@@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
 
      status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
                                 &copy->fh_src, &copy->cp_dst_stateid,
-                                &copy->fh_dst);
+                                &copy->fh_dst, &copy->stid);
      if (status)
              goto out;
 

@@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
      memcpy(&copy->fh, &cstate->current_fh.fh_handle,
              sizeof(struct knfsd_fh));
      copy->net = SVC_NET(rqstp);
-     /* for now disable asynchronous copy feature */
-     copy->cp_synchronous = 1;
      if (!copy->cp_synchronous) {
+             struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
              status = nfsd4_init_copy_res(copy, 0);
              async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
              if (!async_copy) {
                      status = nfserrno(-ENOMEM);
                      goto out;
              }
+             copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+             if (!copy->cps) {
+                     status = nfserrno(-ENOMEM);
+                     kfree(async_copy);
+                     goto out;
+             }
+             /* take a reference on the parent stateid so it's not
+              * not freed by the copy compound
+              */
+             refcount_inc(&copy->stid->sc_count);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+                     sizeof(copy->cps->cp_stateid));
              dup_copy_fields(copy, async_copy);
-             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-                     sizeof(copy->cp_dst_stateid));
              async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
                              async_copy, "%s", "copy thread");
              if (IS_ERR(async_copy->copy_task)) {

fs/nfsd/nfs4state.c +69 -0

index e05832b..bd76bd1 100644
@@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
      /* Will be incremented before return to client: */
      refcount_set(&stid->sc_count, 1);
      spin_lock_init(&stid->sc_lock);
+     INIT_LIST_HEAD(&stid->sc_cp_list);
 
      /*
       * It shouldn't be a problem to reuse an opaque stateid value.

@@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
      return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+                                            struct nfs4_stid *p_stid)
+{
+     struct nfs4_cp_state *cps;
+     int new_id;
+
+     cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+     if (!cps)
+             return NULL;
+     idr_preload(GFP_KERNEL);
+     spin_lock(&nn->s2s_cp_lock);
+     new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+     spin_unlock(&nn->s2s_cp_lock);
+     idr_preload_end();
+     if (new_id < 0)
+             goto out_free;
+     cps->cp_stateid.si_opaque.so_id = new_id;
+     cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+     cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+     cps->cp_p_stid = p_stid;
+     spin_lock(&p_stid->sc_lock);
+     list_add(&cps->cp_list, &p_stid->sc_cp_list);
+     spin_unlock(&p_stid->sc_lock);
+
+     return cps;
+out_free:
+     kfree(cps);
+     return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+     struct nfsd_net *nn;
+
+     nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+     spin_lock(&nn->s2s_cp_lock);
+     idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+     spin_unlock(&nn->s2s_cp_lock);
+
+     kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+     struct nfs4_cp_state *cps;
+
+     might_sleep();
+
+     spin_lock(&stid->sc_lock);
+     while (!list_empty(&stid->sc_cp_list)) {
+             cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+                                    cp_list);
+             list_del(&cps->cp_list);
+             nfs4_free_cp_state(cps);
+     }
+     spin_unlock(&stid->sc_lock);
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
      struct nfs4_stid *stid;

@@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
      }
      idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
      spin_unlock(&clp->cl_lock);
+
+     nfs4_free_cp_statelist(s);
+
      s->sc_free(s);
      if (fp)
              put_nfs4_file(fp);

@@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
      INIT_LIST_HEAD(&nn->close_lru);
      INIT_LIST_HEAD(&nn->del_recall_lru);
      spin_lock_init(&nn->client_lock);
+     spin_lock_init(&nn->s2s_cp_lock);
+     idr_init(&nn->s2s_cp_stateids);
 
      spin_lock_init(&nn->blocked_locks_lock);
      INIT_LIST_HEAD(&nn->blocked_locks_lru);

fs/nfsd/nfsctl.c +1 -0

index d107b44..63edf68 100644
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
      nn->nfsd4_grace = 90;
      nn->clverifier_counter = prandom_u32();
      nn->clientid_counter = prandom_u32();
+     nn->s2s_cp_cl_id = nn->clientid_counter++;
 
      atomic_set(&nn->ntf_refcnt, 0);
      init_waitqueue_head(&nn->ntf_wq);

fs/nfsd/state.h +14 -0

index 9b7d7a0..49709d1 100644
@@ -94,6 +94,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+     struct list_head        sc_cp_list;
      unsigned char           sc_type;
      stateid_t               sc_stateid;
      spinlock_t              sc_lock;

@@ -103,6 +104,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+     stateid_t               cp_stateid;
+     struct list_head        cp_list;        /* per parent nfs4_stid */
+     struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:

@@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
                   struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
                                void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);

fs/nfsd/xdr4.h +2 -1

index 3b6b655..5d493b9 100644
@@ -535,6 +535,8 @@ struct nfsd4_copy {
      struct file             *fh_src;
      struct file             *fh_dst;
      struct net              *net;
+     struct nfs4_stid        *stid;
+     struct nfs4_cp_state    *cps;
 
      struct list_head        copies;
      struct task_struct      *copy_task;
--

Quoted 1 line

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 12 lines · J. Bruce Fields

Quoted 8 lines · Olga Kornievskaia

Generate a new stateid to be used for reply to the asynchronous COPY (this would also be used later by COPY_NOTIFY as well). Associate the stateid with the parent OPEN/LOCK/DELEG stateid that can be freed during the free of the parent stateid. However, right now deciding to bind the lifetime to when the vfs copy is done. This way don't need to keep the nfsd_net structure for the callback. The drawback is that time copy state information is available for query by OFFLOAD_STATUS is slightly less.

I don't understand yet the purpose of the parent stateid.

Quoted 3 lines

I understand that it's part of the COPY arguments and that we need to check it when we first process the COPY. But what's it used for after that? Why do we need to keep it around?

Parent stateid tie is needed for when the "CLOSE" is received. CLOSE only has the parent state. We have a global list of async copies and then each copy has a pointer to parent to provide the match.

Quoted 304 lines

--b.

Quoted 295 lines

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
 fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 +++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 118 insertions(+), 10 deletions(-)

fs/nfsd/netns.h +8 -0

index 36358d4..e83cf6e 100644
@@ -122,6 +122,14 @@ struct nfsd_net {
 
      wait_queue_head_t ntf_wq;
      atomic_t ntf_refcnt;
+
+     /*
+      * clientid and stateid data for construction of net unique COPY
+      * stateids.
+      */
+     u32             s2s_cp_cl_id;
+     struct idr      s2s_cp_stateids;
+     spinlock_t      s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */

fs/nfsd/nfs4proc.c +24 -10

index aafd5a58..eb9f528 100644
@@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                stateid_t *src_stateid, struct file **src,
-               stateid_t *dst_stateid, struct file **dst)
+               stateid_t *dst_stateid, struct file **dst,
+               struct nfs4_stid **stid)
 {
      __be32 status;
 

@@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
      status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
                                          dst_stateid, WR_STATE, dst, NULL,
-                                         NULL);
+                                         stid);
      if (status) {
              dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
              goto out_put_src;

@@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
      __be32 status;
 
      status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-                                &clone->cl_dst_stateid, &dst);
+                                &clone->cl_dst_stateid, &dst, NULL);
      if (status)
              goto out;
 

@@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-             sizeof(copy->cp_dst_stateid));
      copy->cp_res.wr_stable_how = NFS_UNSTABLE;
      copy->cp_consecutive = 1;
      copy->cp_synchronous = sync;

@@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
      atomic_inc(&dst->cp_clp->cl_refcount);
      dst->fh_dst = get_file(src->fh_dst);
      dst->fh_src = get_file(src->fh_src);
+     dst->stid = src->stid;
+     dst->cps = src->cps;
 }
 
 static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
 {
+     list_del(&copy->cps->cp_list);
+     nfs4_free_cp_state(copy->cps);
+     nfs4_put_stid(copy->stid);
      fput(copy->fh_dst);
      fput(copy->fh_src);
      if (remove) {

@@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
 
      status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
                                 &copy->fh_src, &copy->cp_dst_stateid,
-                                &copy->fh_dst);
+                                &copy->fh_dst, &copy->stid);
      if (status)
              goto out;
 

@@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
      memcpy(&copy->fh, &cstate->current_fh.fh_handle,
              sizeof(struct knfsd_fh));
      copy->net = SVC_NET(rqstp);
-     /* for now disable asynchronous copy feature */
-     copy->cp_synchronous = 1;
      if (!copy->cp_synchronous) {
+             struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
              status = nfsd4_init_copy_res(copy, 0);
              async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
              if (!async_copy) {
                      status = nfserrno(-ENOMEM);
                      goto out;
              }
+             copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+             if (!copy->cps) {
+                     status = nfserrno(-ENOMEM);
+                     kfree(async_copy);
+                     goto out;
+             }
+             /* take a reference on the parent stateid so it's not
+              * not freed by the copy compound
+              */
+             refcount_inc(&copy->stid->sc_count);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+                     sizeof(copy->cps->cp_stateid));
              dup_copy_fields(copy, async_copy);
-             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-                     sizeof(copy->cp_dst_stateid));
              async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
                              async_copy, "%s", "copy thread");
              if (IS_ERR(async_copy->copy_task)) {

fs/nfsd/nfs4state.c +69 -0

index e05832b..bd76bd1 100644
@@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
      /* Will be incremented before return to client: */
      refcount_set(&stid->sc_count, 1);
      spin_lock_init(&stid->sc_lock);
+     INIT_LIST_HEAD(&stid->sc_cp_list);
 
      /*
       * It shouldn't be a problem to reuse an opaque stateid value.

@@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
      return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+                                            struct nfs4_stid *p_stid)
+{
+     struct nfs4_cp_state *cps;
+     int new_id;
+
+     cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+     if (!cps)
+             return NULL;
+     idr_preload(GFP_KERNEL);
+     spin_lock(&nn->s2s_cp_lock);
+     new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+     spin_unlock(&nn->s2s_cp_lock);
+     idr_preload_end();
+     if (new_id < 0)
+             goto out_free;
+     cps->cp_stateid.si_opaque.so_id = new_id;
+     cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+     cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+     cps->cp_p_stid = p_stid;
+     spin_lock(&p_stid->sc_lock);
+     list_add(&cps->cp_list, &p_stid->sc_cp_list);
+     spin_unlock(&p_stid->sc_lock);
+
+     return cps;
+out_free:
+     kfree(cps);
+     return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+     struct nfsd_net *nn;
+
+     nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+     spin_lock(&nn->s2s_cp_lock);
+     idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+     spin_unlock(&nn->s2s_cp_lock);
+
+     kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+     struct nfs4_cp_state *cps;
+
+     might_sleep();
+
+     spin_lock(&stid->sc_lock);
+     while (!list_empty(&stid->sc_cp_list)) {
+             cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+                                    cp_list);
+             list_del(&cps->cp_list);
+             nfs4_free_cp_state(cps);
+     }
+     spin_unlock(&stid->sc_lock);
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
      struct nfs4_stid *stid;

@@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
      }
      idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
      spin_unlock(&clp->cl_lock);
+
+     nfs4_free_cp_statelist(s);
+
      s->sc_free(s);
      if (fp)
              put_nfs4_file(fp);

@@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
      INIT_LIST_HEAD(&nn->close_lru);
      INIT_LIST_HEAD(&nn->del_recall_lru);
      spin_lock_init(&nn->client_lock);
+     spin_lock_init(&nn->s2s_cp_lock);
+     idr_init(&nn->s2s_cp_stateids);
 
      spin_lock_init(&nn->blocked_locks_lock);
      INIT_LIST_HEAD(&nn->blocked_locks_lru);

fs/nfsd/nfsctl.c +1 -0

index d107b44..63edf68 100644
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
      nn->nfsd4_grace = 90;
      nn->clverifier_counter = prandom_u32();
      nn->clientid_counter = prandom_u32();
+     nn->s2s_cp_cl_id = nn->clientid_counter++;
 
      atomic_set(&nn->ntf_refcnt, 0);
      init_waitqueue_head(&nn->ntf_wq);

fs/nfsd/state.h +14 -0

index 9b7d7a0..49709d1 100644
@@ -94,6 +94,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+     struct list_head        sc_cp_list;
      unsigned char           sc_type;
      stateid_t               sc_stateid;
      spinlock_t              sc_lock;

@@ -103,6 +104,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+     stateid_t               cp_stateid;
+     struct list_head        cp_list;        /* per parent nfs4_stid */
+     struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:

@@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
                   struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
                                void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);

fs/nfsd/xdr4.h +2 -1

index 3b6b655..5d493b9 100644
@@ -535,6 +535,8 @@ struct nfsd4_copy {
      struct file             *fh_src;
      struct file             *fh_dst;
      struct net              *net;
+     struct nfs4_stid        *stid;
+     struct nfs4_cp_state    *cps;
 
      struct list_head        copies;
      struct task_struct      *copy_task;
--

Quoted 1 line

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 17 lines · Olga Kornievskaia

Quoted 15 lines · J. Bruce Fields

Quoted 8 lines · Olga Kornievskaia

Generate a new stateid to be used for reply to the asynchronous COPY (this would also be used later by COPY_NOTIFY as well). Associate the stateid with the parent OPEN/LOCK/DELEG stateid that can be freed during the free of the parent stateid. However, right now deciding to bind the lifetime to when the vfs copy is done. This way don't need to keep the nfsd_net structure for the callback. The drawback is that time copy state information is available for query by OFFLOAD_STATUS is slightly less.

I don't understand yet the purpose of the parent stateid.

I understand that it's part of the COPY arguments and that we need to check it when we first process the COPY. But what's it used for after that? Why do we need to keep it around?

Quoted 3 lines

Parent stateid tie is needed for when the "CLOSE" is received. CLOSE only has the parent state. We have a global list of async copies and then each copy has a pointer to parent to provide the match.

Why do you need it on CLOSE? Is it that CLOSE is supposed to find any related copies and shut them down or return an error? (I thought we decided that we could leave that to the client--but maybe I've lost track of that discussion.)

--b.

Quoted 304 lines

Quoted 304 lines

--b.

Quoted 295 lines

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/netns.h     |  8 +++++++
 fs/nfsd/nfs4proc.c  | 34 ++++++++++++++++++--------
 fs/nfsd/nfs4state.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 fs/nfsd/nfsctl.c    |  1 +
 fs/nfsd/state.h     | 14 +++++++++++
 fs/nfsd/xdr4.h      |  2 ++
 6 files changed, 118 insertions(+), 10 deletions(-)

fs/nfsd/netns.h +8 -0

index 36358d4..e83cf6e 100644
@@ -122,6 +122,14 @@ struct nfsd_net {
 
      wait_queue_head_t ntf_wq;
      atomic_t ntf_refcnt;
+
+     /*
+      * clientid and stateid data for construction of net unique COPY
+      * stateids.
+      */
+     u32             s2s_cp_cl_id;
+     struct idr      s2s_cp_stateids;
+     spinlock_t      s2s_cp_lock;
 };
 
 /* Simple check to find out if a given net was properly initialized */

fs/nfsd/nfs4proc.c +24 -10

index aafd5a58..eb9f528 100644
@@ -1030,7 +1030,8 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 static __be32
 nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                stateid_t *src_stateid, struct file **src,
-               stateid_t *dst_stateid, struct file **dst)
+               stateid_t *dst_stateid, struct file **dst,
+               struct nfs4_stid **stid)
 {
      __be32 status;
 

@@ -1044,7 +1045,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 
      status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
                                          dst_stateid, WR_STATE, dst, NULL,
-                                         NULL);
+                                         stid);
      if (status) {
              dprintk("NFSD: %s: couldn't process dst stateid!\n", __func__);
              goto out_put_src;

@@ -1075,7 +1076,7 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
      __be32 status;
 
      status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src,
-                                &clone->cl_dst_stateid, &dst);
+                                &clone->cl_dst_stateid, &dst, NULL);
      if (status)
              goto out;
 

@@ -1108,8 +1109,6 @@ static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 
 static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
 {
-     memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-             sizeof(copy->cp_dst_stateid));
      copy->cp_res.wr_stable_how = NFS_UNSTABLE;
      copy->cp_consecutive = 1;
      copy->cp_synchronous = sync;

@@ -1171,10 +1170,15 @@ static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
      atomic_inc(&dst->cp_clp->cl_refcount);
      dst->fh_dst = get_file(src->fh_dst);
      dst->fh_src = get_file(src->fh_src);
+     dst->stid = src->stid;
+     dst->cps = src->cps;
 }
 
 static void cleanup_async_copy(struct nfsd4_copy *copy, bool remove)
 {
+     list_del(&copy->cps->cp_list);
+     nfs4_free_cp_state(copy->cps);
+     nfs4_put_stid(copy->stid);
      fput(copy->fh_dst);
      fput(copy->fh_src);
      if (remove) {

@@ -1218,7 +1222,7 @@ static int nfsd4_do_async_copy(void *data)
 
      status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
                                 &copy->fh_src, &copy->cp_dst_stateid,
-                                &copy->fh_dst);
+                                &copy->fh_dst, &copy->stid);
      if (status)
              goto out;
 

@@ -1226,18 +1230,28 @@ static int nfsd4_do_async_copy(void *data)
      memcpy(&copy->fh, &cstate->current_fh.fh_handle,
              sizeof(struct knfsd_fh));
      copy->net = SVC_NET(rqstp);
-     /* for now disable asynchronous copy feature */
-     copy->cp_synchronous = 1;
      if (!copy->cp_synchronous) {
+             struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+
              status = nfsd4_init_copy_res(copy, 0);
              async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
              if (!async_copy) {
                      status = nfserrno(-ENOMEM);
                      goto out;
              }
+             copy->cps = nfs4_alloc_init_cp_state(nn, copy->stid);
+             if (!copy->cps) {
+                     status = nfserrno(-ENOMEM);
+                     kfree(async_copy);
+                     goto out;
+             }
+             /* take a reference on the parent stateid so it's not
+              * not freed by the copy compound
+              */
+             refcount_inc(&copy->stid->sc_count);
+             memcpy(&copy->cp_res.cb_stateid, &copy->cps->cp_stateid,
+                     sizeof(copy->cps->cp_stateid));
              dup_copy_fields(copy, async_copy);
-             memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
-                     sizeof(copy->cp_dst_stateid));
              async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
                              async_copy, "%s", "copy thread");
              if (IS_ERR(async_copy->copy_task)) {

fs/nfsd/nfs4state.c +69 -0

index e05832b..bd76bd1 100644
@@ -667,6 +667,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
      /* Will be incremented before return to client: */
      refcount_set(&stid->sc_count, 1);
      spin_lock_init(&stid->sc_lock);
+     INIT_LIST_HEAD(&stid->sc_cp_list);
 
      /*
       * It shouldn't be a problem to reuse an opaque stateid value.

@@ -683,6 +684,69 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
      return NULL;
 }
 
+/*
+ * Create a unique stateid_t to represent each COPY. Hang the copy
+ * stateids off the OPEN/LOCK/DELEG stateid from the client open
+ * the source file.
+ */
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn,
+                                            struct nfs4_stid *p_stid)
+{
+     struct nfs4_cp_state *cps;
+     int new_id;
+
+     cps = kzalloc(sizeof(struct nfs4_cp_state), GFP_KERNEL);
+     if (!cps)
+             return NULL;
+     idr_preload(GFP_KERNEL);
+     spin_lock(&nn->s2s_cp_lock);
+     new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, cps, 0, 0, GFP_NOWAIT);
+     spin_unlock(&nn->s2s_cp_lock);
+     idr_preload_end();
+     if (new_id < 0)
+             goto out_free;
+     cps->cp_stateid.si_opaque.so_id = new_id;
+     cps->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
+     cps->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+     cps->cp_p_stid = p_stid;
+     spin_lock(&p_stid->sc_lock);
+     list_add(&cps->cp_list, &p_stid->sc_cp_list);
+     spin_unlock(&p_stid->sc_lock);
+
+     return cps;
+out_free:
+     kfree(cps);
+     return NULL;
+}
+
+void nfs4_free_cp_state(struct nfs4_cp_state *cps)
+{
+     struct nfsd_net *nn;
+
+     nn = net_generic(cps->cp_p_stid->sc_client->net, nfsd_net_id);
+     spin_lock(&nn->s2s_cp_lock);
+     idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.si_opaque.so_id);
+     spin_unlock(&nn->s2s_cp_lock);
+
+     kfree(cps);
+}
+
+static void nfs4_free_cp_statelist(struct nfs4_stid *stid)
+{
+     struct nfs4_cp_state *cps;
+
+     might_sleep();
+
+     spin_lock(&stid->sc_lock);
+     while (!list_empty(&stid->sc_cp_list)) {
+             cps = list_first_entry(&stid->sc_cp_list, struct nfs4_cp_state,
+                                    cp_list);
+             list_del(&cps->cp_list);
+             nfs4_free_cp_state(cps);
+     }
+     spin_unlock(&stid->sc_lock);
+}
+
 static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 {
      struct nfs4_stid *stid;

@@ -828,6 +892,9 @@ static void block_delegations(struct knfsd_fh *fh)
      }
      idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
      spin_unlock(&clp->cl_lock);
+
+     nfs4_free_cp_statelist(s);
+
      s->sc_free(s);
      if (fp)
              put_nfs4_file(fp);

@@ -7115,6 +7182,8 @@ static int nfs4_state_create_net(struct net *net)
      INIT_LIST_HEAD(&nn->close_lru);
      INIT_LIST_HEAD(&nn->del_recall_lru);
      spin_lock_init(&nn->client_lock);
+     spin_lock_init(&nn->s2s_cp_lock);
+     idr_init(&nn->s2s_cp_stateids);
 
      spin_lock_init(&nn->blocked_locks_lock);
      INIT_LIST_HEAD(&nn->blocked_locks_lru);

fs/nfsd/nfsctl.c +1 -0

index d107b44..63edf68 100644
@@ -1241,6 +1241,7 @@ static __net_init int nfsd_init_net(struct net *net)
      nn->nfsd4_grace = 90;
      nn->clverifier_counter = prandom_u32();
      nn->clientid_counter = prandom_u32();
+     nn->s2s_cp_cl_id = nn->clientid_counter++;
 
      atomic_set(&nn->ntf_refcnt, 0);
      init_waitqueue_head(&nn->ntf_wq);

fs/nfsd/state.h +14 -0

index 9b7d7a0..49709d1 100644
@@ -94,6 +94,7 @@ struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+     struct list_head        sc_cp_list;
      unsigned char           sc_type;
      stateid_t               sc_stateid;
      spinlock_t              sc_lock;

@@ -103,6 +104,17 @@ struct nfs4_stid {
 };
 
 /*
+ * Keep a list of stateids issued by the COPY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid. Used for lookup by
+ * OFFLOAD_CANCEL and OFFLOAD_STATUS (as well as COPY_NOTIFY)
+ */
+struct nfs4_cp_state {
+     stateid_t               cp_stateid;
+     struct list_head        cp_list;        /* per parent nfs4_stid */
+     struct nfs4_stid        *cp_p_stid;     /* pointer to parent */
+};
+
+/*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
  * returned by the client:

@@ -612,6 +624,8 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
                   struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
                                void (*sc_free)(struct nfs4_stid *));
+struct nfs4_cp_state *nfs4_alloc_init_cp_state(struct nfsd_net *nn, struct nfs4_stid *p_stid);
+void nfs4_free_cp_state(struct nfs4_cp_state *cps);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);

fs/nfsd/xdr4.h +2 -1

index 3b6b655..5d493b9 100644
@@ -535,6 +535,8 @@ struct nfsd4_copy {
      struct file             *fh_src;
      struct file             *fh_dst;
      struct net              *net;
+     struct nfs4_stid        *stid;
+     struct nfs4_cp_state    *cps;
 
      struct list_head        copies;
      struct task_struct      *copy_task;
--

Quoted 1 line

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html