NFSD support for asynchronous COPY

Upon receiving a request for async copy, create a new kthread. If we get asynchronous request, make sure to copy the needed arguments/state from the stack before starting the copy. Then start the thread and reply back to the client indicating copy is asynchronous.

nfsd_copy_file_range() will copy in a loop over the total number of bytes is needed to copy. In case a failure happens in the middle, we can return an error as well as how much we copied so far. Once done creating a workitem for the callback workqueue and send CB_OFFLOAD with the results.

Signed-off-by Olga Kornievskaia <[email protected]>

---
 fs/nfsd/nfs4proc.c  | 177 ++++++++++++++++++++++++++++++++++++++++++++++------
 fs/nfsd/nfs4state.c |   2 +
 fs/nfsd/state.h     |   2 +
 fs/nfsd/xdr4.h      |   9 +++
 4 files changed, 171 insertions(+), 19 deletions(-)

fs/nfsd/nfs4proc.c +33 -0

index ea698a4..2787c95 100644
@@ -35,6 +35,7 @@
 #include <linux/file.h>
 #include <linux/falloc.h>
 #include <linux/slab.h>
+#include <linux/kthread.h>
 
 #include "idmap.h"
 #include "cache.h"

@@ -1085,39 +1086,177 @@ static int fill_in_write_vector(struct kvec *vec, struct nfsd4_write *write)
 out:
        return status;
 }
+static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
+{
+       struct nfsd4_copy *copy = container_of(cb, struct nfsd4_copy, cp_cb);
+
+       atomic_dec(&copy->cp_clp->cl_refcount);
+       kfree(copy);
+}
+
+static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
+                                struct rpc_task *task)
+{
+       return 1;
+}
+
+static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = {
+       .release = nfsd4_cb_offload_release,
+       .done = nfsd4_cb_offload_done
+};
+
+static int nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)
+{
+       memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+               sizeof(copy->cp_dst_stateid));
+       copy->cp_res.wr_stable_how = NFS_UNSTABLE;
+       copy->cp_consecutive = 1;
+       copy->cp_synchronous = sync;
+       gen_boot_verifier(&copy->cp_res.wr_verifier, copy->net);
+
+       return nfs_ok;
+}
+
+static int _nfsd_copy_file_range(struct nfsd4_copy *copy)

fs/nfsd/nfs4proc.c +115 -19

+               else
+                       bytes_to_copy = bytes_total;
+               bytes_copied = vfs_copy_file_range(copy->fh_src, src_pos,
+                               copy->fh_dst, dst_pos, bytes_to_copy, 0);
+               if (bytes_copied <= 0)
+                       break;
+               bytes_total -= bytes_copied;
+               copy->cp_res.wr_bytes_written += bytes_copied;
+               src_pos += bytes_copied;
+               dst_pos += bytes_copied;
+       } while (bytes_total > 0 && !copy->cp_synchronous);
+       return bytes_copied;
+}
+
+static int nfsd4_do_copy(struct nfsd4_copy *copy, bool sync)
+{
+       __be32 status;
+       ssize_t bytes;
+
+       bytes = _nfsd_copy_file_range(copy);
+       if (bytes < 0)
+               status = nfserrno(bytes);
+       else
+               status = nfsd4_init_copy_res(copy, sync);
+
+       fput(copy->fh_src);
+       fput(copy->fh_dst);
+       return status;
+}
+
+static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
+{
+       memcpy(&dst->cp_src_stateid, &src->cp_src_stateid, sizeof(stateid_t));
+       memcpy(&dst->cp_dst_stateid, &src->cp_dst_stateid, sizeof(stateid_t));
+       dst->cp_src_pos = src->cp_src_pos;
+       dst->cp_dst_pos = src->cp_dst_pos;
+       dst->cp_count = src->cp_count;
+       dst->cp_consecutive = src->cp_consecutive;
+       dst->cp_synchronous = src->cp_synchronous;
+       memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res));
+       /* skipping nfsd4_callback */
+       memcpy(&dst->fh, &src->fh, sizeof(src->fh));
+       dst->net = src->net;
+       dst->cp_clp = src->cp_clp;
+       atomic_inc(&dst->cp_clp->cl_refcount);
+       dst->fh_dst = get_file(src->fh_dst);
+       dst->fh_src = get_file(src->fh_src);
+}
+
+static void cleanup_async_copy(struct nfsd4_copy *copy)
+{
+       fput(copy->fh_dst);
+       fput(copy->fh_src);
+       spin_lock(&copy->cp_clp->async_lock);
+       list_del(&copy->copies);
+       spin_unlock(&copy->cp_clp->async_lock);
+       atomic_dec(&copy->cp_clp->cl_refcount);
+       kfree(copy);
+}
+
+static int nfsd4_do_async_copy(void *data)
+{
+       struct nfsd4_copy *copy = (struct nfsd4_copy *)data;
+       struct nfsd4_copy *cb_copy;
+
+       copy->nfserr = nfsd4_do_copy(copy, 0);
+       cb_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+       if (!cb_copy)
+               goto out;
+       memcpy(&cb_copy->cp_res, &copy->cp_res, sizeof(copy->cp_res));
+       cb_copy->cp_clp = copy->cp_clp;
+       atomic_inc(&cb_copy->cp_clp->cl_refcount);
+       cb_copy->nfserr = copy->nfserr;
+       memcpy(&cb_copy->fh, &copy->fh, sizeof(copy->fh));
+       nfsd4_init_cb(&cb_copy->cp_cb, cb_copy->cp_clp,
+                       &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD);
+       nfsd4_run_cb(&cb_copy->cp_cb);
+out:
+       cleanup_async_copy(copy);
+       return 0;
+}
 
 static __be32
 nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
                union nfsd4_op_u *u)
 {
        struct nfsd4_copy *copy = &u->copy;
-       struct file *src, *dst;
        __be32 status;
-       ssize_t bytes;
+       struct nfsd4_copy *async_copy = NULL;
 
-       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid, &src,
-                                  &copy->cp_dst_stateid, &dst);
+       status = nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,
+                                  &copy->fh_src, &copy->cp_dst_stateid,
+                                  &copy->fh_dst);
        if (status)
                goto out;
 
-       bytes = nfsd_copy_file_range(src, copy->cp_src_pos,
-                       dst, copy->cp_dst_pos, copy->cp_count);
-
-       if (bytes < 0)
-               status = nfserrno(bytes);
-       else {
-               copy->cp_res.wr_bytes_written = bytes;
-               copy->cp_res.wr_stable_how = NFS_UNSTABLE;
-               copy->cp_consecutive = 1;
-               copy->cp_synchronous = 1;
-               gen_boot_verifier(&copy->cp_res.wr_verifier, SVC_NET(rqstp));
-               status = nfs_ok;
+       copy->cp_clp = cstate->clp;
+       memcpy(&copy->fh, &cstate->current_fh.fh_handle,
+               sizeof(struct knfsd_fh));
+       copy->net = SVC_NET(rqstp);
+       if (!copy->cp_synchronous) {
+               status = nfsd4_init_copy_res(copy, 0);
+               async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
+               if (!async_copy) {
+                       status = nfserrno(-ENOMEM);
+                       goto out;
+               }
+               dup_copy_fields(copy, async_copy);
+               memcpy(&copy->cp_res.cb_stateid, &copy->cp_dst_stateid,
+                       sizeof(copy->cp_dst_stateid));
+               spin_lock(&async_copy->cp_clp->async_lock);
+               list_add(&async_copy->copies,
+                               &async_copy->cp_clp->async_copies);
+               spin_unlock(&async_copy->cp_clp->async_lock);
+               async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
+                               async_copy, "%s", "copy thread");
+               if (IS_ERR(async_copy->copy_task)) {
+                       status = PTR_ERR(async_copy->copy_task);
+                       goto out_err_dec;
+               }
+               wake_up_process(async_copy->copy_task);
+       } else {
+               status = nfsd4_do_copy(copy, 1);
        }
-
-       fput(src);
-       fput(dst);
 out:
        return status;
+out_err_dec:
+       cleanup_async_copy(async_copy);
+       goto out;
 }
 
 static __be32

fs/nfsd/nfs4state.c +2 -0

index 0c04f81..d7767a1 100644
@@ -1774,6 +1774,8 @@ static struct nfs4_client *alloc_client(struct xdr_netobj name)
 #ifdef CONFIG_NFSD_PNFS
        INIT_LIST_HEAD(&clp->cl_lo_states);
 #endif
+       INIT_LIST_HEAD(&clp->async_copies);
+       spin_lock_init(&clp->async_lock);
        spin_lock_init(&clp->cl_lock);
        rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table");
        return clp;

fs/nfsd/state.h +2 -0

index f8b0210..9189062 100644
@@ -352,6 +352,8 @@ struct nfs4_client {
        struct rpc_wait_queue   cl_cb_waitq;    /* backchannel callers may */
                                                /* wait here for slots */
        struct net              *net;
+       struct list_head        async_copies;   /* list of async copies */
+       spinlock_t              async_lock;     /* lock for async copies */
 };
 
 /* struct nfs4_client_reset

fs/nfsd/xdr4.h +9 -1

index 9b0c099..0a19954 100644
@@ -529,6 +529,15 @@ struct nfsd4_copy {
        struct nfsd4_callback   cp_cb;
        __be32                  nfserr;
        struct knfsd_fh         fh;
+
+       struct nfs4_client      *cp_clp;
+
+       struct file             *fh_src;
+       struct file             *fh_dst;
+       struct net              *net;
+
+       struct list_head        copies;
+       struct task_struct      *copy_task;
 };
 
 struct nfsd4_seek {
--

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

v5:
1. reimplementing asynchronous copy to use kthreads instead of the workqueue
2. store asynchronous copies in the list of the nfs4_client structure
3. when copying nfsd4_copy datastructure for the async copy refcount the nfs4_client structure as well as struct file src/dst structure
4. add refcount to the nfsd4_copy structure to coordinate with offload_status and offload_cancel
5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to interrupt do_splice and calls kthread_stop to wait for the copy to stop
6. offload_cancel adds MODIFIES_SOMETHING to flags
7. offload_status reports the size of the destination file form i_size_read() instead of before 4MB loop updated chunk size.
8. for async copy call vfs_copy_file_range() with the whole copy size. still keep the loop to do more the MAX_RW_COUNT that do_splice can do.

Olga Kornievskaia (10):
NFSD CB_OFFLOAD xdr
NFSD OFFLOAD_STATUS xdr
NFSD OFFLOAD_CANCEL xdr
NFSD xdr callback stateid in async COPY reply NFSD first draft of async copy
NFSD return nfs4_stid in nfs4_preprocess_stateid_op NFSD create new stateid for async copy
NFSD handle OFFLOAD_CANCEL op
NFSD support OFFLOAD_STATUS
NFSD stop queued async copies on client shutdown

 fs/nfsd/netns.h        |   8 ++
 fs/nfsd/nfs4callback.c |  97 +++++++++++++++
 fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
 fs/nfsd/nfs4state.c    |  77 +++++++++++-
 fs/nfsd/nfs4xdr.c      |  50 ++++++--
 fs/nfsd/nfsctl.c       |   1 +
 fs/nfsd/state.h        |  21 +++-
 fs/nfsd/xdr4.h         |  28 +++++
 fs/nfsd/xdr4cb.h       |  10 ++
 9 files changed, 576 insertions(+), 37 deletions(-)

Signature · 1 line

1.8.3.1

Quoted 17 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

Quoted 3 lines

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

That assumes we're copying into a previously empty file?

--b.

Quoted 39 lines

v5:
1. reimplementing asynchronous copy to use kthreads instead of the workqueue
2. store asynchronous copies in the list of the nfs4_client structure
3. when copying nfsd4_copy datastructure for the async copy refcount the nfs4_client structure as well as struct file src/dst structure
4. add refcount to the nfsd4_copy structure to coordinate with offload_status and offload_cancel
5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to interrupt do_splice and calls kthread_stop to wait for the copy to stop
6. offload_cancel adds MODIFIES_SOMETHING to flags
7. offload_status reports the size of the destination file form i_size_read() instead of before 4MB loop updated chunk size.
8. for async copy call vfs_copy_file_range() with the whole copy size. still keep the loop to do more the MAX_RW_COUNT that do_splice can do.

Olga Kornievskaia (10):
NFSD CB_OFFLOAD xdr
NFSD OFFLOAD_STATUS xdr
NFSD OFFLOAD_CANCEL xdr
NFSD xdr callback stateid in async COPY reply NFSD first draft of async copy
NFSD return nfs4_stid in nfs4_preprocess_stateid_op NFSD create new stateid for async copy
NFSD handle OFFLOAD_CANCEL op
NFSD support OFFLOAD_STATUS
NFSD stop queued async copies on client shutdown

 fs/nfsd/netns.h        |   8 ++
 fs/nfsd/nfs4callback.c |  97 +++++++++++++++
 fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
 fs/nfsd/nfs4state.c    |  77 +++++++++++-
 fs/nfsd/nfs4xdr.c      |  50 ++++++--
 fs/nfsd/nfsctl.c       |   1 +
 fs/nfsd/state.h        |  21 +++-
 fs/nfsd/xdr4.h         |  28 +++++
 fs/nfsd/xdr4cb.h       |  10 ++
 9 files changed, 576 insertions(+), 37 deletions(-)

Quoted 3 lines

--
1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 18 lines · J. Bruce Fields

Quoted 17 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

Quoted 5 lines

Quoted 3 lines

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

That assumes we're copying into a previously empty file?

Sigh. Alright, then it's back to my original solution where I broke everything into 4MB calls and kept track of bytes copies so far.

Quoted 43 lines

--b.

Quoted 39 lines

v5:
1. reimplementing asynchronous copy to use kthreads instead of the workqueue
2. store asynchronous copies in the list of the nfs4_client structure
3. when copying nfsd4_copy datastructure for the async copy refcount the nfs4_client structure as well as struct file src/dst structure
4. add refcount to the nfsd4_copy structure to coordinate with offload_status and offload_cancel
5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to interrupt do_splice and calls kthread_stop to wait for the copy to stop
6. offload_cancel adds MODIFIES_SOMETHING to flags
7. offload_status reports the size of the destination file form i_size_read() instead of before 4MB loop updated chunk size.
8. for async copy call vfs_copy_file_range() with the whole copy size. still keep the loop to do more the MAX_RW_COUNT that do_splice can do.

Olga Kornievskaia (10):
NFSD CB_OFFLOAD xdr
NFSD OFFLOAD_STATUS xdr
NFSD OFFLOAD_CANCEL xdr
NFSD xdr callback stateid in async COPY reply NFSD first draft of async copy
NFSD return nfs4_stid in nfs4_preprocess_stateid_op NFSD create new stateid for async copy
NFSD handle OFFLOAD_CANCEL op
NFSD support OFFLOAD_STATUS
NFSD stop queued async copies on client shutdown

 fs/nfsd/netns.h        |   8 ++
 fs/nfsd/nfs4callback.c |  97 +++++++++++++++
 fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
 fs/nfsd/nfs4state.c    |  77 +++++++++++-
 fs/nfsd/nfs4xdr.c      |  50 ++++++--
 fs/nfsd/nfsctl.c       |   1 +
 fs/nfsd/state.h        |  21 +++-
 fs/nfsd/xdr4.h         |  28 +++++
 fs/nfsd/xdr4cb.h       |  10 ++
 9 files changed, 576 insertions(+), 37 deletions(-)

Quoted 7 lines

Quoted 3 lines

--
1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 23 lines · Olga Kornievskaia

Quoted 22 lines · J. Bruce Fields

Quoted 20 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

Quoted 4 lines

Quoted 1 line

That assumes we're copying into a previously empty file?

Sigh. Alright, then it's back to my original solution where I broke everything into 4MB calls and kept track of bytes copies so far.

Do they have to be 4MB calls? Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

Quoted 46 lines

Quoted 46 lines

--b.

Quoted 42 lines

v5:
1. reimplementing asynchronous copy to use kthreads instead of the workqueue
2. store asynchronous copies in the list of the nfs4_client structure
3. when copying nfsd4_copy datastructure for the async copy refcount the nfs4_client structure as well as struct file src/dst structure
4. add refcount to the nfsd4_copy structure to coordinate with offload_status and offload_cancel
5. offload_cancel/copy_shutdown sets the SIGPENDING in the copy's thread to interrupt do_splice and calls kthread_stop to wait for the copy to stop
6. offload_cancel adds MODIFIES_SOMETHING to flags
7. offload_status reports the size of the destination file form i_size_read() instead of before 4MB loop updated chunk size.
8. for async copy call vfs_copy_file_range() with the whole copy size. still keep the loop to do more the MAX_RW_COUNT that do_splice can do.

Olga Kornievskaia (10):
NFSD CB_OFFLOAD xdr
NFSD OFFLOAD_STATUS xdr
NFSD OFFLOAD_CANCEL xdr
NFSD xdr callback stateid in async COPY reply NFSD first draft of async copy
NFSD return nfs4_stid in nfs4_preprocess_stateid_op NFSD create new stateid for async copy
NFSD handle OFFLOAD_CANCEL op
NFSD support OFFLOAD_STATUS
NFSD stop queued async copies on client shutdown

 fs/nfsd/netns.h        |   8 ++
 fs/nfsd/nfs4callback.c |  97 +++++++++++++++
 fs/nfsd/nfs4proc.c     | 321 ++++++++++++++++++++++++++++++++++++++++++++-----
 fs/nfsd/nfs4state.c    |  77 +++++++++++-
 fs/nfsd/nfs4xdr.c      |  50 ++++++--
 fs/nfsd/nfsctl.c       |   1 +
 fs/nfsd/state.h        |  21 +++-
 fs/nfsd/xdr4.h         |  28 +++++
 fs/nfsd/xdr4cb.h       |  10 ++
 9 files changed, 576 insertions(+), 37 deletions(-)

--
1.8.3.1

Quoted 9 lines

Quoted 4 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 23 lines · Olga Kornievskaia

Quoted 22 lines · J. Bruce Fields

Quoted 20 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

Quoted 4 lines

Quoted 1 line

That assumes we're copying into a previously empty file?

Sigh. Alright, then it's back to my original solution where I broke everything into 4MB calls and kept track of bytes copies so far.

Do they have to be 4MB calls? Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

Quoted 28 lines · Anna Schumaker

Quoted 25 lines · Olga Kornievskaia

Quoted 23 lines · J. Bruce Fields

Quoted 12 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

Quoted 8 lines

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

That assumes we're copying into a previously empty file?

Quoted 4 lines

Quoted 2 lines

Sigh. Alright, then it's back to my original solution where I broke everything into 4MB calls and kept track of bytes copies so far.

Do they have to be 4MB calls? Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

Sure, we could. Do we have reason to believe there's an advantage to larger sizes?

Quoted 29 lines · J. Bruce Fields

Quoted 28 lines · Anna Schumaker

Quoted 25 lines · Olga Kornievskaia

Quoted 23 lines · J. Bruce Fields

Quoted 12 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

Quoted 8 lines

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

That assumes we're copying into a previously empty file?

Quoted 7 lines

Quoted 4 lines

Quoted 2 lines

Sigh. Alright, then it's back to my original solution where I broke everything into 4MB calls and kept track of bytes copies so far.

Do they have to be 4MB calls? Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

Sure, we could. Do we have reason to believe there's an advantage to larger sizes?

I wouldn't think there'd be a large enough performance advantage with a larger size and there'd be worse OFFLOAD_STATUS information. I'm sure there is a setup cost for calling into do_splice() and the cost of doing a function call but I'd like they would be small.

Hi Bruce,

You were asking for performance numbers for asynchronous vs synchronous intra copy. Here's what Jorge reports:

This is using RHEL 7.4 on both the client and server with COMMIT on the same compound as the COPY for the synchronous case. In this case, improvement is achieved for copies larger than 16MB.

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
1KB file: 0.218760585785 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
1KB file: 0.636984395981 seconds

                                                         65.66%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
2KB file: 0.22707760334 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
2KB file: 0.583548688889 seconds

                                                         61.09%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
4KB file: 0.234200882912 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
4KB file: 0.782712388039 seconds

                                                         70.08%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
8KB file: 0.214556503296 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
8KB file: 0.692702102661 seconds

                                                         69.03%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
16KB file: 0.215230226517 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
16KB file: 0.56289691925 seconds

                                                         61.76%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
32KB file: 0.186200523376 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
32KB file: 0.65691485405 seconds

                                                         71.66%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
64KB file: 0.233846497536 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
64KB file: 0.525265741348 seconds

                                                         55.48%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
128KB file: 0.198684954643 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
128KB file: 0.69602959156 seconds

                                                         71.45%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
256KB file: 0.211255192757 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
256KB file: 0.556627941132 seconds

                                                         62.05%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
512KB file: 0.218777489662 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
512KB file: 0.496951031685 seconds

                                                         55.98%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
1MB file: 0.179558849335 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
1MB file: 0.50447602272 seconds

                                                         64.41%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
2MB file: 0.252070856094 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
2MB file: 0.570275163651 seconds

                                                         55.80%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
4MB file: 0.289573478699 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
4MB file: 0.656079149246 seconds

                                                         55.86%

performance degradation by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
8MB file: 0.50943710804 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
8MB file: 0.696055078506 seconds

                                                         26.81%

performance degradation by async

Performance Improvement:
/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
16MB file: 0.920844507217 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
16MB file: 0.817601919174 seconds

                                                         11.21%

performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
32MB file: 1.46817543507 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
32MB file: 1.24578406811 seconds

                                                         15.15%

performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
64MB file: 2.42379112244 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
64MB file: 1.58639280796 seconds

                                                         34.55%

performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
128MB file: 4.16012530327 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
128MB file: 2.58433949947 seconds

                                                         37.88%

performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
256MB file: 7.56400749683 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
256MB file: 4.43859291077 seconds

                                                         41.32%

performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
512MB file: 14.5191983461 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
512MB file: 8.18448216915 seconds

                                                         43.63%

performance improvement by async

/home/mora/logs/nfstest_ssc_20171022201303.log: PASS: SSC copy of
1GB file: 28.7398069143 seconds
/home/mora/logs/nfstest_ssc_20171015171323.log: PASS: SSC copy of
1GB file: 16.1399238825 seconds

                                                         43.84%

performance improvement by async

Quoted 42 lines · Olga Kornievskaia

Quoted 29 lines · J. Bruce Fields

Quoted 28 lines · Anna Schumaker

Quoted 25 lines · Olga Kornievskaia

Quoted 23 lines · J. Bruce Fields

Quoted 12 lines · Olga Kornievskaia

To do asynchronous copies, NFSD creates a new kthread to handle the request. Upon receiving the COPY, it generates a unique copy stateid (stored in a global list for keeping track of state for OFFLOAD_STATUS to be queried by), starts the thread, and replies back to the client. nfsd4_copy arguments that are allocated on the stack are copies for the kthread.

In the async copy handler, it calls into VFS copy_file_range() (for synch we keep the 4MB chunk and requested size for the async copy). If error is encountered it's saved but also we save the amount of data copied so far. Once done, the results are queued for the callback workqueue and sent via CB_OFFLOAD.

Quoted 8 lines

When the server received an OFFLOAD_CANCEL, it will find the kthread running the copy and will send a SIGPENDING and kthread_stop() and it will interrupt the ongoing do_splice() and once vfs returns we are choosing not to send the CB_OFFLOAD back to the client.

When the server receives an OFFLOAD_STATUS, it will find the kthread running the copy and will query the i_size_read() of the associated filehandle of the destination file and return the result.

That assumes we're copying into a previously empty file?

Quoted 7 lines

Quoted 4 lines

Quoted 2 lines

Sigh. Alright, then it's back to my original solution where I broke everything into 4MB calls and kept track of bytes copies so far.

Do they have to be 4MB calls? Assuming clients don't need a super-accurate results, you could probably use a larger copy size and still have decent copy performance.

Sure, we could. Do we have reason to believe there's an advantage to larger sizes?

I wouldn't think there'd be a large enough performance advantage with a larger size and there'd be worse OFFLOAD_STATUS information. I'm sure there is a setup cost for calling into do_splice() and the cost of doing a function call but I'd like they would be small.

Do you have any more comments on the current async patches or should I post a new version? I plan to do the original OFFLOAD_STATUS looping and I'm planning to change server to not send the EINVAL error to the client and instead send a partial copy and 0 when reading past the end of the file.