NFS: Add support for fallocate and llseek
Subsystems (2)
cover NFS: Add support for fallocate and llseek
These patches add in support for the fallocate() and llseek() system calls using the NFS v4.2 operations WRITE_PLUS and SEEK to do the work on the server. This patchset uses constants defined in the matching server patches, so these should probably be applied second.
Questions, comments and thoughts are appreciated!
Anna
Anna Schumaker (3):
NFS: Use WRITE_PLUS for hole punches
NFS: Allow for asynchronous WRITE_PLUS calls
NFS: Implement SEEK
fs/nfs/callback.h | 13 +++ fs/nfs/callback_proc.c | 8 ++ fs/nfs/callback_xdr.c | 54 ++++++++++- fs/nfs/inode.c | 2 + fs/nfs/nfs4_fs.h | 9 ++ fs/nfs/nfs4file.c | 143 +++++++++++++++++++++++++++++ fs/nfs/nfs4proc.c | 68 ++++++++++++++ fs/nfs/nfs4xdr.c | 233 ++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4.h | 4 + include/linux/nfs_xdr.h | 49 ++++++++++ 10 files changed, 580 insertions(+), 3 deletions(-)
Signature · 1 line
1.8.4.1
PATCH 1/3 NFS: Use WRITE_PLUS for hole punches
This patch implements a version of fallocate for NFS v4. In the v4.2 case, we use WRITE_PLUS with DATA_CONTENT_HOLE set to punch a hole in a file. In NFS < v4.2, we fall back to the generic VFS fallocate implementation.
--- fs/nfs/inode.c | 2 + fs/nfs/nfs4_fs.h | 5 ++ fs/nfs/nfs4file.c | 53 ++++++++++++++++++ fs/nfs/nfs4proc.c | 39 +++++++++++++ fs/nfs/nfs4xdr.c | 142 ++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4.h | 3 + include/linux/nfs_xdr.h | 29 ++++++++++ 7 files changed, 273 insertions(+)
fs/nfs/inode.c +2 -0
@@ -680,6 +680,7 @@ struct nfs_lock_context *nfs_get_lock_context(struct nfs_open_context *ctx)kfree (new);return res; } +EXPORT_SYMBOL_GPL (nfs_get_lock_context);void nfs_put_lock_context (struct nfs_lock_context *l_ctx) {
@@ -692,6 +693,7 @@ void nfs_put_lock_context(struct nfs_lock_context *l_ctx)spin_unlock (&inode->i_lock);kfree (l_ctx); } +EXPORT_SYMBOL_GPL (nfs_put_lock_context);/** * nfs_close_context - Common close_context() routine NFSv2/v3
fs/nfs/nfs4_fs.h +5 -0
@@ -352,6 +352,11 @@ nfs4_state_protect_write(struct nfs_client *clp, struct rpc_clnt **clntp, }#endif /* CONFIG_NFS_V4_1 */ +#ifdef CONFIG_NFS_V4_2 +int nfs42_proc_fallocate (struct nfs_server *,struct nfs42_write_plus_args *, +struct nfs42_write_res *,struct rpc_cred *); +#endif /* CONFIG_NFS_V4_2 */ +extern const struct nfs4_minor_version_ops *nfs_v4_minor_ops[];extern const u32 nfs4_fattr_bitmap[3 ];
fs/nfs/nfs4file.c +53 -0
65 lines not on this page · raw message
fs/nfs/nfs4proc.c +39 -0
@@ -7828,6 +7828,45 @@ static bool nfs41_match_stateid(const nfs4_stateid *s1,#endif /* CONFIG_NFS_V4_1 */ +#ifdef CONFIG_NFS_V4_2 +int _nfs42_proc_fallocate (struct nfs_server *server, +struct nfs42_write_plus_args *args, +struct nfs42_write_res *res, +struct rpc_cred *cred) +{ +struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_WRITE_PLUS ], + .rpc_argp = args, + .rpc_resp = res, + .rpc_cred = cred, + }; +int status; + + status =nfs4_call_sync (server->client, server, &msg, + &args->seq_args, &res->seq_res,0 ); +if (status == -NFS4ERR_NOTSUPP ) +return -EOPNOTSUPP ; +return status; +} + +int nfs42_proc_fallocate (struct nfs_server *server, +struct nfs42_write_plus_args *args, +struct nfs42_write_res *res, +struct rpc_cred *cred) +{ +struct nfs4_exception exception = { }; +int err; + +do { + err =nfs4_handle_exception (server, +_nfs42_proc_fallocate (server, args, res, cred), + &exception); + }while (exception.retry); + +return err; +} +#endif /* CONFIG_NFS_V4_2 */ +static bool nfs4_match_stateid (const nfs4_stateid *s1,const nfs4_stateid *s2) {
fs/nfs/nfs4xdr.c +142 -0
184 lines not on this page · raw message
include/linux/nfs4.h +3 -0
@@ -489,6 +489,9 @@ enum {NFSPROC4_CLNT_GETDEVICELIST ,NFSPROC4_CLNT_BIND_CONN_TO_SESSION ,NFSPROC4_CLNT_DESTROY_CLIENTID , + +/* nfs42 */ + NFSPROC4_CLNT_WRITE_PLUS, };/* nfs41 types */
include/linux/nfs_xdr.h +29 -0
@@ -1223,6 +1223,35 @@ struct pnfs_ds_commit_info {#endif /* CONFIG_NFS_V4_1 */ +#ifdef CONFIG_NFS_V4_2 +struct nfs42_write_plus_args { +struct nfs4_sequence_args seq_args; + +struct nfs_fh *wp_fh; +nfs4_stateid wp_stateid; +enum nfs3_stable_how wp_stable; +enum data_content4 wp_content; +u64 wp_offset; +u64 wp_length; +u32 wp_allocated; +}; + +struct nfs42_write_res +{ +struct nfs4_sequence_res seq_res; +unsigned int wr_status; + +bool wr_async; +nfs4_stateid wr_stateid; +u64 wr_bytes_copied; +int wr_committed; +struct nfs_write_verifier wr_verf; + +struct list_head wait_list; +struct completion completion; +}; +#endif +struct nfs_page ;#define NFS_PAGEVEC_SIZE (8U)
Signature · 1 line
1.8.4.1
Re: [PATCH 1/3] NFS: Use WRITE_PLUS for hole punches
Quoted 4 lines · Anna Schumaker
This patch implements a version of fallocate for NFS v4. In the v4.2 case, we use
WRITE_PLUSwithDATA_CONTENT_HOLEset to punch a hole in a file. In NFS < v4.2, we fall back to the generic VFS fallocate implementation.
What generic VFS fallocate implementation do you fall back to? Not only can't I find a call to to it, nor does such a thing even exist.
Re: [PATCH 1/3] NFS: Use WRITE_PLUS for hole punches
Quoted 9 lines · Christoph Hellwig
Quoted 4 lines · Anna Schumaker
This patch implements a version of fallocate for NFS v4. In the v4.2 case, we use
WRITE_PLUSwithDATA_CONTENT_HOLEset to punch a hole in a file. In NFS < v4.2, we fall back to the generic VFS fallocate implementation.What generic VFS fallocate implementation do you fall back to? Not only can't I find a call to to it, nor does such a thing even exist.
Hmm... good question. I'm not sure what I was thinking when I wrote that comment, so I'll remove it for v2 of these patches.
Re: [PATCH 1/3] NFS: Use WRITE_PLUS for hole punches
In reply to Anna Schumakerfs/nfs/nfs4file.c nfs4_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) · line 154
Quoted 8 lines · Anna Schumaker
fs/nfs/nfs4file.c +8
-0+struct nfs42_write_plus_args args = { + .wp_fh =NFS_FH (inode), + .wp_stable =NFS_FILE_SYNC , + .wp_content =NFS4_CONTENT_HOLE , + .wp_offset = offset, + .wp_length = len, + .wp_allocated = (mode ==0 ), + };
After spending some time with draft 21 of the NFSv4.2 spec I don't think we can use WRITE_PLUS for the prealloc mode of fallocate. The problem with the NFS4_CONTENT_HOLE arm of WRITE_PLUS is that it is defined to zero the whole range, while fallocate is defined as being a no-op for parts of the range that already contain data.
In addition we'll also need more sanity checks on the flags argument, there already is a FALLOC_FL_KEEP_SIZE not supportable by the NFS semantics (not that it nessecarily should), and more may be added in the future. Take a a look at the other fallocate implementations tha t have a quick check on the top for flags they don't support:
if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) return -EOPNOTSUPP;
PATCH 2/3 NFS: Allow for asynchronous WRITE_PLUS calls
In reply to Anna SchumakerClients are required to support CB_OFFLOAD for the NFS4_CONTENT_HOLE arm of the WRITE_PLUS union.
--- fs/nfs/callback.h | 13 ++++++++++++ fs/nfs/callback_proc.c | 8 ++++++++ fs/nfs/callback_xdr.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++--- fs/nfs/nfs4_fs.h | 3 +++ fs/nfs/nfs4file.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- fs/nfs/nfs4xdr.c | 2 +- 6 files changed, 129 insertions(+), 5 deletions(-)
fs/nfs/callback.h +13 -0
@@ -187,6 +187,19 @@ extern __be32 nfs4_callback_devicenotify( void *dummy, struct cb_process_state *cps);#endif /* CONFIG_NFS_V4_1 */ + +#ifdef CONFIG_NFS_V4_2 +struct cb_offloadargs { +struct nfs_fh dst_fh; +nfs4_stateid stateid; +struct nfs42_write_res write_res; +}; + +extern __be32 nfs4_callback_offload (struct cb_offloadargs *,void *, +struct cb_process_state *); +int nfs_wake_offload (struct cb_offloadargs *); +#endif /* CONFIG_NFS_V4_2 */ +extern int check_gss_callback_principal (struct nfs_client *,struct svc_rqst *); extern __be32 nfs4_callback_getattr(struct cb_getattrargs *args, struct cb_getattrres *res,
fs/nfs/callback_proc.c +8 -0
@@ -536,3 +536,11 @@ out: return status; }#endif /* CONFIG_NFS_V4_1 */ + +#ifdef CONFIG_NFS_V4_2 +__be32 nfs4_callback_offload (struct cb_offloadargs *args,void *dummy, +struct cb_process_state *cps) +{ +return htonl (nfs_wake_offload (args)); +} +#endif /* CONFIG_NFS_V4_2 */
fs/nfs/callback_xdr.c +51 -3
78 lines not on this page · raw message
fs/nfs/nfs4_fs.h +3 -0
@@ -464,6 +464,9 @@ static inline void nfs4_unregister_sysctl(void)/* nfs4xdr.c */ extern struct rpc_procinfo nfs4_procedures[]; +#if defined(CONFIG_NFS_V4_2 ) +int decode_write_response (struct xdr_stream *,struct nfs42_write_res *); +#endif /* CONFIG_NFS_V4_2 */ struct nfs4_mount_data ;
fs/nfs/nfs4file.c +53 -1
72 lines not on this page · raw message
fs/nfs/nfs4xdr.c +1 -1
@@ -6041,7 +6041,7 @@ static int decode_free_stateid(struct xdr_stream *xdr,#endif /* CONFIG_NFS_V4_1 */ #ifdef CONFIG_NFS_V4_2 -static int decode_write_response (struct xdr_stream *xdr, +int decode_write_response (struct xdr_stream *xdr,struct nfs42_write_res *write_res) {__be32 *p;
Signature · 1 line
1.8.4.1
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 2 lines · Anna Schumaker
Clients are required to support
CB_OFFLOADfor theNFS4_CONTENT_HOLEarm of theWRITE_PLUSunion.
This sounds pretty stupid. Just curious, who got this braindamage into the standard?
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Anna Schumaker
Clients are required to support
CB_OFFLOADfor theNFS4_CONTENT_HOLEarm of theWRITE_PLUSunion.This sounds pretty stupid. Just curious, who got this braindamage into the standard?
It exists because the server vendors were worried that operations such as preallocation and/or hole punching can take a more or less unbounded amount of time due to the 64-bit size. By using an (optional) callback method, the server can free up the RPC slot that would otherwise be kept waiting in the synchronous case.
Cheers
Trond
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 1 line · Trond
It exists because the server vendors were worried that operations such as preallocation and/or hole punching can take a more or less unbounded amount of time due to the 64-bit size. By using an (optional) callback method, the server can free up the RPC slot that would otherwise be kept waiting in the synchronous case.
Must be some horried filesystems on those vendors servers. Both operations should be O(nr_extents), where nr_extents should be extremely low for the preallocation and still reasonably low for hole punches, although users control the allocastion pattern.
There's a reason why these aren't aio operations locally either.
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 10 lines · Hellwig Christoph
Quoted 1 line · Trond
It exists because the server vendors were worried that operations such as preallocation and/or hole punching can take a more or less unbounded amount of time due to the 64-bit size. By using an (optional) callback method, the server can free up the RPC slot that would otherwise be kept waiting in the synchronous case.
Must be some horried filesystems on those vendors servers. Both operations should be
O(nr_extents), wherenr_extentsshould be extremely low for the preallocation and still reasonably low for hole punches, although users control the allocastion pattern.There's a reason why these aren't aio operations locally either.
Imagine someone wanting to punch a 50TB hole in NTFS, for instance. It doesn't have real holes, so you'd end up needing to zero out the existing extents in that region of the file. That will take time, even with an O(nr_extents) algorithm.
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 1 line · Trond
Imagine someone wanting to punch a 50TB hole in NTFS, for instance. It doesn't have real holes, so you'd end up needing to zero out the existing extents in that region of the file. That will take time, even with an
O(nr_extents)algorithm.
That behaviour is not a hole punch, and should not be multiplexed onto a whole punch on the wire command! If such a use case is important enough there should be an equivanet of the SCSI WRITE SAME command which might make some sense to be implemented async.
We'd surely not support it in the Linux nfsd, though.
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 9 lines · Hellwig Christoph
Quoted 1 line · Trond
Imagine someone wanting to punch a 50TB hole in NTFS, for instance. It doesn't have real holes, so you'd end up needing to zero out the existing extents in that region of the file. That will take time, even with an
O(nr_extents)algorithm.That behaviour is not a hole punch, and should not be multiplexed onto a whole punch on the wire command! If such a use case is important enough there should be an equivanet of the SCSI WRITE SAME command which might make some sense to be implemented async.
We'd surely not support it in the Linux nfsd, though.
The current draft spec even allows the client to specify a "pattern" to be written to the "hole". It is mainly there for applications like Oracle, that initialise empty blocks with a non-trivial pattern so that they can do the on-disk equivalent of memory poisoning. By doing it in this way, it allows the more powerful storage arrays to just implement that patterned region as a set of deduplicated blocks instead of actually allocating all the blocks.
You can indeed argue that filesystems that don't have holes and/or deduplication should not implement this operation at all, however the functionality is there in case they do...
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 1 line · Trond
The current draft spec even allows the client to specify a "pattern" to be written to the "hole".
That's indeed the WRITE SAME lookalike then. At least the SCSI people were smart enough to define an unmap bit for "hole punching" and allow the target to reject all other versions if they don't want to support it.
Given that NFS v4.2 isn't finalized I'd very much recommend to
a) properly separate those case s
b) do not make the async version mandatory
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
Quoted 8 lines · Hellwig Christoph
Quoted 1 line · Trond
The current draft spec even allows the client to specify a "pattern" to be written to the "hole".
That's indeed the WRITE SAME lookalike then. At least the SCSI people were smart enough to define an unmap bit for "hole punching" and allow the target to reject all other versions if they don't want to support it.
Quoted 4 lines
Given that NFS v4.2 isn't finalized I'd very much recommend to
a) properly separate those case s
b) do not make the async version mandatory
union write_plus_arg4 switch (data_content4 wpa_content) {
case NFS4_CONTENT_DATA:
data4 wpa_data;
case NFS4_CONTENT_APP_DATA_HOLE:
app_data_hole4 wpa_adh;
case NFS4_CONTENT_HOLE:
data_info4 wpa_hole;
default:
void;
};I suppose we could add cases: NFS4_CONTENT_APP_DATA_HOLE_SYNC, NFS4_CONTENT_HOLE_SYNC to allow the client to specify that it doesn't want asynchronous behaviour (and add error cases to allow the server to specify that it cannot safely do so).
There is already a 'di_allocated' boolean for the NFS4_CONTENT_HOLE case (which is used to distinguish between holes and zeroed extents). The only thing missing there is an error case, AFAICS, to allow the server to return that it doesn't support holes.
Tom, any thoughts on whether or not this kind of change to the spec is doable at this time?
Re: [PATCH 2/3] NFS: Allow for asynchronous WRITE_PLUS calls
In reply to Hellwig ChristophQuoted 8 lines · Hellwig Christoph
Quoted 1 line · Trond
The current draft spec even allows the client to specify a "pattern" to be written to the "hole".
That's indeed the WRITE SAME lookalike then. At least the SCSI people were smart enough to define an unmap bit for "hole punching" and allow the target to reject all other versions if they don't want to support it.
Quoted 4 lines
Given that NFS v4.2 isn't finalized I'd very much recommend to
a) properly separate those case s
b) do not make the async version mandatory
That sounds reasonable to me.
By the way, source is available from https://github.com/loghyr/NFSv4.2,
and Tom takes patches.....
(Or maybe Anna can be talked into that.)
--b.
PATCH 3/3 NFS: Implement SEEK
In reply to Anna SchumakerThe SEEK operation is used when an application makes an lseek call with either the SEEK_HOLE or SEEK_DATA flags set. I fall back on nfs_file_llseek() when other flags are set or in the NFS < 4.2 case.
--- fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4file.c | 38 +++++++++++++++++++++ fs/nfs/nfs4proc.c | 29 ++++++++++++++++ fs/nfs/nfs4xdr.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/nfs4.h | 1 + include/linux/nfs_xdr.h | 20 +++++++++++ 6 files changed, 180 insertions(+)
fs/nfs/nfs4_fs.h +1 -0
@@ -355,6 +355,7 @@ nfs4_state_protect_write(struct nfs_client *clp, struct rpc_clnt **clntp,#ifdef CONFIG_NFS_V4_2 int nfs42_proc_fallocate (struct nfs_server *,struct nfs42_write_plus_args *,struct nfs42_write_res *,struct rpc_cred *); +loff_t nfs42_proc_llseek (struct inode *,nfs4_stateid *,loff_t ,int );#endif /* CONFIG_NFS_V4_2 */ extern const struct nfs4_minor_version_ops *nfs_v4_minor_ops[];
fs/nfs/nfs4file.c +38 -0
@@ -218,10 +218,48 @@ static long nfs42_fallocate(struct file *file, int mode, loff_t offset, loff_t l err =nfs_wait_for_offload (&res);return err; } + +static loff_t nfs42_file_llseek (struct file *filep,loff_t offset,int whence) +{ +nfs4_stateid stateid; +struct nfs_open_context *ctx; +struct inode *inode =file_inode (filep); +loff_t pos; + + pos =nfs42_select_stateid (filep, &stateid,FMODE_READ |FMODE_WRITE , &ctx); +if (pos <0 ) +return pos; + +nfs_wb_all (inode); + pos =nfs42_proc_llseek (inode, &stateid, offset, whence); +if (pos <0 ) +return pos; +return vfs_setpos (filep, pos, inode->i_sb->s_maxbytes); +} + +static loff_t nfs4_file_llseek (struct file *filep,loff_t offset,int whence) +{ +struct nfs_server *server =NFS_SERVER (file_inode (filep)); + +if (server->nfs_client->cl_minorversion <2 ) +return nfs_file_llseek (filep, offset, whence); + +switch (whence) { +case SEEK_HOLE : +case SEEK_DATA : +return nfs42_file_llseek (filep, offset, whence); +default : +return nfs_file_llseek (filep, offset, whence); + } +}#endif /* CONFIG_NFS_V4_2 */ const struct file_operations nfs4_file_operations = { +#ifdef CONFIG_NFS_V4_2 + .llseek = nfs4_file_llseek, +#else .llseek = nfs_file_llseek, +#endif .read = do_sync_read, .write = do_sync_write, .aio_read = nfs_file_read,
fs/nfs/nfs4proc.c +29 -0
@@ -7865,6 +7865,35 @@ int nfs42_proc_fallocate(struct nfs_server *server,return err; } + +loff_t nfs42_proc_llseek (struct inode *inode,nfs4_stateid *stateid, +loff_t offset,int whence) +{ +struct nfs42_seek_args args = { + .sa_fh =NFS_FH (inode), + .sa_stateid = stateid, + .sa_offset = offset, + }; +struct nfs42_seek_res res; +struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SEEK ], + .rpc_argp = &args, + .rpc_resp = &res, + }; +struct nfs_server *server =NFS_SERVER (inode); +int status; + +if (whence ==SEEK_HOLE ) + args.sa_what =NFS4_CONTENT_HOLE ; +else + args.sa_what =NFS4_CONTENT_DATA ; + + status =nfs4_call_sync (server->client, server, &msg, + &args.seq_args, &res.seq_res,0 ); +if (status) +return status; +return res.sr_offset; +}#endif /* CONFIG_NFS_V4_2 */ static bool nfs4_match_stateid(const nfs4_stateid *s1,
fs/nfs/nfs4xdr.c +91 -0
133 lines not on this page · raw message
include/linux/nfs4.h +1 -0
@@ -492,6 +492,7 @@ enum {/* nfs42 */ NFSPROC4_CLNT_WRITE_PLUS , + NFSPROC4_CLNT_SEEK, };/* nfs41 types */
include/linux/nfs_xdr.h +20 -0
@@ -1236,6 +1236,26 @@ struct nfs42_write_plus_args {u32 wp_allocated; }; +struct nfs42_seek_args { +struct nfs4_sequence_args seq_args; + +struct nfs_fh *sa_fh; +nfs4_stateid *sa_stateid; +u64 sa_offset; +u32 sa_what; +}; + +struct nfs42_seek_res { +struct nfs4_sequence_res seq_res; +unsigned int status; + +u32 sr_eof; +u32 sr_whence; +u64 sr_offset; +u64 sr_length; +u32 sr_allocated; +}; +struct nfs42_write_res {struct nfs4_sequence_res seq_res;
Signature · 1 line
1.8.4.1