NFSv4: Don't try to recover NFSv4 locks when they are lost.
Subsystems (3)
PATCH 1/1 NFSv4: Don't try to recover NFSv4 locks when they are lost.
When an NFSv4 client loses contact with the server it can lose any locks that it holds.
Currently when it reconnects to the server it simply tries to reclaim those locks. This might succeed even though some other client has held and released a lock in the mean time. So the first client might think the file is unchanged, but it isn't. This isn't good.
If, when recovery happens, the locks cannot be claimed because some other client still holds the lock, then we get a message in the kernel logs, but the client can still write. So two clients can both think they have a lock and can both write at the same time. This is equally not good.
There was a patch a while ago
http://comments.gmane.org/gmane.linux.nfs/41917
which tried to address some of this, but it didn't seem to go anywhere. That patch would also send a signal to the process. That might be useful but for now this patch just causes writes to fail.
For NFSv4 (unlike v2/v3) there is a strong link between the lock and the write request so we can fairly easily fail any IO of the lock is gone. While some applications might not expect this, it is still safer than allowing the write to succeed.
Because this is a fairly big change in behaviour a module parameter, "recover_locks", is introduced which defaults to true (the current behaviour) but can be set to "false" to tell the client not to try to recover things that were lost.
--
This alternative uses a module parameter which defaults to the current, incorrect, behaviour.
I suspect we don't want that one..
NeilBrown
fs/nfs/nfs3proc.c +4 -2
@@ -826,9 +826,10 @@ static void nfs3_proc_read_setup(struct nfs_read_data *data, struct rpc_message msg->rpc_proc = &nfs3_procedures[NFS3PROC_READ ]; } -static void nfs3_proc_read_rpc_prepare (struct rpc_task *task,struct nfs_read_data *data) +static int nfs3_proc_read_rpc_prepare (struct rpc_task *task,struct nfs_read_data *data) {rpc_call_start (task); +return 0 ; }static int nfs3_write_done (struct rpc_task *task,struct nfs_write_data *data)
@@ -847,9 +848,10 @@ static void nfs3_proc_write_setup(struct nfs_write_data *data, struct rpc_messag msg->rpc_proc = &nfs3_procedures[NFS3PROC_WRITE ]; } -static void nfs3_proc_write_rpc_prepare (struct rpc_task *task,struct nfs_write_data *data) +static int nfs3_proc_write_rpc_prepare (struct rpc_task *task,struct nfs_write_data *data) {rpc_call_start (task); +return 0 ; }static void nfs3_proc_commit_rpc_prepare (struct rpc_task *task,struct nfs_commit_data *data)
fs/nfs/nfs4_fs.h +1 -0
@@ -135,6 +135,7 @@ struct nfs4_lock_state {struct list_head ls_locks;/* Other lock stateids */ struct nfs4_state * ls_state;/* Pointer to open state */ #define NFS_LOCK_INITIALIZED 0 +#define NFS_LOCK_LOST 1unsigned long ls_flags;struct nfs_seqid_counter ls_seqid;nfs4_stateid ls_stateid;
fs/nfs/nfs4proc.c +25 -8
@@ -3905,15 +3905,19 @@ static void nfs4_proc_read_setup(struct nfs_read_data *data, struct rpc_messagenfs41_init_sequence (&data->args.seq_args, &data->res.seq_res,0 ); } -static void nfs4_proc_read_rpc_prepare (struct rpc_task *task,struct nfs_read_data *data) +static int nfs4_proc_read_rpc_prepare (struct rpc_task *task,struct nfs_read_data *data) {if (nfs4_setup_sequence (NFS_SERVER (data->header->inode), &data->args.seq_args, &data->res.seq_res, task)) -return ; -nfs4_set_rw_stateid (&data->args.stateid, data->args.context, - data->args.lock_context,FMODE_READ ); +return 0 ; +if (nfs4_set_rw_stateid (&data->args.stateid, data->args.context, + data->args.lock_context,FMODE_READ ) == -EIO ) +return -EIO ; +if (unlikely (test_bit (NFS_CONTEXT_BAD , &data->args.context->flags))) +return -EIO ; +return 0 ; }static int nfs4_write_done_cb (struct rpc_task *task,struct nfs_write_data *data)
@@ -3988,15 +3992,19 @@ static void nfs4_proc_write_setup(struct nfs_write_data *data, struct rpc_messagnfs41_init_sequence (&data->args.seq_args, &data->res.seq_res,1 ); } -static void nfs4_proc_write_rpc_prepare (struct rpc_task *task,struct nfs_write_data *data) +static int nfs4_proc_write_rpc_prepare (struct rpc_task *task,struct nfs_write_data *data) {if (nfs4_setup_sequence (NFS_SERVER (data->header->inode), &data->args.seq_args, &data->res.seq_res, task)) -return ; -nfs4_set_rw_stateid (&data->args.stateid, data->args.context, - data->args.lock_context,FMODE_WRITE ); +return 0 ; +if (nfs4_set_rw_stateid (&data->args.stateid, data->args.context, + data->args.lock_context,FMODE_WRITE ) == -EIO ) +return -EIO ; +if (unlikely (test_bit (NFS_CONTEXT_BAD , &data->args.context->flags))) +return -EIO ; +return 0 ; }static void nfs4_proc_commit_rpc_prepare (struct rpc_task *task,struct nfs_commit_data *data)
@@ -5378,6 +5386,11 @@ static int nfs4_lock_reclaim(struct nfs4_state *state, struct file_lock *requestreturn err; } +bool recover_locks = true; +module_param (recover_locks, bool,0644 ); +MODULE_PARM_DESC (recovery_locks, +"If the server reports that a lock might be lost, " +"try to recovery it risking corruption." );static int nfs4_lock_expired (struct nfs4_state *state,struct file_lock *request) {struct nfs_server *server =NFS_SERVER (state->inode);
@@ -5389,6 +5402,10 @@ static int nfs4_lock_expired(struct nfs4_state *state, struct file_lock *request err =nfs4_set_lock_state (state, request);if (err !=0 )return err; +if (!recover_locks) { +set_bit (NFS_LOCK_LOST , &request->fl_u.nfs4_fl.owner->ls_flags); +return 0 ; + } do { if (test_bit(NFS_DELEGATED_STATE, &state->flags) != 0) return 0;
fs/nfs/nfs4state.c +11 -3
@@ -998,7 +998,9 @@ static int nfs4_copy_lock_stateid(nfs4_stateid *dst, fl_pid = lockowner->l_pid;spin_lock (&state->state_lock); lsp =__nfs4_find_lock_state (state, fl_owner, fl_pid,NFS4_ANY_LOCK_TYPE ); -if (lsp !=NULL &&test_bit (NFS_LOCK_INITIALIZED , &lsp->ls_flags) !=0 ) { +if (lsp &&test_bit (NFS_LOCK_LOST , &lsp->ls_flags)) + ret = -EIO ; +else if (lsp !=NULL &&test_bit (NFS_LOCK_INITIALIZED , &lsp->ls_flags) !=0 ) {nfs4_stateid_copy (dst, &lsp->ls_stateid); ret =0 ;smp_rmb ();
@@ -1038,11 +1040,17 @@ static int nfs4_copy_open_stateid(nfs4_stateid *dst, struct nfs4_state *state)int nfs4_select_rw_stateid (nfs4_stateid *dst,struct nfs4_state *state,fmode_t fmode,const struct nfs_lockowner *lockowner) { -int ret =0 ; +int ret =nfs4_copy_lock_stateid (dst, state, lockowner); +if (ret == -EIO ) +/* A lost lock - don't even consider delegations */ + goto out;if (nfs4_copy_delegation_stateid (dst, state->inode, fmode)) goto out; - ret =nfs4_copy_lock_stateid (dst, state, lockowner);if (ret != -ENOENT ) +/* nfs4_copy_delegation_stateid() didn't over-write +* dst, so it still has the lock stateid which we now +* choose to use. +*/ goto out; ret =nfs4_copy_open_stateid (dst, state); out:
fs/nfs/proc.c +4 -2
@@ -623,9 +623,10 @@ static void nfs_proc_read_setup(struct nfs_read_data *data, struct rpc_message * msg->rpc_proc = &nfs_procedures[NFSPROC_READ ]; } -static void nfs_proc_read_rpc_prepare (struct rpc_task *task,struct nfs_read_data *data) +static int nfs_proc_read_rpc_prepare (struct rpc_task *task,struct nfs_read_data *data) {rpc_call_start (task); +return 0 ; }static int nfs_write_done (struct rpc_task *task,struct nfs_write_data *data)
@@ -644,9 +645,10 @@ static void nfs_proc_write_setup(struct nfs_write_data *data, struct rpc_message msg->rpc_proc = &nfs_procedures[NFSPROC_WRITE ]; } -static void nfs_proc_write_rpc_prepare (struct rpc_task *task,struct nfs_write_data *data) +static int nfs_proc_write_rpc_prepare (struct rpc_task *task,struct nfs_write_data *data) {rpc_call_start (task); +return 0 ; }static void nfs_proc_commit_rpc_prepare (struct rpc_task *task,struct nfs_commit_data *data)
fs/nfs/read.c +4 -3
@@ -513,9 +513,10 @@ static void nfs_readpage_release_common(void *calldata)void nfs_read_prepare (struct rpc_task *task,void *calldata) {struct nfs_read_data *data = calldata; -NFS_PROTO (data->header->inode)->read_rpc_prepare (task, data); -if (unlikely (test_bit (NFS_CONTEXT_BAD , &data->args.context->flags))) -rpc_exit (task, -EIO ); +int err; + err =NFS_PROTO (data->header->inode)->read_rpc_prepare (task, data); +if (err) +rpc_exit (task, err); }static const struct rpc_call_ops nfs_read_common_ops = {
fs/nfs/write.c +4 -3
@@ -1265,9 +1265,10 @@ EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);void nfs_write_prepare (struct rpc_task *task,void *calldata) {struct nfs_write_data *data = calldata; -NFS_PROTO (data->header->inode)->write_rpc_prepare (task, data); -if (unlikely (test_bit (NFS_CONTEXT_BAD , &data->args.context->flags))) -rpc_exit (task, -EIO ); +int err; + err =NFS_PROTO (data->header->inode)->write_rpc_prepare (task, data); +if (err) +rpc_exit (task, err); }void nfs_commit_prepare (struct rpc_task *task,void *calldata)
include/linux/nfs_xdr.h +2 -2
@@ -1419,12 +1419,12 @@ struct nfs_rpc_ops {void (*read_setup) (struct nfs_read_data *,struct rpc_message *);void (*read_pageio_init)(struct nfs_pageio_descriptor *,struct inode *,const struct nfs_pgio_completion_ops *); -void (*read_rpc_prepare)(struct rpc_task *,struct nfs_read_data *); +int (*read_rpc_prepare)(struct rpc_task *,struct nfs_read_data *);int (*read_done) (struct rpc_task *,struct nfs_read_data *);void (*write_setup) (struct nfs_write_data *,struct rpc_message *);void (*write_pageio_init)(struct nfs_pageio_descriptor *,struct inode *,int ,const struct nfs_pgio_completion_ops *); -void (*write_rpc_prepare)(struct rpc_task *,struct nfs_write_data *); +int (*write_rpc_prepare)(struct rpc_task *,struct nfs_write_data *);int (*write_done) (struct rpc_task *,struct nfs_write_data *);void (*commit_setup) (struct nfs_commit_data *,struct rpc_message *);void (*commit_rpc_prepare)(struct rpc_task *,struct nfs_commit_data *);
Quoted 27 lines · NeilBrown
When an NFSv4 client loses contact with the server it can lose any locks that it holds.
Currently when it reconnects to the server it simply tries to reclaim those locks. This might succeed even though some other client has held and released a lock in the mean time. So the first client might think the file is unchanged, but it isn't. This isn't good.
If, when recovery happens, the locks cannot be claimed because some other client still holds the lock, then we get a message in the kernel logs, but the client can still write. So two clients can both think they have a lock and can both write at the same time. This is equally not good.
There was a patch a while ago
http://comments.gmane.org/gmane.linux.nfs/41917which tried to address some of this, but it didn't seem to go anywhere. That patch would also send a signal to the process. That might be useful but for now this patch just causes writes to fail.
For NFSv4 (unlike v2/v3) there is a strong link between the lock and the write request so we can fairly easily fail any IO of the lock is gone. While some applications might not expect this, it is still safer than allowing the write to succeed.
Quoted 6 lines
Because this is a fairly big change in behaviour a module parameter, "
recover_locks", is introduced which defaults to true (the current behaviour) but can be set to "false" to tell the client not to try to recover things that were lost.Signed-off-by NeilBrown <[email protected]>
Thanks!
Quoted 4 lines
--
This alternative uses a module parameter which defaults to the current, incorrect, behaviour.
I suspect we don't want that one..
Agreed. We also need to document the module parameter, so I'm adding a little blurb in Documentation/kernel-parameters.txt (see attachment).
I'd also like to change the parameter name to "recover_lost_locks" to make it a little more obvious.
Finally, I'd like to move the parameter to fs/nfs/super.c so that we can use the same modprobe.conf 'options' lines for back ports of this patch (yes I strongly suspect we will want to back port this patch to distro kernels).
See attachment.
Cheers
Trond
Signature · 6 lines
Trond Myklebust Linux NFS client maintainer NetApp [email protected] www.netapp.com
From 7722b2fa150b9b7b21218923ad01b757a6653b10 Mon Sep 17 00:00:00 2001
From: Trond Myklebust <[email protected]>
Date: Wed, 4 Sep 2013 10:08:54 -0400
Subject: [PATCH] NFSv4: Document the recover_lost_locks kernel parameter
Rename the new 'recover_locks' kernel parameter to 'recover_lost_locks' and change the default to 'false'. Document why in Documentation/kernel-parameters.txt
Move the 'recover_lost_locks' kernel parameter to fs/nfs/super.c to make it easy to backport to kernels prior to 3.6.x, which don't have a separate NFSv4 module.
--- Documentation/kernel-parameters.txt | 12 ++++++++++++ fs/nfs/nfs4_fs.h | 1 + fs/nfs/nfs4proc.c | 8 +------- fs/nfs/super.c | 8 ++++++++ 4 files changed, 22 insertions(+), 7 deletions(-)
Documentation/kernel-parameters.txt +12 -0
@@ -1847,6 +1847,18 @@ bytes respectively. Such letter suffixes can also be entirely omitted. will be sent. The default is to send the implementation identification information. + + nfs.recover_lost_locks = + [NFSv4] Attempt to recover locks that were lost due + to a lease timeout on the server. Please note that + doing this risks data corruption, since there are + no guarantees that the file will remain unchanged + after the locks are lost. + If you want to enable the kernel legacy behaviour of + attempting to recover these locks, then set this + parameter to '1'. + The default parameter value of '0' causes the kernel + not to attempt recovery of lost locks. nfsd.nfs4_disable_idmapping= [NFSv4] When set to the default of '1', the NFSv4
fs/nfs/nfs4_fs.h +1 -0
@@ -372,6 +372,7 @@ struct dentry *nfs4_try_mount(int, const char *, struct nfs_mount_info *, structextern bool nfs4_disable_idmapping;extern unsigned short max_session_slots;extern unsigned short send_implementation_id; +extern bool recover_lost_locks;#define NFS4_CLIENT_ID_UNIQ_LEN (64)extern char nfs4_client_id_uniquifier[NFS4_CLIENT_ID_UNIQ_LEN ];
fs/nfs/nfs4proc.c +1 -7
@@ -5523,12 +5523,6 @@ static int nfs4_lock_reclaim(struct nfs4_state *state, struct file_lock *requestreturn err; } -bool recover_locks = true; -module_param (recover_locks, bool,0644 ); -MODULE_PARM_DESC (recover_locks, -"If the server reports that a lock might be lost, " -"try to recovery it risking corruption." ); -static int nfs4_lock_expired (struct nfs4_state *state,struct file_lock *request) {struct nfs_server *server =NFS_SERVER (state->inode);
@@ -5540,7 +5534,7 @@ static int nfs4_lock_expired(struct nfs4_state *state, struct file_lock *request err =nfs4_set_lock_state (state, request);if (err !=0 )return err; -if (!recover_locks) { +if (!recover_lost_locks) {set_bit (NFS_LOCK_LOST , &request->fl_u.nfs4_fl.owner->ls_flags);return 0 ; }
fs/nfs/super.c +8 -0
@@ -2758,6 +2758,7 @@ bool nfs4_disable_idmapping = true;unsigned short max_session_slots =NFS4_DEF_SLOT_TABLE_SIZE ;unsigned short send_implementation_id =1 ;char nfs4_client_id_uniquifier[NFS4_CLIENT_ID_UNIQ_LEN ] ="" ; +bool recover_lost_locks = false;EXPORT_SYMBOL_GPL (nfs_callback_set_tcpport);EXPORT_SYMBOL_GPL (nfs_callback_tcpport);
@@ -2766,6 +2767,7 @@ EXPORT_SYMBOL_GPL(nfs4_disable_idmapping);EXPORT_SYMBOL_GPL (max_session_slots);EXPORT_SYMBOL_GPL (send_implementation_id);EXPORT_SYMBOL_GPL (nfs4_client_id_uniquifier); +EXPORT_SYMBOL_GPL (recover_lost_locks);#define NFS_CALLBACK_MAXPORTNR (65535U)
@@ -2803,4 +2805,10 @@ MODULE_PARM_DESC(send_implementation_id, "Send implementation ID with NFSv4.1 exchange_id");MODULE_PARM_DESC (nfs4_unique_id,"nfs_client_id4 uniquifier string" ); +module_param (recover_lost_locks, bool,0644 ); +MODULE_PARM_DESC (recover_lost_locks, +"If the server reports that a lock might be lost, " +"try to recover it risking data corruption." ); + +#endif /* CONFIG_NFS_V4 */
Signature · 1 line
1.8.3.1
Quoted 48 lines · Trond
Quoted 27 lines · NeilBrown
When an NFSv4 client loses contact with the server it can lose any locks that it holds.
Currently when it reconnects to the server it simply tries to reclaim those locks. This might succeed even though some other client has held and released a lock in the mean time. So the first client might think the file is unchanged, but it isn't. This isn't good.
If, when recovery happens, the locks cannot be claimed because some other client still holds the lock, then we get a message in the kernel logs, but the client can still write. So two clients can both think they have a lock and can both write at the same time. This is equally not good.
There was a patch a while ago
http://comments.gmane.org/gmane.linux.nfs/41917which tried to address some of this, but it didn't seem to go anywhere. That patch would also send a signal to the process. That might be useful but for now this patch just causes writes to fail.
For NFSv4 (unlike v2/v3) there is a strong link between the lock and the write request so we can fairly easily fail any IO of the lock is gone. While some applications might not expect this, it is still safer than allowing the write to succeed.
Quoted 6 lines
Because this is a fairly big change in behaviour a module parameter, "
recover_locks", is introduced which defaults to true (the current behaviour) but can be set to "false" to tell the client not to try to recover things that were lost.Signed-off-by NeilBrown <[email protected]>Thanks!
Quoted 4 lines
--
This alternative uses a module parameter which defaults to the current, incorrect, behaviour.
I suspect we don't want that one..Agreed. We also need to document the module parameter, so I'm adding a little blurb in
Documentation/kernel-parameters.txt(see attachment).I'd also like to change the parameter name to "
recover_lost_locks" to make it a little more obvious.
Quoted 6 lines
Finally, I'd like to move the parameter to
fs/nfs/super.cso that we can use the same modprobe.conf 'options' lines for back ports of this patch (yes I strongly suspect we will want to back port this patch to distro kernels).See attachment.
Looks good - thanks.
NeilBrown