OCFS2: anti stale inode for nfs (V6)
PATCH 1/1 OCFS2: anti stale inode for nfs (V6)
For nfs exporting, ocfs2_get_dentry() returns the dentry for fh. ocfs2_get_dentry() may read from disk(when inode not in memory) without any cross cluster lock. this leads to load a stale inode.
this patch fixes above problem.
solution is that in case of inode is not in memory, we get the cluster lock(PR) of alloc inode where the inode in question is allocated from(this causes node on which deletion is done sync the alloc inode) before reading out the inode its self. then we check the bitmap in the group(the inode in question allcated from) to see if the bit is clear. if it's clear then it's stale. if the bit is set, we then check generation as the existing code does.
actually we have to read out the inode in question from disk(not cache) first to know its alloc slot(tells alloc inode) and allot bit(tells alloc group). and if its not stale(by above logic) we read it out using ocfs2_iget(). the second read should from cache.
and also we have to add a per superblock nfs_sync_lock to cover the lock for alloc inode and that for inode in question. this is because ocfs2_get_dentry() and ocfs2_delete_inode() lock on them in reverse order. nfs_sync_lock is locked in EX mode in ocfs2_get_dentry() and in PR mode in ocfs2_delete_inode(). so that mutliple ocfs2_delete_inode() can run concurrently in normal case.
--
dlmglue.c | 45 ++++++++++++++++ dlmglue.h | 2 export.c | 77 ++++++++++++++++++++++++++--- inode.c | 24 ++++++++- inode.h | 1 ocfs2.h | 1 ocfs2_lockid.h | 4 + suballoc.c | 151 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ suballoc.h | 2 9 files changed, 298 insertions(+), 9 deletions(-)
export.c +69 -8
@@ -38,6 +38,7 @@#include "inode.h" #include "buffer_head_io.h" +#include "suballoc.h" struct ocfs2_inode_handle {
@@ -49,29 +50,89 @@ static struct dentry *ocfs2_get_dentry(s struct ocfs2_inode_handle *handle) {struct inode *inode; +struct ocfs2_super *osb =OCFS2_SB (sb); +u64 blkno = handle->ih_blkno; +int status, set;struct dentry *result;mlog_entry ("(0x%p, 0x%p)\n" , sb, handle); -if (handle->ih_blkno ==0 ) { -mlog_errno (-ESTALE ); -return ERR_PTR (-ESTALE ); +if (blkno ==0 ) { +mlog (0 ,"nfs wants inode with blkno: 0\n" ); + result =ERR_PTR (-ESTALE ); + goto bail; + } + + inode =ocfs2_ilookup (sb, blkno); +/* found in-memory inode, goes to check generation */ +if (inode) + goto check_gen; + +/* takes nfs_sync_lock in EX mode */ + status =ocfs2_nfs_sync_lock (osb,1 ); +if (status <0 ) { +mlog (ML_ERROR ,"getting nfs sync lock(EX) failed %d\n" , status); + goto check_err; } - inode =ocfs2_iget (OCFS2_SB (sb), handle->ih_blkno,0 ,0 ); + status =ocfs2_test_inode_bit (osb, blkno, &set); +if (status <0 ) { +if (status == -EINVAL ) { +/* meta block never be re-allocated as data block. +* nfs gives us wrong blkno, we return ESTALE though */ +mlog (0 ,"test inode bit failed %d\n" , status); + status = -ESTALE ; + }else { +mlog (ML_ERROR ,"test inode bit failed %d\n" , status); + } + goto unlock_nfs_sync; + } + +/* allocate bit is clear, inode is a stale inode */ +if (!set) { +mlog (0 ,"inode %llu suballoc bit is clear\n" , blkno); + status = -ESTALE ; + goto unlock_nfs_sync; + } + + inode =ocfs2_iget (osb, blkno,0 ,0 ); + +unlock_nfs_sync: +ocfs2_nfs_sync_unlock (osb,1 ); -if (IS_ERR (inode)) -return (void *)inode; +check_err: +if (status <0 ) { +if (status == -ESTALE ) { +mlog (0 ,"stale inode ino: %llu generation: %u\n" , + blkno, handle->ih_generation); + } + result =ERR_PTR (status); + goto bail; + } +if (IS_ERR (inode)) { +mlog_errno (PTR_ERR (inode)); + result = (void *)inode; + goto bail; + } + +check_gen:if (handle->ih_generation != inode->i_generation) {iput (inode); -return ERR_PTR (-ESTALE ); +mlog (0 ,"stale inode ino: %llu generation: %u\n" , blkno, + handle->ih_generation); + result =ERR_PTR (-ESTALE ); + goto bail; } result =d_obtain_alias (inode); -if (!IS_ERR (result)) +if (!IS_ERR (result)) { result->d_op = &ocfs2_dentry_ops; + }else { +mlog_errno (PTR_ERR (result)); + } +bail:mlog_exit_ptr (result);return result; }
suballoc.c +151 -0
154 lines not on this page · raw message
dlmglue.c +45 -0
81 lines not on this page · raw message
dlmglue.h +2 -0
@@ -115,6 +115,8 @@ void ocfs2_super_unlock(struct ocfs2_sup int ex);int ocfs2_rename_lock (struct ocfs2_super *osb);void ocfs2_rename_unlock (struct ocfs2_super *osb); +int ocfs2_nfs_sync_lock (struct ocfs2_super *osb,int ex); +void ocfs2_nfs_sync_unlock (struct ocfs2_super *osb,int ex);int ocfs2_dentry_lock (struct dentry *dentry,int ex);void ocfs2_dentry_unlock (struct dentry *dentry,int ex);int ocfs2_file_lock (struct file *file,int ex,int trylock);
inode.c +23 -1
@@ -112,6 +112,17 @@ void ocfs2_get_inode_flags(struct ocfs2_ oi->ip_attr |=OCFS2_DIRSYNC_FL ; } +struct inode *ocfs2_ilookup (struct super_block *sb,u64 blkno) +{ +struct ocfs2_find_inode_args args; + + args.fi_blkno = blkno; + args.fi_flags =0 ; + args.fi_ino =ino_from_blkno (sb, blkno); + args.fi_sysfile_type =0 ; + +return ilookup5 (sb, blkno, ocfs2_find_actor, &args); +}struct inode *ocfs2_iget (struct ocfs2_super *osb,u64 blkno,unsigned flags,int sysfile_type) {
@@ -949,6 +960,13 @@ void ocfs2_delete_inode(struct inode *in goto bail; } +/* Lock down the nfs_sync lock in PR mode */ + status =ocfs2_nfs_sync_lock (OCFS2_SB (inode->i_sb),0 ); +if (status <0 ) { +mlog (ML_ERROR ,"getting nfs sync lock(PR) failed %d\n" , status); +ocfs2_cleanup_delete_inode (inode,0 ); + goto bail_unblock; + }/* Lock down the inode. This gives us an up to date view of * it's metadata (for verification), and allows us to * serialize delete_inode on multiple nodes.
@@ -962,7 +980,7 @@ void ocfs2_delete_inode(struct inode *inif (status != -ENOENT )mlog_errno (status);ocfs2_cleanup_delete_inode (inode,0 ); - goto bail_unblock; + goto bail_unlock_nfs_sync; }/* Query the cluster. This will be the final decision made
@@ -1005,6 +1023,10 @@ void ocfs2_delete_inode(struct inode *in bail_unlock_inode:ocfs2_inode_unlock (inode,1 );brelse (di_bh); + +bail_unlock_nfs_sync: +ocfs2_nfs_sync_unlock (OCFS2_SB (inode->i_sb),0 ); + bail_unblock: status =sigprocmask (SIG_SETMASK , &oldset,NULL );if (status <0 )
inode.h +1 -0
@@ -124,6 +124,7 @@ void ocfs2_drop_inode(struct inode *inod/* Flags for ocfs2_iget() */ #define OCFS2_FI_FLAG_SYSFILE 0x1#define OCFS2_FI_FLAG_ORPHAN_RECOVERY 0x2 +struct inode *ocfs2_ilookup (struct super_block *sb,u64 feoff);struct inode *ocfs2_iget (struct ocfs2_super *osb,u64 feoff,unsigned flags,int sysfile_type);int ocfs2_inode_init_private (struct inode *inode);
ocfs2_lockid.h +4 -0
@@ -47,6 +47,7 @@ enum ocfs2_lock_type {OCFS2_LOCK_TYPE_OPEN ,OCFS2_LOCK_TYPE_FLOCK ,OCFS2_LOCK_TYPE_QINFO , +OCFS2_LOCK_TYPE_NFS_SYNC , OCFS2_NUM_LOCK_TYPES };
@@ -81,6 +82,9 @@ static inline char ocfs2_lock_type_char(case OCFS2_LOCK_TYPE_QINFO : c ='Q' ;break ; +case OCFS2_LOCK_TYPE_NFS_SYNC : + c ='Y' ; +break ;default : c ='\0' ; }
suballoc.h +2 -0
@@ -186,4 +186,6 @@ int ocfs2_lock_allocators(struct inode * u32 clusters_to_add, u32 extents_to_split, struct ocfs2_alloc_context **data_ac, struct ocfs2_alloc_context **meta_ac); + +int ocfs2_test_inode_bit (struct ocfs2_super *osb,u64 blkno,int *res);#endif /* _CHAINALLOC_H_ */
ocfs2.h +1 -0
@@ -305,6 +305,7 @@ struct ocfs2_superstruct ocfs2_cluster_connection *cconn;struct ocfs2_lock_res osb_super_lockres;struct ocfs2_lock_res osb_rename_lockres; +struct ocfs2_lock_res osb_nfs_sync_lockres;struct ocfs2_dlm_debug *osb_dlm_debug;struct dentry *osb_debug_root;
PATCH 1/1 OCFS2: anti stale inode for nfs (V6)
Quoted 17 lines · wengang wang
For nfs exporting,
ocfs2_get_dentry()returns the dentry for fh.ocfs2_get_dentry()may read from disk(when inode not in memory) without any cross cluster lock. this leads to load a stale inode.this patch fixes above problem.
solution is that in case of inode is not in memory, we get the cluster lock(PR) of alloc inode where the inode in question is allocated from(this causes node on which deletion is done sync the alloc inode) before reading out the inode its self. then we check the bitmap in the group(the inode in question allcated from) to see if the bit is clear. if it's clear then it's stale. if the bit is set, we then check generation as the existing code does.actually we have to read out the inode in question from disk(not cache) first to know its alloc slot(tells alloc inode) and allot bit(tells alloc group). and if its not stale(by above logic) we read it out using
ocfs2_iget(). the second read should from cache.
Quoted 7 lines
and also we have to add a per superblock
nfs_sync_lockto cover the lock for alloc inode and that for inode in question. this is becauseocfs2_get_dentry()andocfs2_delete_inode()lock on them in reverse order.nfs_sync_lockis locked in EX mode inocfs2_get_dentry()and in PR mode inocfs2_delete_inode(). so that mutlipleocfs2_delete_inode()can run concurrently in normal case.Signed-off-by Wengang Wang <[email protected]>
Wengang,
Thanks for working so hard on this. I'm going to poke Mark to
include it. Now we can go back and do the 1.4 patch. Can you cook that up? It should be almost the same.
Joel
Signature · 8 lines
"A narcissist is someone better looking than you are."
- Gore Vidal
Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127PATCH 1/1 OCFS2: anti stale inode for nfs (V6)
Quoted 2 lines
Wengang,
Thanks for working so hard on this.
My pleasure! and I also learnt during the progress:)
I'm going to poke Mark to
Quoted 2 lines
include it. Now we can go back and do the 1.4 patch. Can you cook that up? It should be almost the same.
sure.
well, what about 1.2?
regards,
wengang.
PATCH 1/1 OCFS2: anti stale inode for nfs (V6)
Quoted 11 lines · Wengang Wang
Quoted 2 lines
Wengang,
Thanks for working so hard on this.My pleasure! and I also learnt during the progress:)
I'm going to poke Mark to
Quoted 2 lines
include it. Now we can go back and do the 1.4 patch. Can you cook that up? It should be almost the same.
sure.
well, what about 1.2?
1.2 has a very different scheme, and I don't think we need to
worry about it. 1.4 has been out and stable for a while now.
Joel
Signature · 9 lines
"There are only two ways to live your life. One is as though nothing
is a miracle. The other is as though everything is a miracle."
- Albert Einstein
Joel Becker
Principal Software Developer
Oracle
E-mail: joel.becker at oracle.com
Phone: (650) 506-8127PATCH 1/1 OCFS2: anti stale inode for nfs (V6)
In reply to wengang wangexport.c
Quoted 4 lines · wengang wang
Index:
export.c
===================================================================export.c +0
-0
Can I please get this patch in the standard -p1 strip level?...
Thanks, Mark
--
Mark Fasheh